fix: smart chunking, extraction, and process-safety issuesHEADmain

1 files changed, 41 insertions, 0 deletions

diff --git a/app/tests/test_chunking.py b/app/tests/test_chunking.py
index bcae6b2..5c6f60b 100644
--- a/app/tests/test_chunking.py
+++ b/app/tests/test_chunking.py
@@ -203,6 +203,47 @@ class SplitIntoChunksTests(unittest.TestCase):
         self.assertTrue(all(len(c) <= 250 * 8 for c in chunks))
         self.assertEqual("".join(chunks), text)
 
+    def test_non_spaced_script_respects_the_word_limit(self):
+        # A punctuation-free CJK token longer than the limit counts one
+        # word per character: it must be sliced by the word limit too,
+        # not only by the character cap (which alone would allow chunks
+        # up to CHUNK_SIZE * 8 characters).
+        from converter.chunking import _count_words
+
+        text = "字" * 500
+        chunks = split_into_chunks(text, max_words=250, smart=True)
+        self.assertGreater(len(chunks), 1)
+        self.assertTrue(all(_count_words(c) <= 250 for c in chunks),
+                        [(_count_words(c), c[:20]) for c in chunks])
+        self.assertEqual("".join(chunks), text)
+
+        text = "字" * 5000
+        chunks = split_into_chunks(text, max_words=250, smart=True)
+        self.assertTrue(all(_count_words(c) <= 250 for c in chunks),
+                        [(_count_words(c), c[:20]) for c in chunks])
+        self.assertEqual("".join(chunks), text)
+
+    def test_oversized_sentence_keeps_quote_state_through_fallback(self):
+        # An oversized opening sentence of a dialogue must not present
+        # its mid-quote remainder as a clean boundary: the fallback
+        # carries the sentence's quote state, so the close of the quote
+        # stays attached to the words before it in the next chunk.
+        text = "“" + "go " * 17 + "go. End now.”"
+        chunks = split_into_chunks(text, max_words=10, smart=True)
+        self._assert_bounded(chunks, 10)
+        self._assert_content(text, chunks)
+        self.assertEqual([len(c.split()) for c in chunks], [10, 10])
+
+    def test_oversized_sentence_breaks_cleanly_when_quote_closes_mid_sentence(self):
+        # A quote opened and closed inside one long sentence: the
+        # clause-split pieces after the closing quote are clean
+        # boundaries and can be packed together.
+        text = ("He answered " + "very " * 8 + "“calmly” and then kept "
+                "talking on and on for a while.")
+        chunks = split_into_chunks(text, max_words=6, smart=True)
+        self._assert_bounded(chunks, 6)
+        self._assert_content(text, chunks)
+
     def test_non_spaced_script_stays_bounded(self):
         # CJK text: sentence terminators break units without
         # whitespace, and each ideograph counts as a word.