From e2da233cd1baa859a5721542fc8b80d9f3f880e7 Mon Sep 17 00:00:00 2001 From: historia Date: Thu, 10 Sep 2026 00:16:00 -0400 Subject: fix: smart chunking, extraction, and process-safety issues --- app/tests/test_chunking.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) (limited to 'app/tests/test_chunking.py') diff --git a/app/tests/test_chunking.py b/app/tests/test_chunking.py index bcae6b2..5c6f60b 100644 --- a/app/tests/test_chunking.py +++ b/app/tests/test_chunking.py @@ -203,6 +203,47 @@ class SplitIntoChunksTests(unittest.TestCase): self.assertTrue(all(len(c) <= 250 * 8 for c in chunks)) self.assertEqual("".join(chunks), text) + def test_non_spaced_script_respects_the_word_limit(self): + # A punctuation-free CJK token longer than the limit counts one + # word per character: it must be sliced by the word limit too, + # not only by the character cap (which alone would allow chunks + # up to CHUNK_SIZE * 8 characters). + from converter.chunking import _count_words + + text = "字" * 500 + chunks = split_into_chunks(text, max_words=250, smart=True) + self.assertGreater(len(chunks), 1) + self.assertTrue(all(_count_words(c) <= 250 for c in chunks), + [(_count_words(c), c[:20]) for c in chunks]) + self.assertEqual("".join(chunks), text) + + text = "字" * 5000 + chunks = split_into_chunks(text, max_words=250, smart=True) + self.assertTrue(all(_count_words(c) <= 250 for c in chunks), + [(_count_words(c), c[:20]) for c in chunks]) + self.assertEqual("".join(chunks), text) + + def test_oversized_sentence_keeps_quote_state_through_fallback(self): + # An oversized opening sentence of a dialogue must not present + # its mid-quote remainder as a clean boundary: the fallback + # carries the sentence's quote state, so the close of the quote + # stays attached to the words before it in the next chunk. + text = "“" + "go " * 17 + "go. End now.”" + chunks = split_into_chunks(text, max_words=10, smart=True) + self._assert_bounded(chunks, 10) + self._assert_content(text, chunks) + self.assertEqual([len(c.split()) for c in chunks], [10, 10]) + + def test_oversized_sentence_breaks_cleanly_when_quote_closes_mid_sentence(self): + # A quote opened and closed inside one long sentence: the + # clause-split pieces after the closing quote are clean + # boundaries and can be packed together. + text = ("He answered " + "very " * 8 + "“calmly” and then kept " + "talking on and on for a while.") + chunks = split_into_chunks(text, max_words=6, smart=True) + self._assert_bounded(chunks, 6) + self._assert_content(text, chunks) + def test_non_spaced_script_stays_bounded(self): # CJK text: sentence terminators break units without # whitespace, and each ideograph counts as a word. -- cgit v1.2.3