From d72d274bd9895bdf97d31d56690b7df07fa74ad4 Mon Sep 17 00:00:00 2001 From: historia Date: Tue, 18 Aug 2026 01:56:25 -0400 Subject: fix: timing/punctuation edge cases, process chunks as uncompressed wavs --- tests/test_chunking.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) (limited to 'tests/test_chunking.py') diff --git a/tests/test_chunking.py b/tests/test_chunking.py index da45f65..659a771 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -22,13 +22,34 @@ class SplitIntoChunksTests(unittest.TestCase): self.assertEqual(sum(len(c.split()) for c in chunks), 90) def test_long_sentence_split_keeps_punctuation(self): - # 10 clauses of 5 words each, joined by commas - sentence = ",".join([" ".join(["w"] * 5) for _ in range(10)]) + "." + # 10 clauses of 5 words each, joined by comma+space + sentence = ", ".join([" ".join(["w"] * 5) for _ in range(10)]) + "." chunks = split_into_chunks(sentence, max_words=12) self.assertGreater(len(chunks), 1) self.assertTrue(all(len(c.split()) <= 12 for c in chunks)) self.assertIn(",", chunks[0]) # commas retained for TTS prosody + def test_clause_split_never_breaks_numbers(self): + # Regression: the clause split used to fire at every comma even + # without whitespace, mutating "1,000,000" into "1, 000, 000". + sentence = ("There were exactly 1,000,000 soldiers marching at 12:30, " + + "and they kept marching onward " * 30) + "endlessly." + chunks = split_into_chunks(sentence, max_words=25) + self.assertGreater(len(chunks), 1) + joined = " ".join(chunks) + self.assertIn("1,000,000", joined) + self.assertIn("12:30", joined) + self.assertNotIn("1, 000", joined) + self.assertNotIn("000, 000", joined) + self.assertNotIn("12: 30", joined) + + def test_clause_split_requires_whitespace_after_punctuation(self): + # Run-on clauses without spaces after commas have no split point and + # must stay byte-identical rather than being re-joined with spaces. + sentence = ",".join([" ".join(["w"] * 5) for _ in range(10)]) + "." + chunks = split_into_chunks(sentence, max_words=12) + self.assertEqual(chunks, [sentence]) + def test_single_oversized_sentence_stays_intact(self): sentence = " ".join(["word"] * 30) + "." chunks = split_into_chunks(sentence, max_words=10) -- cgit v1.2.3