From 9d4d7ef806c17387af9778725cd65a5e7ed10e39 Mon Sep 17 00:00:00 2001
From: historia <historiavg@proton.me>
Date: Wed, 19 Aug 2026 04:45:28 -0400
Subject: fix: limit chunk size to 250

---
 tests/test_chunking.py | 70 ++++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 60 insertions(+), 10 deletions(-)

(limited to 'tests/test_chunking.py')

diff --git a/tests/test_chunking.py b/tests/test_chunking.py
index d1d95bd..2062b4e 100644
--- a/tests/test_chunking.py
+++ b/tests/test_chunking.py
@@ -1,18 +1,45 @@
 """Tests for text chunking."""
 
 import unittest
+from unittest.mock import patch
 
 from converter import config
 from converter.chunking import split_into_chunks
 
 
 class ChunkSizeDefaultTests(unittest.TestCase):
-    """Guard the default chunk size: each API call is one model generation,
-    and long single generations lose prosody, can turn garbled, and are
-    truncated at the model's token limit (text past it is never spoken)."""
+    """Guard the request-size settings: each API call is one model
+    generation, and the servers silently truncate audio past their caps
+    (~2.5 min faster backend, ~11 min Gradio demo), so both the default
+    chunk size and the hard ceiling must stay well inside that budget."""
 
-    def test_default_chunk_size_within_single_generation_budget(self):
-        self.assertLessEqual(config.CHUNK_SIZE_WORDS, 60)
+    def test_default_chunk_size_within_request_ceiling(self):
+        self.assertLessEqual(config.CHUNK_SIZE_WORDS, config.MAX_REQUEST_WORDS)
+
+    def test_request_ceiling_within_single_generation_budget(self):
+        self.assertLessEqual(config.MAX_REQUEST_WORDS, 300)
+
+    def test_sizes_are_positive(self):
+        self.assertGreaterEqual(config.CHUNK_SIZE_WORDS, 1)
+        self.assertGreaterEqual(config.MAX_REQUEST_WORDS, 1)
+
+
+class RequestCeilingClampTests(unittest.TestCase):
+    def test_oversized_chunk_size_is_clamped_with_warning(self):
+        text = " ".join(f"word{i}" for i in range(30)) + "."
+        with patch.object(config, "MAX_REQUEST_WORDS", 10), \
+                self.assertLogs("converter.chunking", level="WARNING") as logs:
+            chunks = split_into_chunks(text, max_words=5000)
+        self.assertTrue(all(len(chunk.split()) <= 10 for chunk in chunks))
+        self.assertIn("clamped", " ".join(logs.output))
+
+    def test_default_ceiling_clamps_realistic_configuration(self):
+        sentences = " ".join(
+            f"S{i} " + " ".join(["word"] * 8) + "." for i in range(60))
+        chunks = split_into_chunks(sentences, max_words=5000)
+        self.assertGreater(len(chunks), 1)
+        self.assertTrue(all(len(chunk.split()) <= config.MAX_REQUEST_WORDS
+                            for chunk in chunks))
 
 
 class SplitIntoChunksTests(unittest.TestCase):
@@ -54,16 +81,39 @@ class SplitIntoChunksTests(unittest.TestCase):
         self.assertNotIn("12: 30", joined)
 
     def test_clause_split_requires_whitespace_after_punctuation(self):
-        # Run-on clauses without spaces after commas have no split point and
-        # must stay byte-identical rather than being re-joined with spaces.
+        # Run-on clauses without spaces after commas have no clause split
+        # point, so the last-resort word-boundary split fires instead.
+        # Tokens themselves (and numbers like "1,000,000") stay intact.
         sentence = ",".join([" ".join(["w"] * 5) for _ in range(10)]) + "."
         chunks = split_into_chunks(sentence, max_words=12)
-        self.assertEqual(chunks, [sentence])
+        self.assertGreater(len(chunks), 1)
+        self.assertTrue(all(len(chunk.split()) <= 12 for chunk in chunks))
+        tokens = sentence.replace(",", " , ").split()
+        rejoined = " ".join(chunks).replace(",", " , ").split()
+        self.assertEqual(rejoined, tokens)
 
-    def test_single_oversized_sentence_stays_intact(self):
+    def test_single_oversized_sentence_is_word_split(self):
+        # A punctuation-free sentence longer than the limit is split at word
+        # boundaries: the request-size ceiling is a hard limit because the
+        # TTS servers silently truncate oversized generations.
         sentence = " ".join(["word"] * 30) + "."
         chunks = split_into_chunks(sentence, max_words=10)
-        self.assertEqual(chunks, [sentence])
+        self.assertGreater(len(chunks), 1)
+        self.assertTrue(all(len(chunk.split()) <= 10 for chunk in chunks))
+        self.assertEqual(sum(len(chunk.split()) for chunk in chunks), 30)
+
+    def test_word_split_never_breaks_number_tokens(self):
+        # Numbers and other punctuation-bearing tokens are single words and
+        # must never be broken apart by the last-resort word split.
+        sentence = ("There were exactly 1,000,000 soldiers marching at 12:30 "
+                    "and " + "they kept marching onward " * 20) + "endlessly."
+        chunks = split_into_chunks(sentence, max_words=10)
+        self.assertGreater(len(chunks), 1)
+        joined = " ".join(chunks)
+        self.assertIn("1,000,000", joined)
+        self.assertIn("12:30", joined)
+        self.assertNotIn("1, 000", joined)
+        self.assertNotIn("12: 30", joined)
 
 
 if __name__ == "__main__":
-- 
cgit v1.2.3