fix: limit chunk size to 250

1 files changed, 39 insertions, 8 deletions

diff --git a/converter/chunking.py b/converter/chunking.py
index 0c85adf..425765f 100644
--- a/converter/chunking.py
+++ b/converter/chunking.py
@@ -1,21 +1,40 @@
 """Split extracted book text into TTS-sized chunks."""
 
+import logging
 import re
 from typing import List
 
 from . import config
 
+logger = logging.getLogger(__name__)
+
 
 def split_into_chunks(text: str, max_words: int = config.CHUNK_SIZE_WORDS) -> List[str]:
     """Split text into chunks of at most ``max_words`` words.
 
-    Splits on sentence boundaries. Sentences longer than the limit are split
-    further at clause punctuation (which is kept attached for TTS prosody).
-    Clause splits only happen at whitespace after punctuation, so tokens like
-    "1,000,000" or "12:30" are never broken apart or re-joined with added
-    spaces. A single sentence with no usable split point longer than the
-    limit is kept intact as one oversized chunk.
+    ``max_words`` is clamped to ``config.MAX_REQUEST_WORDS``: requests
+    beyond that ceiling are silently truncated by the TTS servers (no
+    error is reported), so chunks larger than the ceiling are never
+    produced regardless of configuration.
+
+    Splits on sentence boundaries. Sentences longer than the limit are
+    split further at clause punctuation (which is kept attached for TTS
+    prosody). Clause splits only happen at whitespace after punctuation,
+    so tokens like "1,000,000" or "12:30" are never broken apart. A piece
+    with no usable punctuation split point longer than the limit is split
+    at word boundaries as a last resort: individual tokens stay intact,
+    but whitespace between them is normalized.
     """
+    if max_words > config.MAX_REQUEST_WORDS:
+        logger.warning(
+            "Requested chunk size of %d words exceeds the %d-word request ceiling; "
+            "larger requests are silently truncated by the TTS servers, so the "
+            "size is clamped to %d words (see MAX_REQUEST_WORDS in converter/config.py)",
+            max_words, config.MAX_REQUEST_WORDS, config.MAX_REQUEST_WORDS)
+        max_words = config.MAX_REQUEST_WORDS
+    if max_words < 1:
+        max_words = 1
+
     if not text.strip():
         return []
 
@@ -35,11 +54,23 @@ def split_into_chunks(text: str, max_words: int = config.CHUNK_SIZE_WORDS) -> Li
 
             # Split long sentences at clause boundaries, keeping punctuation.
             # Only split where whitespace already follows the punctuation so
-            # the reassembled text is byte-identical to the input (no spaces
-            # injected into "1,000,000" or "12:30").
+            # tokens are never broken apart or re-joined with added spaces
+            # (no spaces are injected into "1,000,000" or "12:30").
             parts = re.split(r"(?<=[,;:])\s+", sentence)
             for part in parts:
                 part_words = len(part.split())
+                if part_words > max_words:
+                    # Last resort: no punctuation split point is available,
+                    # so split at word boundaries. Tokens themselves (and
+                    # therefore numbers like "1,000,000") stay intact.
+                    if current_chunk:
+                        chunks.append(current_chunk.strip())
+                        current_chunk = ""
+                        current_words = 0
+                    words = part.split()
+                    for start in range(0, len(words), max_words):
+                        chunks.append(" ".join(words[start:start + max_words]))
+                    continue
                 if current_words + part_words <= max_words:
                     current_chunk += part + " "
                     current_words += part_words