diff options
Diffstat (limited to 'converter/chunking.py')
| -rw-r--r-- | converter/chunking.py | 11 |
1 files changed, 8 insertions, 3 deletions
diff --git a/converter/chunking.py b/converter/chunking.py index f649310..0c85adf 100644 --- a/converter/chunking.py +++ b/converter/chunking.py @@ -11,8 +11,10 @@ def split_into_chunks(text: str, max_words: int = config.CHUNK_SIZE_WORDS) -> Li Splits on sentence boundaries. Sentences longer than the limit are split further at clause punctuation (which is kept attached for TTS prosody). - A single sentence with no clause punctuation longer than the limit is - kept intact as one oversized chunk. + Clause splits only happen at whitespace after punctuation, so tokens like + "1,000,000" or "12:30" are never broken apart or re-joined with added + spaces. A single sentence with no usable split point longer than the + limit is kept intact as one oversized chunk. """ if not text.strip(): return [] @@ -32,7 +34,10 @@ def split_into_chunks(text: str, max_words: int = config.CHUNK_SIZE_WORDS) -> Li current_words = 0 # Split long sentences at clause boundaries, keeping punctuation. - parts = re.split(r"(?<=[,;:])\s*", sentence) + # Only split where whitespace already follows the punctuation so + # the reassembled text is byte-identical to the input (no spaces + # injected into "1,000,000" or "12:30"). + parts = re.split(r"(?<=[,;:])\s+", sentence) for part in parts: part_words = len(part.split()) if current_words + part_words <= max_words: |