diff options
Diffstat (limited to 'converter/chunking.py')
| -rw-r--r-- | converter/chunking.py | 30 |
1 files changed, 9 insertions, 21 deletions
diff --git a/converter/chunking.py b/converter/chunking.py index 1ce69a3..800b76a 100644 --- a/converter/chunking.py +++ b/converter/chunking.py @@ -2,28 +2,21 @@ import logging import re -from typing import List +from typing import List, Optional from . import config logger = logging.getLogger(__name__) -# Hard ceiling on words per request, regardless of the configured chunk -# size. Both TTS servers silently truncate audio when a single generation -# exceeds its cap (~2.5 min for the faster backend's static KV cache, -# ~11 min for the Gradio demo) without reporting any error, so larger -# requests are always split client-side. Keep a margin below ~300 words -# to survive slow narration on the faster backend. -MAX_REQUEST_WORDS = 250 - -def split_into_chunks(text: str, max_words: int = config.CHUNK_SIZE) -> List[str]: +def split_into_chunks(text: str, max_words: Optional[int] = None) -> List[str]: """Split text into chunks of at most ``max_words`` words. - ``max_words`` is clamped to ``MAX_REQUEST_WORDS``: requests - beyond that ceiling are silently truncated by the TTS servers (no - error is reported), so chunks larger than the ceiling are never - produced regardless of configuration. + ``max_words`` defaults to ``config.CHUNK_SIZE`` (read at call time). + There is no ceiling beyond that setting, but note that the TTS + servers silently truncate audio when a single generation runs too + long without reporting an error, so very large values are at your + own risk (see CHUNK_SIZE in converter/config.py). Splits on sentence boundaries. Sentences longer than the limit are split further at clause punctuation (which is kept attached for TTS @@ -33,13 +26,8 @@ def split_into_chunks(text: str, max_words: int = config.CHUNK_SIZE) -> List[str at word boundaries as a last resort: individual tokens stay intact, but whitespace between them is normalized. """ - if max_words > MAX_REQUEST_WORDS: - logger.warning( - "Requested chunk size of %d words exceeds the %d-word request ceiling; " - "larger requests are silently truncated by the TTS servers, so the " - "size is clamped to %d words (see MAX_REQUEST_WORDS in converter/chunking.py)", - max_words, MAX_REQUEST_WORDS, MAX_REQUEST_WORDS) - max_words = MAX_REQUEST_WORDS + if max_words is None: + max_words = config.CHUNK_SIZE if max_words < 1: max_words = 1 |