From 31459b281b6a5368c692b3c42c91e522995ebd57 Mon Sep 17 00:00:00 2001 From: historia Date: Wed, 9 Sep 2026 23:25:52 -0400 Subject: feat: smart chunking to avoid chunk boundaries mid-sentence --- app/converter/chunking.py | 400 +++++++++++++++++++++++++++++++++++++++++--- app/converter/config.py | 7 + app/converter/converter.py | 36 ++-- app/tests/test_chunking.py | 129 ++++++++++++++ app/tests/test_converter.py | 18 ++ app/tests/test_hub.py | 53 +++++- app/ui/hub.py | 7 + 7 files changed, 616 insertions(+), 34 deletions(-) (limited to 'app') diff --git a/app/converter/chunking.py b/app/converter/chunking.py index 9ef6a5d..0ffaa21 100644 --- a/app/converter/chunking.py +++ b/app/converter/chunking.py @@ -8,32 +8,346 @@ from . import config logger = logging.getLogger(__name__) +# Smart-chunking tuning, as fractions of the word limit (not exposed as +# settings): once a smart chunk crosses the target it ends at the next +# clean boundary, so most chunks land between the target and the limit +# with some headroom under the configured size. When the next sentence +# will not fit the running chunk, the break retreats to the previous +# boundary that is clean and at least the floor, so a quotation is not +# severed when a later clean boundary exists nearby. +_TARGET_RATIO = 0.85 +_FLOOR_RATIO = 0.7 +# A single token with no whitespace cannot be broken at a word +# boundary; if such a token runs longer than this many characters per +# allowed word it is sliced at the hard character cap (a last-resort +# bound for punctuation-free input such as some CJK text or damage from +# text extraction). +_MAX_CHARS_PER_WORD = 8 -def split_into_chunks(text: str, max_words: Optional[int] = None) -> List[str]: - """Split text into chunks of at most ``max_words`` words. +_ABBREVIATIONS = frozenset( + "mr mrs ms miss dr prof rev hon pres gov sen rep capt lt col gen " + "sgt maj cpl pvt jr sr st mt vs etc eg ie aka fig figs no nos vol " + "vols ch pp p pg dept univ ave blvd rd inc ltd co corp bros apt " + "ste est approx min sec hrs msg mme mlle".split()) - ``max_words`` defaults to ``config.CHUNK_SIZE`` (read at call time). - There is no ceiling beyond that setting, but note that the TTS - servers silently truncate audio when a single generation runs too - long without reporting an error, so very large values are at your - own risk (see CHUNK_SIZE in app/converter/config.py). +_EVENT = re.compile( + r"(?P[.!?…。！？][\"'”’»)}\]]*)" + r"|(?P\n[ \t]*\n)" + r"|(?P[“«])" + r"|(?P[”»])" + r"|(?P\")") + +_PARA_AFTER = re.compile(r"[ \t\r]*\n[ \t]*\n") + +# Scripts whose characters are not whitespace-delimited: count each one +# as a word so a long CJK passage is bounded like English text of the +# same spoken length instead of reading as one giant "word". +_NON_SPACED_RUN = re.compile(r"[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff" + r"\uf900-\ufaff]") + + +class _Unit: + """A sentence-sized piece of text plus its packing metadata. - Splits on sentence boundaries. Sentences longer than the limit are - split further at clause punctuation (which is kept attached for TTS - prosody). Clause splits only happen at whitespace after punctuation, - so tokens like "1,000,000" or "12:30" are never broken apart. A piece - with no usable punctuation split point longer than the limit is split - at word boundaries as a last resort: individual tokens stay intact, - but whitespace between them is normalized. + ``in_quote_end`` is the quotation state after the unit's terminator + (double-quote open marks and closes are tracked; apostrophes do not + count as quotes), and ``para_after`` marks a unit that a blank line + follows. """ - if max_words is None: - max_words = config.CHUNK_SIZE - if max_words < 1: - max_words = 1 - if not text.strip(): - return [] + __slots__ = ("text", "words", "para_after", "in_quote_end") + + def __init__(self, text: str, words: int, para_after: bool, + in_quote_end: bool): + self.text = text + self.words = words + self.para_after = para_after + self.in_quote_end = in_quote_end + + +def _count_words(text: str) -> int: + """Words in TEXT; non-spaced-script characters each count as one.""" + return len(text.split()) + len(_NON_SPACED_RUN.findall(text)) + + +def _is_abbreviation(text: str, dot: int) -> bool: + """True when the '.' before which a sentence split is being + considered belongs to an abbreviation or a name initial (Mr. St. + U.S. J.), where a break would fall mid-sentence.""" + start = dot + while start > 0 and (text[start - 1].isalnum() + or text[start - 1] in ".-'’"): + start -= 1 + token = text[start:dot] + if not token: + return False + tail = token.split(".")[-1] + if len(tail) == 1 and tail.isalpha(): + return True + return token.lower() in _ABBREVIATIONS + + +def _sentence_continues(text: str, end: int, attach: bool) -> bool: + """True when the text after a terminator belongs to the same unit. + + Only a closing quote or an ellipsis can pull a lowercase word into + the unit (dialogue tags: ``“Come here!” she said.``); a plain + period always ends the sentence, whatever follows. + """ + match = re.compile(r"\S").search(text, end) + if match is None: + return False + return attach and match.group().islower() + + +def _scan_units(text: str) -> List[_Unit]: + """Split TEXT into sentence units, tracking quotes and paragraphs. + + Splits at sentence terminators (.!?…) plus any closing quotes or + brackets that trail them, suppressing false boundaries for + abbreviations, initials, and lowercase continuations. Blank lines + close a paragraph (and reset the quotation state, so one unclosed + quote cannot silence clean boundaries for the rest of a book). + Non-whitespace content is preserved; only whitespace between + boundaries may be normalized. + """ + units: List[_Unit] = [] + seg_start = 0 + in_quote = False + depth = 0 + + def emit(end: int, para_after: bool) -> None: + nonlocal seg_start + segment = text[seg_start:end] + seg_start = end + if not segment.strip(): + return + units.append(_Unit(segment, _count_words(segment), para_after, + in_quote or depth > 0)) + + for match in _EVENT.finditer(text): + kind = match.lastgroup + if kind == "end": + run = match.group() + for ch in run[1:]: + if ch == '"': + in_quote = not in_quote + elif ch in "”»": + depth = max(0, depth - 1) + end = match.end() + nxt = text[end:end + 1] + if nxt and not nxt.isspace() \ + and not _NON_SPACED_RUN.match(nxt): + continue # glued punctuation: "3.14" is not a boundary + if (run[0] == "." and _is_abbreviation(text, match.start())) \ + or _sentence_continues(text, end, len(run) > 1 + or run[0] == "…"): + continue + emit(end, _PARA_AFTER.match(text, end) is not None) + elif kind == "para": + emit(match.start(), True) + in_quote = False + depth = 0 + elif kind == "open": + depth += 1 + elif kind == "close": + depth = max(0, depth - 1) + elif kind == "straight": + in_quote = not in_quote + + emit(len(text), False) + return units + + +def _join(units: List[_Unit]) -> str: + """Join units into one chunk, keeping paragraph breaks inside it.""" + parts: List[str] = [] + previous = None + for unit in units: + text = unit.text.strip() + if not text: + continue + if parts: + parts.append("\n\n" if previous.para_after else " ") + parts.append(text) + previous = unit + return "".join(parts) + + +def _retreat_point(cur: List[_Unit], floor: int) -> int: + """Index in CUR to break at when the next unit will not fit. + + Picks the largest prefix of at least FLOOR words that ends outside + a quotation; failing that, the largest prefix of at least FLOOR + words that ends a paragraph; failing both, the whole prefix (the + chunk keeps as much as fits and carries the open quotation into the + next chunk). + """ + best_clean = 0 + best_para = 0 + words = 0 + for k, unit in enumerate(cur, 1): + words += unit.words + if words < floor: + continue + if not unit.in_quote_end: + best_clean = k + elif unit.para_after: + best_para = k + return best_clean or best_para or len(cur) + + +def _split_oversized(text: str, max_words: int) -> List[str]: + """Break a sentence longer than the limit into packable pieces. + + First at clause punctuation (kept attached for TTS prosody, and + never at commas without whitespace so tokens like "1,000,000" and + "12:30" survive), then at word boundaries. A token with no + whitespace at all is sliced at the hard character cap so a + punctuation-free run cannot exceed the bound. + """ + parts = re.split(r"(?:(?<=[,;:])\s+|(?<=[，、；：]))", text) + pieces: List[str] = [] + cap = max_words * _MAX_CHARS_PER_WORD + for part in parts: + part = part.strip() + if not part: + continue + if len(part) <= cap and _count_words(part) <= max_words: + pieces.append(part) + continue + pieces.extend(_hard_slices(part, max_words, cap)) + return pieces + + +def _hard_slices(text: str, max_words: int, cap: int) -> List[str]: + """Slice TEXT at word boundaries, then at CAP characters when a + single token has no whitespace to break at. Buffers never exceed + the word limit or the character cap, so every piece is packable.""" + pieces: List[str] = [] + buf: List[str] = [] + buf_words = 0 + buf_chars = 0 + for word in text.split(): + if len(word) > cap: + if buf: + pieces.append(" ".join(buf)) + buf, buf_words, buf_chars = [], 0, 0 + for start in range(0, len(word), cap): + pieces.append(word[start:start + cap]) + continue + words = _count_words(word) + if buf and buf_words + words <= max_words \ + and buf_chars + 1 + len(word) <= cap: + buf.append(word) + buf_words += words + buf_chars += 1 + len(word) + else: + if buf: + pieces.append(" ".join(buf)) + buf, buf_words, buf_chars = [word], words, len(word) + if buf: + pieces.append(" ".join(buf)) + return pieces + + +def _joined_chars(units: List[_Unit]) -> int: + """The character length of the text _join would produce.""" + total = 0 + previous = None + for unit in units: + text = unit.text.strip() + if not text: + continue + if previous is not None: + total += 2 if previous.para_after else 1 + total += len(text) + previous = unit + return total + +def _split_smart(text: str, max_words: int) -> List[str]: + """Smart strategy: fill chunks up to MAX_WORDS at sentence + boundaries, end them at the target once a quotation has closed or + a paragraph ends, and retreat cleanly instead of breaking quotes + when the next sentence does not fit. Every chunk also respects the + hard character cap, so punctuation-free or non-spaced input stays + bounded.""" + target = max(1, round(max_words * _TARGET_RATIO)) + floor = max(1, round(max_words * _FLOOR_RATIO)) + char_cap = max_words * _MAX_CHARS_PER_WORD + chunks: List[str] = [] + cur: List[_Unit] = [] + cur_words = 0 + cur_chars = 0 + + def flush() -> None: + nonlocal cur, cur_words, cur_chars + if cur: + chunks.append(_join(cur)) + cur, cur_words, cur_chars = [], 0, 0 + + for unit in _scan_units(text): + if cur and cur_words >= target \ + and (not cur[-1].in_quote_end or cur[-1].para_after): + flush() + if unit.words <= max_words \ + and len(unit.text.strip()) <= char_cap \ + and _joined_chars(cur + [unit]) <= char_cap: + if cur_words + unit.words <= max_words: + cur.append(unit) + cur_words += unit.words + cur_chars = _joined_chars(cur) + continue + # The sentence does not fit: break earlier if a clean + # boundary keeps the chunk above the floor. + point = _retreat_point(cur, floor) + if point < len(cur): + chunks.append(_join(cur[:point])) + cur = cur[point:] + cur_words = sum(u.words for u in cur) + cur_chars = _joined_chars(cur) + if cur_words + unit.words <= max_words: + cur.append(unit) + cur_words += unit.words + cur_chars = _joined_chars(cur) + continue + flush() + cur.append(unit) + cur_words = unit.words + cur_chars = _joined_chars(cur) + continue + # One piece exceeds the limit itself: send its pieces out + # packed as tightly as the pieces allow. + flush() + for piece_text in _split_oversized(unit.text, max_words): + if _count_words(piece_text) > max_words \ + or len(piece_text) > char_cap: + # A character-sliced piece of degenerate input: its own + # chunk (already at the hard cap). + flush() + chunks.append(piece_text) + continue + piece = _Unit(piece_text, _count_words(piece_text), False, False) + if cur and cur_words + piece.words <= max_words \ + and cur_chars + 1 + len(piece_text) <= char_cap: + cur.append(piece) + cur_words += piece.words + cur_chars += 1 + len(piece_text) + continue + flush() + cur.append(piece) + cur_words = piece.words + cur_chars = len(piece_text) + flush() + return [chunk for chunk in chunks if chunk.strip()] + + +def _split_legacy(text: str, max_words: int) -> List[str]: + """The pre-smart splitter, kept intact for the Smart chunking=off + setting. Chunks pack whole sentences up to MAX_WORDS; sentences + longer than the limit split at clause punctuation, then at word + boundaries.""" sentences = re.split(r"(?<=[.!?])\s+", text) chunks = [] current_chunk = "" @@ -89,3 +403,49 @@ def split_into_chunks(text: str, max_words: Optional[int] = None) -> List[str]: chunks.append(current_chunk.strip()) return [chunk for chunk in chunks if chunk.strip()] + + +def split_into_chunks(text: str, max_words: Optional[int] = None, + smart: Optional[bool] = None) -> List[str]: + """Split text into chunks of at most ``max_words`` words. + + ``max_words`` defaults to ``config.CHUNK_SIZE`` (read at call time). + There is no ceiling beyond that setting, but note that the TTS + servers silently truncate audio when a single generation runs too + long without reporting an error, so very large values are at your + own risk (see CHUNK_SIZE in app/converter/config.py). + + With smart chunking on (``config.SMART_CHUNKING``, the default), a + chunk crossing the target (~85% of the limit) ends at the next + sentence end outside a quotation or at a paragraph break, so most + requests stop at natural boundaries instead of pressing against the + limit. When the next sentence will not fit, the break retreats to + the last boundary that ends outside a quotation (never smaller than + ~70% of the limit), so short quotations stay in one request when + possible; very long or unclosed quotes still split, carrying their + open state into the next chunk. Malformed punctuation never grows a + chunk beyond the limit: every break consumes text, and input without + sentence punctuation or whitespace (some CJK text, extraction + damage) falls back to clause punctuation, then word boundaries, + then a hard character cap. Content is preserved: no closing quote + or punctuation is ever invented or dropped. + + Both strategies split on sentence boundaries. Sentences longer than + the limit are split further at clause punctuation (which is kept + attached for TTS prosody). Clause splits only happen at whitespace + after punctuation, so tokens like "1,000,000" or "12:30" are never + broken apart. + """ + if max_words is None: + max_words = config.CHUNK_SIZE + if max_words < 1: + max_words = 1 + + if not text.strip(): + return [] + + if smart is None: + smart = bool(getattr(config, "SMART_CHUNKING", False)) + if smart: + return _split_smart(text, max_words) + return _split_legacy(text, max_words) diff --git a/app/converter/config.py b/app/converter/config.py index da73427..c021375 100644 --- a/app/converter/config.py +++ b/app/converter/config.py @@ -10,6 +10,13 @@ HEARTBEAT_INTERVAL_SECONDS = 30 # Print "still working" in console logs every N # Words per TTS generation request (client-side chunking). CHUNK_SIZE = 250 +# Chunking strategy. Smart chunking ends each request at the end of a +# sentence or quotation instead of a mid-sentence overflow: chunks stay +# at or below CHUNK_SIZE (most land slightly under it, aiming for a +# natural boundary near 85%) and short quotations are kept in one +# request when they fit. Turn it off for the exact legacy splitting. +SMART_CHUNKING = True + # Where books are read from and where finished audiobooks are written. # Relative paths resolve against the project root. INPUT_DIR = "input" diff --git a/app/converter/converter.py b/app/converter/converter.py index 2b849a2..f8815d8 100644 --- a/app/converter/converter.py +++ b/app/converter/converter.py @@ -251,11 +251,11 @@ def chunk_clamp_needed(entry) -> bool: def chunk_clamp_message(entry) -> List[str]: """The popup text for a chunk_words-capped ENTRY (short lines).""" return [ - f"{entry.label} can narrate at most ~{entry.chunk_words} words " - "per request: the server caps", - "each request's prompt plus generation at a fixed window. " - "Longer sub-chunks", - "may cut off mid-sentence.", + f"{entry.label} narrates best at ~{entry.chunk_words} words " + "per request on this server:", + "each request's prompt plus generation must fit a fixed " + "window, so longer", + "sub-chunks may cut off mid-sentence.", ] @@ -305,6 +305,9 @@ class AudiobookConverter: # Class-level default so a partially-constructed instance (tests build # these with __new__) behaves like a plain console run. _progress = None + # Same for the per-run chunk override: without __init__ there is no + # clamp, so chunking follows the config.CHUNK_SIZE setting. + chunk_size = None def __init__(self, voice_mode: str = VOICE_MODE_CUSTOM, voice_clone_ref_audio: Optional[str] = None, voice_clone_ref_text: Optional[str] = None, skip_transcription: bool = False, @@ -340,6 +343,12 @@ class AudiobookConverter: self.backend = backend self.voice = voice self.debug = bool(debug) + # Per-run chunk-word override (the pre-flight popup's clamp for + # models whose engine caps one request below a full sub-chunk, + # SGLang-Omni's Higgs): caps the chapter chunks themselves so + # progress and debug dumps span one generation request each. + # None follows the config.CHUNK_SIZE setting. + self.chunk_size = chunk_size # The run's model selection (audio.cpp: a server entry id, sglomni: # the resolved catalog key, None elsewhere) — the startup banner # reports it. @@ -404,10 +413,9 @@ class AudiobookConverter: "catalog key for --model (see the backend docs).") model_id = entry.key self.model_id = model_id - # CHUNK_SIZE carries a per-run override (the pre-flight chunk - # popup's clamp for models whose engine caps one request below - # a full sub-chunk); None follows the config.CHUNK_SIZE setting. - self.chunk_size = chunk_size + # The catalog/request shape is resolved below; the per-run + # chunk override (self.chunk_size) was recorded above and + # reaches the client here. self.tts = SgOmniTTSClient( chunks_dir=CHUNKS_FOLDER, model=model_id, voice=voice, ref_audio=voice_clone_ref_audio, @@ -839,8 +847,14 @@ class AudiobookConverter: return results def _chapter_chunks(self, text: str) -> List[str]: - """Split chapter text into CHUNK_SIZE-word TTS requests.""" - return chunking.split_into_chunks(text) + """Split chapter text into word-capped TTS request chunks. + + Each chunk is one progress unit; when the pre-flight popup set + a per-run clamp (models whose engine caps one request below + CHUNK_SIZE), it caps these chunks too so a chunk is never + wider than one generation request. + """ + return chunking.split_into_chunks(text, max_words=self.chunk_size) def _convert_text(self, text: str, output_path: Path, start_time: float, speed: Optional[float] = None, diff --git a/app/tests/test_chunking.py b/app/tests/test_chunking.py index 2904e40..bcae6b2 100644 --- a/app/tests/test_chunking.py +++ b/app/tests/test_chunking.py @@ -113,6 +113,135 @@ class SplitIntoChunksTests(unittest.TestCase): self.assertNotIn("1, 000", joined) self.assertNotIn("12: 30", joined) + def test_smart_strategy_distributes_sentences(self): + # The smart strategy leaves whole sentences intact: a chunk + # crossing the target (85% of the limit) ends at the next clean + # boundary early; legacy keeps packing until the limit. + sentences = " ".join( + " ".join(f"Sw{j}" for j in range(n)) + "." + for n in (9, 8, 3, 9, 8, 3)) + smart = split_into_chunks(sentences, max_words=20, smart=True) + legacy = split_into_chunks(sentences, max_words=20, smart=False) + self.assertEqual([len(c.split()) for c in smart], [17, 20, 3]) + self.assertEqual([len(c.split()) for c in legacy], [20, 20]) + self.assertEqual(self._content(smart), + self._content(sentences.split(" "))) + + def test_dialogue_tag_stays_with_its_quote(self): + # A closing quote followed by a lowercase attribution ("she + # said.") is one sentence unit: the chunk never separates them. + text = "“Come here!” she said. “Are you coming?” He did not answer." + chunks = split_into_chunks(text, max_words=10, smart=True) + self.assertIn("“Come here!” she said.", chunks[0]) + self._assert_bounded(chunks, 10) + self._assert_content(text, chunks) + + def test_abbreviations_and_initials_do_not_split(self): + text = ("Dr. Smith met J. K. R. at the U.S. border with No. 3 " + "shortly. He continued onward.") + chunks = split_into_chunks(text, max_words=50, smart=True) + self.assertEqual(len(chunks), 1) + self.assertIn("Dr. Smith", chunks[0]) + self.assertIn("J. K. R.", chunks[0]) + self.assertIn("U.S.", chunks[0]) + self._assert_content(text, chunks) + + def test_decimals_are_never_boundaries(self): + text = "Pi is 3.14159 and the toll was 1,000,000 miles. Next one." + chunks = split_into_chunks(text, max_words=12, smart=True) + joined = " ".join(chunks) + self.assertIn("3.14159", joined) + self.assertIn("1,000,000", joined) + self._assert_bounded(chunks, 12) + self._assert_content(text, chunks) + + def test_unclosed_quote_still_respects_the_limit(self): + # A quote with no closing mark must not grow chunks or silence + # later boundaries; units carry the open state and stay bounded. + text = "“No closing quote follows. " + "filler words here. " * 20 + chunks = split_into_chunks(text, max_words=10, smart=True) + self._assert_bounded(chunks, 10) + self._assert_content(text, chunks) + + def test_paragraph_boundary_resets_an_open_quote(self): + # Blank lines reset the quotation state: an unclosed quote in + # one paragraph cannot trap later paragraphs inside the quote. + text = ("“Still open. filler words inside the quote here.\n\n" + "New paragraph. Words outside any quote now, plenty.") + chunks = split_into_chunks(text, max_words=10, smart=True) + self._assert_bounded(chunks, 10) + self._assert_content(text, chunks) + second = [c for c in chunks if "New paragraph." in c] + self.assertTrue(second) + # The paragraph's text starts a fresh chunk: the open-quote + # prefix did not swallow it. + self.assertTrue( + any(chunk.startswith("New paragraph.") or + "“Still open." not in chunk + for chunk in chunks)) + + def test_quote_retreat_keeps_short_quotes_whole(self): + # When the next sentence would overflow, the break retreats to + # the last boundary that ended outside a quotation — before the + # quote — so a short quotation lands in one request. + text = ("Plain opening words here to fill. " + "“A short quote. With two sentences. Inside.” " + "More trailing prose follows this quote. And more.") + chunks = split_into_chunks(text, max_words=14, smart=True) + self._assert_bounded(chunks, 14) + self._assert_content(text, chunks) + quote_chunks = [c for c in chunks if "short quote" in c] + self.assertTrue(quote_chunks) + self.assertIn("Inside.", quote_chunks[0]) + + def test_oversized_single_tokens_respect_char_cap(self): + # Degenerate text (punctuation-free run): bounded by the hard + # character cap, with every character preserved. + text = "a" * 5000 + chunks = split_into_chunks(text, max_words=250, smart=True) + self.assertGreater(len(chunks), 1) + self.assertTrue(all(len(c) <= 250 * 8 for c in chunks)) + self.assertEqual("".join(chunks), text) + + def test_non_spaced_script_stays_bounded(self): + # CJK text: sentence terminators break units without + # whitespace, and each ideograph counts as a word. + text = "。".join(["字" * 15 for _ in range(30)]) + "。" + chunks = split_into_chunks(text, max_words=40, smart=True) + self._assert_bounded(chunks, 40) + self._assert_content(text, chunks) + joined = " ".join(chunks).split() + self.assertTrue(all(len(j) <= 33 for j in joined)) + + def test_legacy_and_smart_matching_smart_negation(self): + # smart=False preserves the legacy splitting exactly. + sentences = " ".join( + " ".join(f"Sw{j}" for j in range(9)) + "." for _ in range(10)) + with patch.object(config, "CHUNK_SIZE", 25): + legacy_default = split_into_chunks(sentences, smart=False) + self.assertEqual([len(c.split()) for c in legacy_default], + [18] * 5) + self.assertEqual(self._content(legacy_default), + self._content(sentences.split())) + + # -- helpers --------------------------------------------------------- + + @staticmethod + def _content(chunks_or_words): + """Non-whitespace content of the pieces (or of a token list).""" + pieces = (chunks_or_words if isinstance(chunks_or_words, str) + else " ".join(chunks_or_words)) + return "".join(pieces.split()) + + def _assert_content(self, text, chunks): + self.assertEqual(self._content(chunks), self._content(text)) + + def _assert_bounded(self, chunks, max_words): + from converter.chunking import _count_words + for chunk in chunks: + self.assertLessEqual(_count_words(chunk), max_words, + chunk[:40]) + if __name__ == "__main__": unittest.main() diff --git a/app/tests/test_converter.py b/app/tests/test_converter.py index 85ae6ca..9084898 100644 --- a/app/tests/test_converter.py +++ b/app/tests/test_converter.py @@ -585,6 +585,24 @@ class ChunkClampPromptTests(unittest.TestCase): with patch.object(config, "CHUNK_SIZE", 80): self.assertFalse(chunk_clamp_needed(self._entry())) + def test_per_run_clamp_caps_the_chapter_chunks(self): + # The popup's clamp caps the chapter chunks themselves, so each + # progress unit and debug dump spans one generation request. + converter = AudiobookConverter.__new__(AudiobookConverter) + converter.chunk_size = 4 + with patch.object(config, "CHUNK_SIZE", 250): + chunks = converter._chapter_chunks(" ".join(f"Sw{i} ." for i in range(12))) + self.assertGreater(len(chunks), 1) + self.assertTrue(all(len(c.split()) <= 4 for c in chunks)) + + def test_chapter_chunks_follow_chunk_size_without_a_clamp(self): + converter = AudiobookConverter.__new__(AudiobookConverter) + self.assertIsNone(converter.chunk_size) + with patch.object(config, "CHUNK_SIZE", 8): + chunks = converter._chapter_chunks(" ".join(f"Sw{i} ." for i in range(20))) + self.assertGreater(len(chunks), 1) + self.assertTrue(all(len(c.split()) <= 8 for c in chunks)) + def test_prompt_answers(self): entry = self._entry() with patch("builtins.input", return_value=""): diff --git a/app/tests/test_hub.py b/app/tests/test_hub.py index 90ed2d4..cbcdc8a 100644 --- a/app/tests/test_hub.py +++ b/app/tests/test_hub.py @@ -3529,7 +3529,8 @@ class SettingsTests(unittest.TestCase): # Keys _apply_settings persists; every test that triggers a real or # fake config write restores these afterwards. _SETTING_KEYS = ("AUDIO_FORMAT", "AUDIO_BITRATE", "LANGUAGE", - "CHUNK_SIZE", "INPUT_DIR", "OUTPUT_DIR", + "CHUNK_SIZE", "SMART_CHUNKING", "INPUT_DIR", + "OUTPUT_DIR", "CLONE_WAV_DIR", "SPEED", "DEBUG", "STOP_SERVER_AND_EXIT", "AUDIOCPP_UNLOAD_MODELS", @@ -3604,6 +3605,7 @@ class SettingsTests(unittest.TestCase): original_folders[1]) values = {"audio_format": "ogg", "audio_bitrate": " 192k ", "language": "en", "chunk_size": "300", + "smart_chunking": True, "input_dir": " /books ", "output_dir": "/audiobooks", "clone_wav_dir": " /refs/wavs ", "speed": "1.5", "debug": True, @@ -3626,6 +3628,7 @@ class SettingsTests(unittest.TestCase): "AUDIO_BITRATE": "192k", "LANGUAGE": "English", "CHUNK_SIZE": 300, + "SMART_CHUNKING": True, "INPUT_DIR": "/books", "OUTPUT_DIR": "/audiobooks", "CLONE_WAV_DIR": "/refs/wavs", @@ -3652,6 +3655,7 @@ class SettingsTests(unittest.TestCase): self.assertEqual(hub.config.AUDIO_BITRATE, "192k") self.assertEqual(hub.config.LANGUAGE, "English") self.assertEqual(hub.config.CHUNK_SIZE, 300) + self.assertEqual(hub.config.SMART_CHUNKING, True) self.assertEqual(hub.config.INPUT_DIR, "/books") self.assertEqual(hub.config.OUTPUT_DIR, "/audiobooks") self.assertEqual(hub.config.CLONE_WAV_DIR, "/refs/wavs") @@ -3671,6 +3675,7 @@ class SettingsTests(unittest.TestCase): self._snapshot_settings() base = {"audio_format": "m4b", "audio_bitrate": "128k", "language": "English", "chunk_size": "250", + "smart_chunking": True, "input_dir": "./input", "output_dir": "./output", "clone_wav_dir": "./voices", "speed": "1.0", "debug": False, @@ -3701,9 +3706,46 @@ class SettingsTests(unittest.TestCase): "audiocpp_remote_url": "not a url"}) mk_update.assert_not_called() + def test_smart_chunking_toggle_persists(self): + self._snapshot_settings() + + def fake_update(key, value, config_path=None): + setattr(hub.config, key, value) + return True + + values = {"audio_format": "m4b", "audio_bitrate": "128k", + "language": "English", "chunk_size": "250", + "smart_chunking": False, + "input_dir": "./input", "output_dir": "./output", + "clone_wav_dir": "./voices", + "speed": "1.0", "debug": False, + "stop_and_exit": True, "unload_models": True, + "qwen_port": "7860", + "faster_port": "8000", "audiocpp_port": "8080", + "sglomni_port": "8100"} + with patch.object(hub.common, "update_config_value", fake_update), \ + patch.object(hub, "_sync_audiocpp_server_port"): + hub._apply_settings(values) + self.assertEqual(hub.config.SMART_CHUNKING, False) + + values["smart_chunking"] = True + with patch.object(hub.common, "update_config_value", fake_update), \ + patch.object(hub, "_sync_audiocpp_server_port"): + hub._apply_settings(values) + self.assertEqual(hub.config.SMART_CHUNKING, True) + + def test_smart_chunking_field_defaults_on_with_help(self): + fields = hub._settings_fields() + field = next(f for f in fields if f["key"] == "smart_chunking") + self.assertEqual(field["kind"], "bool") + self.assertEqual(field["value"], hub.config.SMART_CHUNKING) + self.assertTrue(field["help"]) + self.assertTrue(field["label"], "Smart chunking") + def test_field_validators(self): self.assertIsNone(hub._validate_bitrate("128k")) self.assertIsNotNone(hub._validate_bitrate(" ")) + self.assertIsNotNone(hub._validate_bitrate(" ")) self.assertIsNone(hub._validate_language("English")) self.assertIsNone(hub._validate_language("en")) self.assertIsNotNone(hub._validate_language("Klingon")) @@ -3794,6 +3836,7 @@ class SettingsTests(unittest.TestCase): captured["fields"] = fields return {"audio_format": "ogg", "audio_bitrate": "192k", "language": "English", "chunk_size": "300", + "smart_chunking": True, "input_dir": "/books", "output_dir": "/audiobooks", "clone_wav_dir": "/refs/wavs", "speed": "1.0", "debug": False, @@ -3817,7 +3860,8 @@ class SettingsTests(unittest.TestCase): hub._Hub(None).screen_settings() self.assertEqual([f["key"] for f in captured["fields"]], ["audio_format", "audio_bitrate", "language", - "chunk_size", "input_dir", "output_dir", + "chunk_size", "smart_chunking", "input_dir", + "output_dir", "clone_wav_dir", "speed", "debug", "stop_and_exit", "unload_models", @@ -3867,6 +3911,7 @@ class SettingsTests(unittest.TestCase): "audio_bitrate": "192k", "language": "English", "chunk_size": "300", + "smart_chunking": True, "input_dir": "/books", "output_dir": "/audiobooks", "clone_wav_dir": "/refs/wavs", @@ -4020,7 +4065,8 @@ class SettingsTests(unittest.TestCase): original = {name: getattr(hub.config, name) for name in ("AUDIO_FORMAT", "AUDIO_BITRATE", "LANGUAGE", - "CHUNK_SIZE", "INPUT_DIR", "OUTPUT_DIR", + "CHUNK_SIZE", "SMART_CHUNKING", "INPUT_DIR", + "OUTPUT_DIR", "CLONE_WAV_DIR", "SPEED", "DEBUG", "STOP_SERVER_AND_EXIT", "AUDIOCPP_UNLOAD_MODELS", @@ -4042,6 +4088,7 @@ class SettingsTests(unittest.TestCase): 'LANGUAGE = "English"\n' "\n" "CHUNK_SIZE = 250\n" + "SMART_CHUNKING = True\n" 'INPUT_DIR = "./input"\n' 'OUTPUT_DIR = "./output"\n' 'CLONE_WAV_DIR = "./voices"\n' diff --git a/app/ui/hub.py b/app/ui/hub.py index cccf3bb..1b6b5bd 100644 --- a/app/ui/hub.py +++ b/app/ui/hub.py @@ -2464,6 +2464,12 @@ def _settings_fields() -> list: "validate": _validate_language}, {"key": "chunk_size", "label": "Chunk size (words)", "kind": "text", "value": str(config.CHUNK_SIZE), "validate": _validate_chunk_size}, + {"key": "smart_chunking", "label": "Smart chunking", "kind": "bool", + "value": config.SMART_CHUNKING, + "help": ["End each request at the end of a sentence or " + "quotation instead of mid-sentence.", + "Chunks may be shorter than Chunk size; it stays " + "the maximum."]}, {"key": "input_dir", "label": "Input Directory", "kind": "dir", "value": converter_mod.resolve_dir(config.INPUT_DIR, "input"), "validate": _validate_dir}, @@ -2644,6 +2650,7 @@ def _apply_settings(values: dict) -> None: "AUDIO_BITRATE": bitrate, "LANGUAGE": normalize_language(values["language"]), "CHUNK_SIZE": chunk_size, + "SMART_CHUNKING": bool(values["smart_chunking"]), "INPUT_DIR": input_dir, "OUTPUT_DIR": output_dir, "CLONE_WAV_DIR": clone_dir, -- cgit v1.2.3