diff options
Diffstat (limited to 'app/converter')
| -rw-r--r-- | app/converter/audio.py | 13 | ||||
| -rw-r--r-- | app/converter/chunking.py | 473 | ||||
| -rw-r--r-- | app/converter/clients/__init__.py | 2 | ||||
| -rw-r--r-- | app/converter/clients/audiocpp.py | 178 | ||||
| -rw-r--r-- | app/converter/clients/base.py | 5 | ||||
| -rw-r--r-- | app/converter/clients/qwen.py | 18 | ||||
| -rw-r--r-- | app/converter/clients/sglomni.py | 15 | ||||
| -rw-r--r-- | app/converter/config.py | 7 | ||||
| -rw-r--r-- | app/converter/converter.py | 148 | ||||
| -rw-r--r-- | app/converter/extractors.py | 96 |
10 files changed, 897 insertions, 58 deletions
diff --git a/app/converter/audio.py b/app/converter/audio.py index 9df7aff..9536788 100644 --- a/app/converter/audio.py +++ b/app/converter/audio.py @@ -6,6 +6,7 @@ threads them through, so there is no module-global path to mutate. """ import logging +import math import re import shutil import subprocess @@ -29,7 +30,12 @@ def atempo_filters(speed: float) -> str: ``atempo`` accepts 0.5..2.0 per filter; values outside that range are handled by chaining multiple filters. Returns "" when ``speed`` is 1.0. + Non-finite values (``inf``, ``nan``) are rejected: they pass a bare + ``> 0`` check and would either loop forever (``inf / 2.0`` stays + ``inf``) or produce an invalid tempo expression. """ + if not math.isfinite(speed): + raise ValueError(f"Speed must be a finite number, got {speed}") if speed <= 0: raise ValueError(f"Speed must be a positive number, got {speed}") if abs(speed - 1.0) < SPEED_EPSILON: @@ -252,7 +258,10 @@ def build_concat_command(concat_list: Path, output_path: Path, output_format: st raise ValueError("speed_path is required when speed is not 1.0") return [ "ffmpeg", "-y", *inputs, "-filter_complex", - f"[0:a]split=2[base][spd];[spd]{filters}[spdout]", + # asplit (not split): split is a video filter and rejects + # audio streams, which would fail every speed-adjusted + # assembly after synthesis had already completed. + f"[0:a]asplit=2[base][spd];[spd]{filters}[spdout]", "-map", "[base]", *encode, *tags, *cover_block, *container, str(output_path), "-map", "[spdout]", *encode, *tags, *cover_block, *container, str(speed_path), ] @@ -290,7 +299,7 @@ def build_m4b_chapters_command(concat_list: Path, metadata_file: Path, output_pa if speed_path is not None: return [ "ffmpeg", "-y", *inputs, "-filter_complex", - f"[0:a]split=2[base][spd];[spd]{atempo_filters(speed)}[spdout]", + f"[0:a]asplit=2[base][spd];[spd]{atempo_filters(speed)}[spdout]", "-map", "[base]", *cover_block, "-map_metadata", "1", "-map_chapters", "1", *encode, *tags, *container, str(output_path), "-map", "[spdout]", *cover_block, "-map_metadata", "2", "-map_chapters", "2", diff --git a/app/converter/chunking.py b/app/converter/chunking.py index 9ef6a5d..bf104bc 100644 --- a/app/converter/chunking.py +++ b/app/converter/chunking.py @@ -8,32 +8,419 @@ from . import config logger = logging.getLogger(__name__) +# Smart-chunking tuning, as fractions of the word limit (not exposed as +# settings): once a smart chunk crosses the target it ends at the next +# clean boundary, so most chunks land between the target and the limit +# with some headroom under the configured size. When the next sentence +# will not fit the running chunk, the break retreats to the previous +# boundary that is clean and at least the floor, so a quotation is not +# severed when a later clean boundary exists nearby. +_TARGET_RATIO = 0.85 +_FLOOR_RATIO = 0.7 +# A single token with no whitespace cannot be broken at a word +# boundary; if such a token runs longer than this many characters per +# allowed word it is sliced at the hard character cap (a last-resort +# bound for punctuation-free input such as some CJK text or damage from +# text extraction). The slice also honors the word limit itself, since +# every non-spaced-script character counts as one word. +_MAX_CHARS_PER_WORD = 8 -def split_into_chunks(text: str, max_words: Optional[int] = None) -> List[str]: - """Split text into chunks of at most ``max_words`` words. +_ABBREVIATIONS = frozenset( + "mr mrs ms miss dr prof rev hon pres gov sen rep capt lt col gen " + "sgt maj cpl pvt jr sr st mt vs etc eg ie aka fig figs no nos vol " + "vols ch pp p pg dept univ ave blvd rd inc ltd co corp bros apt " + "ste est approx min sec hrs msg mme mlle".split()) - ``max_words`` defaults to ``config.CHUNK_SIZE`` (read at call time). - There is no ceiling beyond that setting, but note that the TTS - servers silently truncate audio when a single generation runs too - long without reporting an error, so very large values are at your - own risk (see CHUNK_SIZE in app/converter/config.py). +_EVENT = re.compile( + r"(?P<end>[.!?…。！？][\"'”’»)}\]]*)" + r"|(?P<para>\n[ \t]*\n)" + r"|(?P<open>[“«])" + r"|(?P<close>[”»])" + r"|(?P<straight>\")") - Splits on sentence boundaries. Sentences longer than the limit are - split further at clause punctuation (which is kept attached for TTS - prosody). Clause splits only happen at whitespace after punctuation, - so tokens like "1,000,000" or "12:30" are never broken apart. A piece - with no usable punctuation split point longer than the limit is split - at word boundaries as a last resort: individual tokens stay intact, - but whitespace between them is normalized. +_PARA_AFTER = re.compile(r"[ \t\r]*\n[ \t]*\n") + +# Scripts whose characters are not whitespace-delimited: count each one +# as a word so a long CJK passage is bounded like English text of the +# same spoken length instead of reading as one giant "word". +_NON_SPACED_RUN = re.compile(r"[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff" + r"\uf900-\ufaff]") + + +class _Unit: + """A sentence-sized piece of text plus its packing metadata. + + ``in_quote_end`` is the quotation state after the unit's terminator + (double-quote open marks and closes are tracked; apostrophes do not + count as quotes), and ``para_after`` marks a unit that a blank line + follows. """ - if max_words is None: - max_words = config.CHUNK_SIZE - if max_words < 1: - max_words = 1 - if not text.strip(): - return [] + __slots__ = ("text", "words", "para_after", "in_quote_end") + + def __init__(self, text: str, words: int, para_after: bool, + in_quote_end: bool): + self.text = text + self.words = words + self.para_after = para_after + self.in_quote_end = in_quote_end + + +def _count_words(text: str) -> int: + """Words in TEXT; non-spaced-script characters each count as one.""" + return len(text.split()) + len(_NON_SPACED_RUN.findall(text)) + + +def _is_abbreviation(text: str, dot: int) -> bool: + """True when the '.' before which a sentence split is being + considered belongs to an abbreviation or a name initial (Mr. St. + U.S. J.), where a break would fall mid-sentence.""" + start = dot + while start > 0 and (text[start - 1].isalnum() + or text[start - 1] in ".-'’"): + start -= 1 + token = text[start:dot] + if not token: + return False + tail = token.split(".")[-1] + if len(tail) == 1 and tail.isalpha(): + return True + return token.lower() in _ABBREVIATIONS + + +def _sentence_continues(text: str, end: int, attach: bool) -> bool: + """True when the text after a terminator belongs to the same unit. + + Only a closing quote or an ellipsis can pull a lowercase word into + the unit (dialogue tags: ``“Come here!” she said.``); a plain + period always ends the sentence, whatever follows. + """ + match = re.compile(r"\S").search(text, end) + if match is None: + return False + return attach and match.group().islower() + + +def _scan_units(text: str) -> List[_Unit]: + """Split TEXT into sentence units, tracking quotes and paragraphs. + + Splits at sentence terminators (.!?…) plus any closing quotes or + brackets that trail them, suppressing false boundaries for + abbreviations, initials, and lowercase continuations. Blank lines + close a paragraph (and reset the quotation state, so one unclosed + quote cannot silence clean boundaries for the rest of a book). + Non-whitespace content is preserved; only whitespace between + boundaries may be normalized. + """ + units: List[_Unit] = [] + seg_start = 0 + in_quote = False + depth = 0 + + def emit(end: int, para_after: bool) -> None: + nonlocal seg_start + segment = text[seg_start:end] + seg_start = end + if not segment.strip(): + return + units.append(_Unit(segment, _count_words(segment), para_after, + in_quote or depth > 0)) + + for match in _EVENT.finditer(text): + kind = match.lastgroup + if kind == "end": + run = match.group() + for ch in run[1:]: + if ch == '"': + in_quote = not in_quote + elif ch in "”»": + depth = max(0, depth - 1) + end = match.end() + nxt = text[end:end + 1] + if nxt and not nxt.isspace() \ + and not _NON_SPACED_RUN.match(nxt): + continue # glued punctuation: "3.14" is not a boundary + if (run[0] == "." and _is_abbreviation(text, match.start())) \ + or _sentence_continues(text, end, len(run) > 1 + or run[0] == "…"): + continue + emit(end, _PARA_AFTER.match(text, end) is not None) + elif kind == "para": + emit(match.start(), True) + in_quote = False + depth = 0 + elif kind == "open": + depth += 1 + elif kind == "close": + depth = max(0, depth - 1) + elif kind == "straight": + in_quote = not in_quote + + emit(len(text), False) + return units + + +def _join(units: List[_Unit]) -> str: + """Join units into one chunk, keeping paragraph breaks inside it.""" + parts: List[str] = [] + previous = None + for unit in units: + text = unit.text.strip() + if not text: + continue + if parts: + parts.append("\n\n" if previous.para_after else " ") + parts.append(text) + previous = unit + return "".join(parts) + + +def _retreat_point(cur: List[_Unit], floor: int) -> int: + """Index in CUR to break at when the next unit will not fit. + + Picks the largest prefix of at least FLOOR words that ends outside + a quotation; failing that, the largest prefix of at least FLOOR + words that ends a paragraph; failing both, the whole prefix (the + chunk keeps as much as fits and carries the open quotation into the + next chunk). + """ + best_clean = 0 + best_para = 0 + words = 0 + for k, unit in enumerate(cur, 1): + words += unit.words + if words < floor: + continue + if not unit.in_quote_end: + best_clean = k + elif unit.para_after: + best_para = k + return best_clean or best_para or len(cur) + + +def _split_oversized(text: str, max_words: int) -> List[str]: + """Break a sentence longer than the limit into packable pieces. + + First at clause punctuation (kept attached for TTS prosody, and + never at commas without whitespace so tokens like "1,000,000" and + "12:30" survive), then at word boundaries. A token with no + whitespace at all is sliced at the hard character cap and the word + limit so a punctuation-free run cannot exceed either bound. + """ + parts = re.split(r"(?:(?<=[,;:])\s+|(?<=[，、；：]))", text) + pieces: List[str] = [] + cap = max_words * _MAX_CHARS_PER_WORD + for part in parts: + part = part.strip() + if not part: + continue + if len(part) <= cap and _count_words(part) <= max_words: + pieces.append(part) + continue + pieces.extend(_hard_slices(part, max_words, cap)) + return pieces + +def _slice_token(word: str, max_words: int, cap: int) -> List[str]: + """Slice a single whitespace-free token into packable pieces. + + Cuts before the character cap is exceeded and before the word count + is exceeded: each non-spaced-script character counts as one word, + so a punctuation-free CJK token is bounded by the word limit too, + not only by the character cap. + """ + slices: List[str] = [] + cur: List[str] = [] + cur_chars = 0 + # Any slice is itself one whitespace-delimited word before its + # non-spaced-script characters are counted on top. + cur_words = 1 + for ch in word: + ch_words = 1 if _NON_SPACED_RUN.match(ch) else 0 + if cur and (cur_chars + 1 > cap or cur_words + ch_words > max_words): + slices.append("".join(cur)) + cur, cur_chars, cur_words = [], 0, 1 + cur.append(ch) + cur_chars += 1 + cur_words += ch_words + if cur: + slices.append("".join(cur)) + return slices + + +def _hard_slices(text: str, max_words: int, cap: int) -> List[str]: + """Slice TEXT at word boundaries, then within tokens when a single + token (no whitespace to break at) exceeds the character cap or the + word limit on its own. Buffers never exceed the word limit or the + character cap, so every piece is packable.""" + pieces: List[str] = [] + buf: List[str] = [] + buf_words = 0 + buf_chars = 0 + for word in text.split(): + if len(word) > cap or _count_words(word) > max_words: + # A single token too big to buffer: slice it at the hard + # character cap and the word limit together. + if buf: + pieces.append(" ".join(buf)) + buf, buf_words, buf_chars = [], 0, 0 + pieces.extend(_slice_token(word, max_words, cap)) + continue + words = _count_words(word) + if buf and buf_words + words <= max_words \ + and buf_chars + 1 + len(word) <= cap: + buf.append(word) + buf_words += words + buf_chars += 1 + len(word) + else: + if buf: + pieces.append(" ".join(buf)) + buf, buf_words, buf_chars = [word], words, len(word) + if buf: + pieces.append(" ".join(buf)) + return pieces + + +def _joined_chars(units: List[_Unit]) -> int: + """The character length of the text _join would produce.""" + total = 0 + previous = None + for unit in units: + text = unit.text.strip() + if not text: + continue + if previous is not None: + total += 2 if previous.para_after else 1 + total += len(text) + previous = unit + return total + + +def _quote_states(pieces: List[str], in_quote_end: bool) -> List[bool]: + """The quote state after each piece of an oversized sentence. + + Pieces of one sentence can sit inside a quotation the sentence does + not close, so ending a chunk at such a piece would sever the quote + (and marking them clean lets the target flush cut mid-quote). The + scanner's quote tracking is replayed over the pieces from the state + the sentence started in; that start state is unknown, so both + possibilities are replayed and the one whose end state matches the + unit's known end state wins (a unit rarely starts mid-quote). When + neither matches, every piece takes the unit's conservative end + state. + """ + def simulate(in_quote: bool, depth: int) -> List[bool]: + states: List[bool] = [] + for piece in pieces: + for ch in piece: + if ch == '"': + in_quote = not in_quote + elif ch in "“«": + depth += 1 + elif ch in "”»": + depth = max(0, depth - 1) + states.append(in_quote or depth > 0) + return states + + states = simulate(False, 0) + if states and states[-1] == in_quote_end: + return states + states = simulate(True, 0) + if states and states[-1] == in_quote_end: + return states + return [in_quote_end] * len(pieces) + + +def _split_smart(text: str, max_words: int) -> List[str]: + """Smart strategy: fill chunks up to MAX_WORDS at sentence + boundaries, end them at the target once a quotation has closed or + a paragraph ends, and retreat cleanly instead of breaking quotes + when the next sentence does not fit. Every chunk also respects the + hard character cap, so punctuation-free or non-spaced input stays + bounded.""" + target = max(1, round(max_words * _TARGET_RATIO)) + floor = max(1, round(max_words * _FLOOR_RATIO)) + char_cap = max_words * _MAX_CHARS_PER_WORD + chunks: List[str] = [] + cur: List[_Unit] = [] + cur_words = 0 + cur_chars = 0 + + def flush() -> None: + nonlocal cur, cur_words, cur_chars + if cur: + chunks.append(_join(cur)) + cur, cur_words, cur_chars = [], 0, 0 + + for unit in _scan_units(text): + if cur and cur_words >= target \ + and (not cur[-1].in_quote_end or cur[-1].para_after): + flush() + if unit.words <= max_words \ + and len(unit.text.strip()) <= char_cap \ + and _joined_chars(cur + [unit]) <= char_cap: + if cur_words + unit.words <= max_words: + cur.append(unit) + cur_words += unit.words + cur_chars = _joined_chars(cur) + continue + # The sentence does not fit: break earlier if a clean + # boundary keeps the chunk above the floor. + point = _retreat_point(cur, floor) + if point < len(cur): + chunks.append(_join(cur[:point])) + cur = cur[point:] + cur_words = sum(u.words for u in cur) + cur_chars = _joined_chars(cur) + if cur_words + unit.words <= max_words: + cur.append(unit) + cur_words += unit.words + cur_chars = _joined_chars(cur) + continue + flush() + cur.append(unit) + cur_words = unit.words + cur_chars = _joined_chars(cur) + continue + # One piece exceeds the limit itself: send its pieces out + # packed as tightly as the pieces allow, carrying the + # sentence's quotation state through the fallback so a chunk + # can end at a clean in-sentence boundary — and a piece still + # inside a quote is never mistaken for one. + flush() + piece_texts = _split_oversized(unit.text, max_words) + quote_states = _quote_states(piece_texts, unit.in_quote_end) + for index, piece_text in enumerate(piece_texts): + if _count_words(piece_text) > max_words \ + or len(piece_text) > char_cap: + # A character-sliced piece of degenerate input: its own + # chunk (already at the hard cap). + flush() + chunks.append(piece_text) + continue + piece = _Unit( + piece_text, _count_words(piece_text), + unit.para_after and index == len(piece_texts) - 1, + quote_states[index]) + if cur and cur_words + piece.words <= max_words \ + and cur_chars + 1 + len(piece_text) <= char_cap: + cur.append(piece) + cur_words += piece.words + cur_chars += 1 + len(piece_text) + continue + flush() + cur.append(piece) + cur_words = piece.words + cur_chars = len(piece_text) + flush() + return [chunk for chunk in chunks if chunk.strip()] + + +def _split_legacy(text: str, max_words: int) -> List[str]: + """The pre-smart splitter, kept intact for the Smart chunking=off + setting. Chunks pack whole sentences up to MAX_WORDS; sentences + longer than the limit split at clause punctuation, then at word + boundaries.""" sentences = re.split(r"(?<=[.!?])\s+", text) chunks = [] current_chunk = "" @@ -89,3 +476,49 @@ def split_into_chunks(text: str, max_words: Optional[int] = None) -> List[str]: chunks.append(current_chunk.strip()) return [chunk for chunk in chunks if chunk.strip()] + + +def split_into_chunks(text: str, max_words: Optional[int] = None, + smart: Optional[bool] = None) -> List[str]: + """Split text into chunks of at most ``max_words`` words. + + ``max_words`` defaults to ``config.CHUNK_SIZE`` (read at call time). + There is no ceiling beyond that setting, but note that the TTS + servers silently truncate audio when a single generation runs too + long without reporting an error, so very large values are at your + own risk (see CHUNK_SIZE in app/converter/config.py). + + With smart chunking on (``config.SMART_CHUNKING``, the default), a + chunk crossing the target (~85% of the limit) ends at the next + sentence end outside a quotation or at a paragraph break, so most + requests stop at natural boundaries instead of pressing against the + limit. When the next sentence will not fit, the break retreats to + the last boundary that ends outside a quotation (never smaller than + ~70% of the limit), so short quotations stay in one request when + possible; very long or unclosed quotes still split, carrying their + open state into the next chunk. Malformed punctuation never grows a + chunk beyond the limit: every break consumes text, and input without + sentence punctuation or whitespace (some CJK text, extraction + damage) falls back to clause punctuation, then word boundaries, + then a hard character cap. Content is preserved: no closing quote + or punctuation is ever invented or dropped. + + Both strategies split on sentence boundaries. Sentences longer than + the limit are split further at clause punctuation (which is kept + attached for TTS prosody). Clause splits only happen at whitespace + after punctuation, so tokens like "1,000,000" or "12:30" are never + broken apart. + """ + if max_words is None: + max_words = config.CHUNK_SIZE + if max_words < 1: + max_words = 1 + + if not text.strip(): + return [] + + if smart is None: + smart = bool(getattr(config, "SMART_CHUNKING", False)) + if smart: + return _split_smart(text, max_words) + return _split_legacy(text, max_words) diff --git a/app/converter/clients/__init__.py b/app/converter/clients/__init__.py index 5f38786..44f423b 100644 --- a/app/converter/clients/__init__.py +++ b/app/converter/clients/__init__.py @@ -47,6 +47,7 @@ from .audiocpp import ( AudioCppFamilyProfile, AudioCppTTSClient, audiocpp_entry_supports_design, + audiocpp_entry_supports_instructions, audiocpp_entry_voice_capability, audiocpp_family_narrates, audiocpp_family_spec_tasks, @@ -89,6 +90,7 @@ __all__ = [ "AudioCppFamilyProfile", "AUDIOCPP_DEFAULT_FAMILY_PROFILE", "AUDIOCPP_FAMILY_PROFILES", "audiocpp_entry_voice_capability", "audiocpp_entry_supports_design", "audiocpp_voice_for_run", + "audiocpp_entry_supports_instructions", "audiocpp_family_narrates", "audiocpp_family_spec_tasks", "audiocpp_family_voice_policy", "audiocpp_request_error", "audiocpp_script_input", diff --git a/app/converter/clients/audiocpp.py b/app/converter/clients/audiocpp.py index 578a13b..9f52c40 100644 --- a/app/converter/clients/audiocpp.py +++ b/app/converter/clients/audiocpp.py @@ -62,10 +62,14 @@ AUDIOCPP_VOICE_DESIGN = "design" # voice described by --instructions (vdes) # top-level "instructions" field (the default, read by most design/style # implementations), or the "instruction" request option inside the # "options" object (BreezeTTS 2: --request-option instruction=... and the -# endpoint example send it there; its loader ignores the top-level field). +# endpoint example send it there). audio.cpp's adapter translates the +# top-level "instructions" field into that same "instruction" request +# option for every family, so both channels reach the model. AUDIOCPP_INSTRUCTION_FIELD = "field" AUDIOCPP_INSTRUCTION_OPTION = "option" +AUDIOCPP_FAMILY_BREEZE_TTS = "breeze_tts" + # HTTP error body fragments identifying deterministic request-configuration # problems: the identical request will fail on every retry, so the chunk # loop must give up immediately instead of burning its attempt budget. @@ -76,6 +80,10 @@ AUDIOCPP_NON_RETRYABLE_ERRORS = ( # Cloning without the reference transcript (Qwen3-TTS Base ICL mode): # the server-side voice has reference audio but no transcript for it. "requires reference text", + # A CustomVoice entry given a voice preset that resolves to reference + # audio rather than a built-in speaker id: the model takes built-in + # speakers only. + "custom voice prefill requires speaker", # The server cannot resolve a model contract for the family (its own # hint text about model_specs/--model-spec-override follows the fragment). "model contract spec not found for family", @@ -168,6 +176,15 @@ AUDIOCPP_HINTED_ERRORS = ( "task and cannot generate audiobooks. Consider deleting the model " "from the server configuration (re-run Configure Backends → " "audio.cpp and unselect it)."), + # A CustomVoice entry whose selected server voice resolved to + # reference audio rather than a built-in speaker id (the flat voice + # list cannot distinguish them, so the connect-time check only + # warns): the model cannot clone reference audio. + ("custom voice prefill requires speaker", + "The CustomVoice model serves built-in speakers only: pick a " + "built-in speaker named on the entry (e.g. Vivian, Ryan, Uncle Fu) " + "with --voice, or select the Base model entry to clone a reference " + "voice."), # Graph allocation failures: mostly device memory. The server's log # carries the exact size the failed allocation attempted. ("failed to allocate", _ALLOCATION_HINT_TEXT()), @@ -323,6 +340,11 @@ def audiocpp_family_voice_policy(family: str) -> str: timbre reference despite the spec not declaring a clone task (Vevo2's zero-shot TTS route). Unknown families (no local specs) keep the conservative clone-only default the client has always applied. + + _AUDIOCPP_KNOWN_FAMILY_POLICIES carries verified policies for + *specific* families, consulted before the local spec lookup: a family + whose local spec is absent (older audio.cpp checkout against a newer + server) or incomplete must not be misread as reference-required. """ if family == AUDIOCPP_FAMILY_QWEN3_TTS \ or family in AUDIOCPP_CLONE_ONLY_FAMILIES \ @@ -330,6 +352,8 @@ def audiocpp_family_voice_policy(family: str) -> str: # Qwen3-TTS is entry-typed (speaker/clone/design capability per # model id), so the family policy stays out of its way. return AUDIOCPP_VOICE_REQUIRED + if family in _AUDIOCPP_KNOWN_FAMILY_POLICIES: + return _AUDIOCPP_KNOWN_FAMILY_POLICIES[family] tasks = audiocpp_family_spec_tasks(family) if not tasks: return AUDIOCPP_VOICE_REQUIRED @@ -585,6 +609,66 @@ AUDIOCPP_FAMILY_PROFILES = { instruction_channel=AUDIOCPP_INSTRUCTION_OPTION), } +# Voice-policy overrides for families whose evidence contradicts their +# local model spec (or whose spec predates audio.cpp's task declarations). +# Kept as data so the spec lookup can stay the single generic path. +_AUDIOCPP_KNOWN_FAMILY_POLICIES = { + # BreezeTTS 2 (upstream spec / implementation: tts, clone ("voice + # direction"), design). Older local checkouts carry no breeze_tts + # spec at all — without this fallback a remote Breeze entry reads as + # clone-only and an instructions-only run (voice direction without a + # reference) is refused at connect time. + AUDIOCPP_FAMILY_BREEZE_TTS: AUDIOCPP_VOICE_OPTIONAL, + # VibeVoice's spec declares only "tts", but its implementation + # explicitly accepts reference audio (the session uses a speaker + # reference when the request carries one): mixed tts+clone. The + # pick-falls-back rules then stop silently dropping the selected + # voice for it. + "vibevoice": AUDIOCPP_VOICE_OPTIONAL, +} + + +def audiocpp_entry_supports_instructions(family: str, task: str, + model_id: str) -> Optional[bool]: + """Whether an entry's model consumes a style instruction, if known. + + Three-valued on purpose: True (the model's implementation reads the + instruction), False (proven not to), None (unknown — a spec the local + checkout does not describe or one without instruction markers). + Distinguishing unknown from unsupported keeps the TUI behind honest + wording instead of implying every family follows instructions. + + Verified cases: + - task "vdes" entries: the voice comes from the instruction. + - BreezeTTS 2: the instruction conditions synthesis with or without a + reference (voice design / voice direction). + - Qwen3-TTS: the CustomVoice and VoiceDesign implementations read + "instruction"; the Base (cloning) implementation does not — a + variant-specific split, which is why Qwen is resolved per entry. + - Other families: True only when their local model spec declares an + "instruction"/"instruct" request option; None otherwise. + """ + if task == AUDIOCPP_TASK_VDES: + return True + if family == AUDIOCPP_FAMILY_BREEZE_TTS: + return True + if family == AUDIOCPP_FAMILY_QWEN3_TTS: + lowered = (model_id or "").lower() + if "customvoice" in lowered: + return True + if "base" in lowered: + return False + return None + spec = _family_spec(family) + if spec is None: + return None + names = {str(option.get("name") or "") + for option in (spec.get("request_options") or []) + if isinstance(option, dict)} + if names & {"instruction", "instruct"}: + return True + return None + def audiocpp_script_input(prefix: str, text: str) -> str: """TEXT formatted as one "<PREFIX>: text" script line. @@ -758,6 +842,15 @@ class AudioCppTTSClient(BaseTTSClient): # Free-form per-request options (--option KEY=VALUE) forwarded in the # request's "options" object; models ignore keys they don't know. self.request_options: Dict[str, str] = dict(request_options or {}) + # The instruction can also arrive as --option instruction=... The + # server folds both sources into one request option (the + # top-level field overwrites the option), so two *different* + # instructions would silently drop one — refuse instead, before a + # server is even contacted. + self._check_instruction_conflict() + # A guidance value recommended for instructed requests on specific + # families (resolved in _connect; None = no automatic guidance). + self._auto_guidance_scale: Optional[float] = None # Set during _connect: design_mode for "vdes" entries, instruction_voice # when a family without built-in speakers gets its voice from the # instruction alone (no voice field), and plain_mode for plain-TTS @@ -778,6 +871,51 @@ class AudioCppTTSClient(BaseTTSClient): # Connection # ------------------------------------------------------------------ + def _resolve_auto_guidance(self) -> None: + """Select the request-strengthening guidance for instructed runs. + + Breeze-TTS 2's model card recommends guidance scale 4 to + strengthen instruction-following (its own SDK examples carry + --cfg-scale 4 for voice direction and design); audio.cpp's + default is 1.0, which leaves the clone's reference delivery + dominant. When this run carries an instruction (the field or the + request option) and the user did not set their own guidance + value, the request gets the recommended one. + """ + if self.family == AUDIOCPP_FAMILY_BREEZE_TTS \ + and self.instructions \ + and "guidance_scale" not in (getattr( + self, "request_options", {}) or {}): + self._auto_guidance_scale = 4.0 + self._report("[INFO] Sending guidance_scale 4 with the " + "instruction (strengthens Breeze instruction " + "following; set --option guidance_scale=... to " + "override)") + + def _check_instruction_conflict(self) -> None: + """Refuse two different instructions given at once. + + The server normalizes both the top-level "instructions" field and + the "instruction" request option into the same request option, + with the field winning — so "Instructions: calm narration" plus + "--option instruction=screaming" would silently drop the field. + Identical values are allowed (same effective instruction). + """ + option_instruction = (self.request_options.get("instruction") + or "").strip() + if self.instructions and option_instruction \ + and self.instructions != option_instruction: + raise RuntimeError( + "Two conflicting instructions were given: the Instructions " + f"text ({self.instructions!r}) and the request option " + f"instruction={option_instruction!r}. Keep one source only: " + "an --option instruction=... overrides --instructions " + "silently, so use whichever you intend.") + if not self.instructions and option_instruction: + # Make the forwarded option visible where --instructions + # would be reported, for the run log's completeness. + self.instructions = option_instruction + def _connected(self, mode: str) -> None: """Report the resolved connection (MODE: speaker/voice/... label).""" self._report(f"[OK] Connected to audio.cpp server at {self.api_url} " @@ -833,6 +971,17 @@ class AudioCppTTSClient(BaseTTSClient): f"--voice cannot be used with the voice design model " f"'{self.model_id}': the voice is described by the " "--instructions text instead (see README).") + if audiocpp_entry_voice_capability( + self.family, self.task, self.model_id) \ + == AUDIOCPP_VOICE_SPEAKER \ + and not is_builtin_speaker(self.voice): + self._report( + "[WARNING] The selected CustomVoice entry serves " + f"built-in speakers, and '{self.voice}' is not one " + "of them: unless this server preset maps to a " + "built-in speaker id, synthesis will fail with " + "'custom voice prefill requires speaker'. To clone " + "reference audio, select the Base model entry.") self._check_voice() self._connected(f"voice '{self.voice}'") else: @@ -892,6 +1041,21 @@ class AudioCppTTSClient(BaseTTSClient): self._report(f"[INFO] Sending instruction with every request: {self.instructions}") self._report("[INFO] Its effect (style, emotion, delivery) depends on the " "model family; models without instruction support ignore it.") + # Breeze-TTS 2's model card recommends guidance scale 4 to + # strengthen instruction-following (its SDK default --cfg-scale + # 4 for voice direction); audio.cpp's default is 1.0, which + # leaves the clone's reference delivery dominant. When the run + # carries an instruction and the user did not set their own + # guidance value, send the recommended one. + self._resolve_auto_guidance() + logger.info( + "audio.cpp run settings: model=%s family=%s task=%s voice=%r " + "instruction=%r request_options=%s auto_guidance_scale=%s seed=%r", + self.model_id, self.family or "<unresolved>", self.task, + self.voice, self.instructions or None, + getattr(self, "request_options", {}) or {}, + getattr(self, "_auto_guidance_scale", None), + None if self._seed < 0 else self._seed) unload = (config.AUDIOCPP_UNLOAD_MODELS if self._unload_models_override is None else self._unload_models_override) @@ -1210,11 +1374,21 @@ class AudioCppTTSClient(BaseTTSClient): if self._seed >= 0: # audio.cpp has no negative "randomize" seed; a negative seed # means "let the server randomize", so the field is omitted. - payload["seed"] = self._seed + # Above 2^53 a JSON number loses precision, so the full-range + # seeds audio.cpp documents (send uint64 as a decimal string) + # are sent as strings. + if self._seed > 2 ** 53: + payload["seed"] = str(self._seed) + else: + payload["seed"] = self._seed if self.request_options: # Generic per-model controls (--option KEY=VALUE): forwarded # verbatim; the model ignores keys it does not know. payload["options"] = dict(self.request_options) + if getattr(self, "_auto_guidance_scale", None) is not None: + # Instructed Breeze request without an explicit guidance + # value: the model card's recommended instruction strength. + payload["guidance_scale"] = self._auto_guidance_scale if self.instructions: # Explicit voice-design or style instruction (required for task # "vdes" entries; a voice/style control on families that read diff --git a/app/converter/clients/base.py b/app/converter/clients/base.py index 9cf6979..7e9d919 100644 --- a/app/converter/clients/base.py +++ b/app/converter/clients/base.py @@ -133,6 +133,11 @@ class BaseTTSClient: try: result = self.generate_chunk(text, chunk_num) if result and Path(result).exists(): + # A cancel that arrived while this request was in + # flight must still stop the run: the generated audio + # is scratch (cleaned per book) and must not be + # returned for assembly. + self._check_cancelled() return Path(result) logger.warning("Chunk %d attempt %d failed", chunk_num, attempt + 1) except ConversionCancelled: diff --git a/app/converter/clients/qwen.py b/app/converter/clients/qwen.py index dd1c8ba..3f4025c 100644 --- a/app/converter/clients/qwen.py +++ b/app/converter/clients/qwen.py @@ -268,7 +268,16 @@ class QwenTTSClient(BaseTTSClient): # ------------------------------------------------------------------ def _generate_custom_voice(self, text: str) -> Tuple: - """Generate audio using CustomVoice mode with the run's speaker.""" + """Generate audio using CustomVoice mode with the run's speaker. + + When the endpoint exposes an instruction parameter and the run + carries one, it is sent as a delivery/style control (the demo's + run_instruct accepts an ``instruct`` argument alongside the + speaker; the model voices the text with that delivery instead of + the speaker's default). With no instruction the request is + unchanged. + """ + instructions = getattr(self, "instructions", "") or "" custom_api = self._resolve_api_name("/run_instruct", "/run_custom_voice", "/generate_custom_voice") if custom_api == "/run_instruct": payload = dict( @@ -276,6 +285,9 @@ class QwenTTSClient(BaseTTSClient): lang_disp=self.language, spk_disp=speaker_display_name_for(self.speaker), ) + if instructions \ + and self._endpoint_accepts_param(custom_api, "instruct"): + payload["instruct"] = instructions else: payload = dict( text=text, @@ -290,6 +302,10 @@ class QwenTTSClient(BaseTTSClient): if self._endpoint_accepts_param(custom_api, "seed"): payload["seed"] = self._seed + if instructions \ + and self._endpoint_accepts_param(custom_api, "instruct"): + payload["instruct"] = instructions + return self.client.predict(**payload, api_name=custom_api) def _generate_voice_design(self, text: str) -> Tuple: diff --git a/app/converter/clients/sglomni.py b/app/converter/clients/sglomni.py index 6b8493b..13b98d7 100644 --- a/app/converter/clients/sglomni.py +++ b/app/converter/clients/sglomni.py @@ -175,6 +175,11 @@ class SgOmniTTSClient(BaseTTSClient): raise RuntimeError( f"{entry.label} designs the voice from an instruction: " 'pass --instructions "..." describing the voice.') + if self.instructions and not entry.supports_instructions: + raise RuntimeError( + f"{entry.label} does not consume style instructions: the " + "server would silently ignore them. Remove --instructions, " + "or pick a model that supports them (Qwen3-TTS, MOSS-TTS).") if entry.capability == "clone" and entry.requires_reference \ and not self.ref_audio: raise RuntimeError( @@ -377,6 +382,16 @@ class SgOmniTTSClient(BaseTTSClient): payload["ref_audio"] = self._ref_audio_value() if self.ref_text: payload["ref_text"] = self.ref_text + if entry.supports_instructions and self.instructions: + # Reference + separate style instruction (Qwen3-TTS Base + # instruction conditioning, MOSS v1.5 user message). NOT + # the design task_type: the reference stays the voice. + payload["instructions"] = self.instructions + elif entry.supports_instructions and self.instructions: + # Speaker models (Qwen3-TTS CustomVoice) also shape the + # delivery with an instruction; unsupported models never get + # one (checked at connect). + payload["instructions"] = self.instructions return payload def _request_wav(self, text: str) -> bytes: diff --git a/app/converter/config.py b/app/converter/config.py index da73427..c021375 100644 --- a/app/converter/config.py +++ b/app/converter/config.py @@ -10,6 +10,13 @@ HEARTBEAT_INTERVAL_SECONDS = 30 # Print "still working" in console logs every N # Words per TTS generation request (client-side chunking). CHUNK_SIZE = 250 +# Chunking strategy. Smart chunking ends each request at the end of a +# sentence or quotation instead of a mid-sentence overflow: chunks stay +# at or below CHUNK_SIZE (most land slightly under it, aiming for a +# natural boundary near 85%) and short quotations are kept in one +# request when they fit. Turn it off for the exact legacy splitting. +SMART_CHUNKING = True + # Where books are read from and where finished audiobooks are written. # Relative paths resolve against the project root. INPUT_DIR = "input" diff --git a/app/converter/converter.py b/app/converter/converter.py index 3b878c8..88e17a9 100644 --- a/app/converter/converter.py +++ b/app/converter/converter.py @@ -1,7 +1,9 @@ """Orchestrates book-to-audiobook conversion.""" +import contextlib import glob import logging +import math import re import shutil import sys @@ -64,6 +66,50 @@ CHUNKS_FOLDER = APP_DIR / "chunks" # Per-chunk scratch audio, cleaned per book LOGS_FOLDER = logging_kit.LOG_DIR DEBUG_FOLDER = APP_DIR / "debug" # --debug dumps, kept across runs + +@contextlib.contextmanager +def _conversion_lock(): + """An exclusive cross-process lock over the shared scratch folder. + + Every converter run from this checkout synthesizes into the same + ``app/chunks`` directory (fixed chunk_NNNN names, cleaned per book), so + two overlapping runs would delete or overwrite each other's audio. + The lock makes a second run fail fast instead. It is an advisory + fcntl/msvcrt lock that dies with its process, so a crashed run never + leaves a stale lock wedging later starts; platforms without file + locking degrade to the old unlocked behavior. + """ + CHUNKS_FOLDER.mkdir(parents=True, exist_ok=True) + handle = open(CHUNKS_FOLDER / ".conversion.lock", "w") + handle.write("0") + handle.flush() + try: + busy: Optional[OSError] = None + try: + import fcntl + try: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError as exc: + busy = exc + except ImportError: + try: + import msvcrt + try: + msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) + except OSError as exc: + busy = exc + except ImportError: + logger.warning("No file locking on this platform; the " + "conversion lock is not enforced") + if busy is not None: + raise RuntimeError( + "Another conversion is already running from this " + "installation: the shared scratch folder (app/chunks) is " + "in use. Wait for the run to finish or stop it first.") + yield + finally: + handle.close() # releases the lock + # Output containers and supported input formats. AUDIO_FORMATS = ("mp3", "m4b", "ogg", "flac") SUPPORTED_FORMATS = [".txt", ".pdf", ".epub"] @@ -137,9 +183,10 @@ def voice_mode_for(backend: str, voice: Optional[str] = None, sglomni resolves from the selected model's capability — a design model takes instructions, a clone-capable model clones when a reference .wav is given and otherwise synthesizes its default voice, and a - speaker-capable model takes a preset name; qwen designs with - instructions, clones only with a reference .wav, and uses a built-in - speaker otherwise), so the hub can run the pre-flight overwrite checks + speaker-capable model takes a preset name; qwen clones with a + reference .wav, takes a built-in speaker otherwise (instructions + sent as that model's delivery control), and designs with + instructions alone), so the hub can run the pre-flight overwrite checks against exactly the output names the conversion will produce. """ if backend == BACKEND_FASTER: @@ -157,6 +204,17 @@ def voice_mode_for(backend: str, voice: Optional[str] = None, return VOICE_MODE_CUSTOM # Unresolved model (the caller resolves it later): the qwen-style # heuristic is the closest pre-flight approximation. + if backend == BACKEND_QWEN: + # Mirrors the CLI routing (audiobook.convert): a reference clone + # wins, then a built-in speaker (with instructions as that + # model's delivery control), then instructions alone (VoiceDesign). + if clone: + return VOICE_MODE_CLONE + if (voice or "").strip(): + return VOICE_MODE_CUSTOM + if (instructions or "").strip(): + return VOICE_MODE_DESIGN + return VOICE_MODE_CUSTOM if (instructions or "").strip(): return VOICE_MODE_DESIGN return VOICE_MODE_CLONE if clone else VOICE_MODE_CUSTOM @@ -239,11 +297,11 @@ def chunk_clamp_needed(entry) -> bool: def chunk_clamp_message(entry) -> List[str]: """The popup text for a chunk_words-capped ENTRY (short lines).""" return [ - f"{entry.label} can narrate at most ~{entry.chunk_words} words " - "per request: the server caps", - "each request's prompt plus generation at a fixed window. " - "Longer sub-chunks", - "may cut off mid-sentence.", + f"{entry.label} narrates best at ~{entry.chunk_words} words " + "per request on this server:", + "each request's prompt plus generation must fit a fixed " + "window, so longer", + "sub-chunks may cut off mid-sentence.", ] @@ -293,6 +351,9 @@ class AudiobookConverter: # Class-level default so a partially-constructed instance (tests build # these with __new__) behaves like a plain console run. _progress = None + # Same for the per-run chunk override: without __init__ there is no + # clamp, so chunking follows the config.CHUNK_SIZE setting. + chunk_size = None def __init__(self, voice_mode: str = VOICE_MODE_CUSTOM, voice_clone_ref_audio: Optional[str] = None, voice_clone_ref_text: Optional[str] = None, skip_transcription: bool = False, @@ -307,8 +368,12 @@ class AudiobookConverter: unload_models: Optional[bool] = None, progress: Optional[Callable[[dict], None]] = None, cancel=None): - if speed <= 0: - raise ValueError(f"Speed must be a positive number, got {speed}") + # Non-finite speeds pass a bare "> 0" check (and float("inf") is + # "positive"): they would loop forever in atempo filter chaining, + # so they are rejected here at the boundary. + if not math.isfinite(speed) or speed <= 0: + raise ValueError( + f"Speed must be a finite positive number, got {speed}") if output_format not in AUDIO_FORMATS: raise ValueError(f"Unsupported output format: {output_format}") if backend is None: @@ -328,6 +393,12 @@ class AudiobookConverter: self.backend = backend self.voice = voice self.debug = bool(debug) + # Per-run chunk-word override (the pre-flight popup's clamp for + # models whose engine caps one request below a full sub-chunk, + # SGLang-Omni's Higgs): caps the chapter chunks themselves so + # progress and debug dumps span one generation request each. + # None follows the config.CHUNK_SIZE setting. + self.chunk_size = chunk_size # The run's model selection (audio.cpp: a server entry id, sglomni: # the resolved catalog key, None elsewhere) — the startup banner # reports it. @@ -392,10 +463,9 @@ class AudiobookConverter: "catalog key for --model (see the backend docs).") model_id = entry.key self.model_id = model_id - # CHUNK_SIZE carries a per-run override (the pre-flight chunk - # popup's clamp for models whose engine caps one request below - # a full sub-chunk); None follows the config.CHUNK_SIZE setting. - self.chunk_size = chunk_size + # The catalog/request shape is resolved below; the per-run + # chunk override (self.chunk_size) was recorded above and + # reaches the client here. self.tts = SgOmniTTSClient( chunks_dir=CHUNKS_FOLDER, model=model_id, voice=voice, ref_audio=voice_clone_ref_audio, @@ -747,6 +817,9 @@ class AudiobookConverter: return False output_path = AUDIOBOOKS_FOLDER / f"{stem}.{self.output_format}" + # Same rule as _convert_text: honor a cancel that arrived during + # the final chapter's synthesis instead of assembling anyway. + self._check_cancelled() if not audio.combine_chapters_to_m4b(chapter_files, titles, output_path, chunks_dir=CHUNKS_FOLDER, speed=self.speed, @@ -827,8 +900,14 @@ class AudiobookConverter: return results def _chapter_chunks(self, text: str) -> List[str]: - """Split chapter text into CHUNK_SIZE-word TTS requests.""" - return chunking.split_into_chunks(text) + """Split chapter text into word-capped TTS request chunks. + + Each chunk is one progress unit; when the pre-flight popup set + a per-run clamp (models whose engine caps one request below + CHUNK_SIZE), it caps these chunks too so a chunk is never + wider than one generation request. + """ + return chunking.split_into_chunks(text, max_words=self.chunk_size) def _convert_text(self, text: str, output_path: Path, start_time: float, speed: Optional[float] = None, @@ -884,6 +963,9 @@ class AudiobookConverter: successful_chunks, total_chunks) return False + # A cancel that arrived while the last chunk was generating + # must still stop the run here, before any output is produced. + self._check_cancelled() success = audio.combine_chunks(total_chunks, output_path, chunk_results=results, chunks_dir=CHUNKS_FOLDER, @@ -1061,6 +1143,27 @@ class AudiobookConverter: tag = f"{name_tag}_{narrator_tag}" if name_tag else narrator_tag names.append((book_file, f"{name}_{tag}")) + # The stem-collision suffix above can itself collide with another + # book's natural stem ("book.txt" and "book_txt.txt" both produce + # "book_txt_<tag>"), so make every planned name unique across the + # batch: two books sharing an output name would silently overwrite + # each other (ffmpeg writes with -y and both are approved here, + # before any output exists). Later occurrences get a numeric + # suffix; the list is sorted, so the result is deterministic. + used: set = set() + unique: List[Tuple[Path, str]] = [] + for book_file, name in names: + if name in used: + suffix = 2 + while f"{name}_{suffix}" in used: + suffix += 1 + print(f"[INFO] Output name {name} is already planned for " + f"another book; {book_file.name} becomes {name}_{suffix}") + name = f"{name}_{suffix}" + used.add(name) + unique.append((book_file, name)) + names = unique + # Ask every overwrite question up front, before any conversion # starts, so the rest of the run is unattended. planned: List[Tuple[Path, str]] = [] @@ -1110,6 +1213,19 @@ class AudiobookConverter: self._emit({"kind": "done", "ok": 0, "total": 0}) return True + # The scratch folder is shared by every run from this checkout; + # hold the cross-process lock for the whole conversion. + with _conversion_lock(): + return self._convert_planned_books(book_files, planned, run_start) + + def _convert_planned_books(self, book_files, planned, + run_start: float) -> bool: + """Convert PLANNED books sequentially (the body of ``run``). + + BOOK_FILES/PLANNED come from the pre-flight (outer or caller); + RUN_START timestamps the console summary. Returns True when every + planned book converted. + """ self._say(f"[INFO] Converting {len(planned)} of {len(book_files)} book(s)") # Per-book outcome ({book file name: ok}), published on the diff --git a/app/converter/extractors.py b/app/converter/extractors.py index 8b01372..0e30ac1 100644 --- a/app/converter/extractors.py +++ b/app/converter/extractors.py @@ -2,11 +2,13 @@ import codecs import logging +import posixpath import re import zipfile from html import unescape from pathlib import Path -from typing import List, NamedTuple, Optional +from typing import Dict, List, NamedTuple, Optional +from urllib.parse import unquote try: from bs4 import BeautifulSoup @@ -216,8 +218,13 @@ def _epub_spine_order(epub_zip: zipfile.ZipFile, html_names: List[str]): Parses the package OPF (located via META-INF/container.xml, else the only *.opf member): manifest item id -> href, then the spine's - idrefs. Returns member paths limited to HTML_NAMES; nav documents - (``properties`` containing "nav") and non-HTML items are excluded. + idrefs. Hrefs are URL-encoded, may carry "./" or "../" segments and + XML entities, and are relative to the OPF's folder — they are + normalized before matching against the zip's member names, so valid + chapters are not silently dropped when their written form differs + from the archived path. Returns member paths limited to HTML_NAMES; + nav documents (``properties`` containing "nav") and non-HTML items + are excluded. """ container = "META-INF/container.xml" opf_name = None @@ -246,6 +253,25 @@ def _epub_spine_order(epub_zip: zipfile.ZipFile, html_names: List[str]): base = "/".join(opf_name.split("/")[:-1]) item_tags = re.findall(r"<item\b[^>]*>", opf) + # Zip member names indexed under their normalized spellings, so an + # exact archive path can be found from any equivalent href form. + # (Percent-encoded and plain spellings both map to the same member.) + members: Dict[str, str] = {} + for name in html_names: + members.setdefault(posixpath.normpath(name), name) + members.setdefault(posixpath.normpath(unquote(name)), name) + + def member_for_href(href: str) -> Optional[str]: + """The zip member an OPF manifest HREF points at (best effort).""" + path = unescape(href).strip() + path = path.split("#", 1)[0] + path = unquote(path).strip() + if not path: + return None + if base: + path = f"{base}/{path}" + return members.get(posixpath.normpath(path).lstrip("/")) + def is_nav(item_tag: str) -> bool: properties = attr(item_tag, "properties") or "" return "nav" in properties.split() @@ -262,10 +288,9 @@ def _epub_spine_order(epub_zip: zipfile.ZipFile, html_names: List[str]): href = attr(match, "href") if not href: continue - path = (f"{base}/{href}" if base else href) - path = re.sub(r"#.*$", "", path) - if path in html_names and path not in order: - order.append(path) + member = member_for_href(href) + if member and member not in order: + order.append(member) return order or None @@ -287,45 +312,82 @@ def _read_epub_manual(file_path: Path): def clean_text(text: str) -> str: - """Normalize whitespace and strip standalone page numbers. + """Normalize whitespace, strip standalone page numbers, and keep + paragraph breaks. Page numbers are removed only when a short number (up to three digits) appears alone on its own line (before whitespace collapsing), so inline numbers like "42 years", "1,000" or "3.5" are preserved, as are four-digit standalone lines, which are usually years ("1984") or chapter numbers rather than page numbers. + + Runs of blank lines survive as a single paragraph break (``\\n\\n``) so + the smart chunker can still end requests at paragraph boundaries and + reset its quotation state there; single line breaks (soft wraps) and + all other whitespace collapse to spaces. """ if not text: return "" # Standalone page numbers (digits alone on a line) must go BEFORE the - # newline-collapsing step below. + # whitespace-collapsing steps below. text = re.sub(r"(?m)^\s*\d{1,3}\s*$", " ", text) - text = re.sub(r"\s+", " ", text) + # Normalize line breaks so the steps below see plain "\n". + text = re.sub(r"\r\n?", "\n", text) + # Any run of blank lines becomes exactly one paragraph break. + text = re.sub(r"\n[^\S\n]*\n(?:[^\S\n]*\n)*", "\n\n", text) + # Remaining single line breaks (soft wraps) become spaces. + text = re.sub(r"(?<!\n)\n(?!\n)", " ", text) + # Collapse every other whitespace run (including nbsp) to one space. + text = re.sub(r"[^\S\n]+", " ", text) + # Trim whatever sits against a paragraph break. + text = re.sub(r"[^\S\n]*\n\n[^\S\n]*", "\n\n", text) return text.strip() +# Block-level elements structure the document into paragraphs and lines: +# replacing them with a paragraph break keeps the chunker's paragraph +# boundaries (and headings separated from prose). Inline elements must NOT +# break words, so they are left for text extraction to unwrap without +# spaces ("un<em>believ</em>able" stays one word). The lookahead keeps +# lookalike tags (<parameter>, <preheat>) out of the match. +_BLOCK_BREAK_RE = re.compile( + r"</?(?:address|article|aside|blockquote|body|br|caption|center|col" + r"|colgroup|dd|div|dl|dt|fieldset|figcaption|figure|footer|form" + r"|h[1-6]|head|header|hr|html|legend|li|main|nav|ol|option|p|pre" + r"|section|summary|table|tbody|td|tfoot|th|thead|tr|ul)(?=[\s/>])" + r"[^>]*>", + re.IGNORECASE) + + def clean_html(html_content: str) -> str: - """Strip markup, scripts and styles from HTML content.""" + """Strip markup, scripts and styles from HTML content. + + Block-level tags become paragraph breaks (so the chunker sees the + document's paragraphs); inline tags vanish without adding spaces, so + no artificial word boundaries are injected. + """ if not html_content: return "" + html_content = _BLOCK_BREAK_RE.sub("\n\n", html_content) + if BS4_AVAILABLE: try: soup = BeautifulSoup(html_content, "html.parser") for tag in soup(["script", "style"]): tag.decompose() - text = soup.get_text(separator=" ") - return re.sub(r"\s+", " ", text).strip() + return clean_text(soup.get_text(separator="")) except Exception as exc: logger.debug("BeautifulSoup cleaning failed, falling back to regex: %s", exc) - # Fallback regex cleaning + # Fallback regex cleaning. Block tags are already paragraph breaks; + # every other tag (inline) disappears without adding a space, so no + # artificial word boundary is injected. html_content = re.sub(r"<style[^>]*>.*?</style>", "", html_content, flags=re.DOTALL | re.IGNORECASE) html_content = re.sub(r"<script[^>]*>.*?</script>", "", html_content, flags=re.DOTALL | re.IGNORECASE) - html_content = re.sub(r"<[^>]+>", " ", html_content) + html_content = re.sub(r"<[^>]+>", "", html_content) html_content = unescape(html_content) - html_content = re.sub(r"\s+", " ", html_content) - return html_content.strip() + return clean_text(html_content) def _natural_key(name: str): |