diff options
Diffstat (limited to 'converter')
| -rw-r--r-- | converter/audio.py | 82 | ||||
| -rw-r--r-- | converter/chunking.py | 47 | ||||
| -rw-r--r-- | converter/config.py | 29 | ||||
| -rw-r--r-- | converter/tts.py | 142 |
4 files changed, 260 insertions, 40 deletions
diff --git a/converter/audio.py b/converter/audio.py index 6e5a910..3e163be 100644 --- a/converter/audio.py +++ b/converter/audio.py @@ -5,6 +5,7 @@ import re import shutil import subprocess import traceback +import wave from pathlib import Path from typing import Dict, List, NamedTuple, Optional, Tuple @@ -50,6 +51,87 @@ def _concat_escape(path: str) -> str: return path.replace("'", "'\\''") +def _concat_wav_files(sources: List[Path], destination: Path) -> bool: + """Concatenate WAV files with matching parameters using the wave module. + + Returns False (touching nothing) when any input is not a readable WAV + or the parameters differ, so the caller can fall back to ffmpeg. + """ + opened = [] + try: + parameters = None + for source in sources: + wav_file = wave.open(str(source), "rb") + opened.append(wav_file) + current = (wav_file.getnchannels(), wav_file.getsampwidth(), + wav_file.getframerate()) + if parameters is None: + parameters = current + elif current != parameters: + return False + if parameters is None or min(parameters) < 1: + return False + with wave.open(str(destination), "wb") as output: + output.setnchannels(parameters[0]) + output.setsampwidth(parameters[1]) + output.setframerate(parameters[2]) + for wav_file in opened: + output.writeframes(wav_file.readframes(wav_file.getnframes())) + return True + except (wave.Error, EOFError, OSError): + return False + finally: + for wav_file in opened: + try: + wav_file.close() + except Exception: + pass + + +def _concat_with_ffmpeg(sources: List[Path], destination: Path) -> None: + """Concatenate audio files with ffmpeg's concat demuxer, re-encoding to + 16-bit PCM WAV (handles inputs the wave module cannot).""" + if shutil.which("ffmpeg") is None: + raise RuntimeError( + "ffmpeg is required to concatenate audio parts in non-WAV formats " + "(install ffmpeg and try again)" + ) + list_path = destination.with_name(destination.stem + "_parts.txt") + try: + with open(list_path, "w", encoding="utf-8") as list_file: + for source in sources: + list_file.write(f"file '{_concat_escape(str(source))}'\n") + command = [ + "ffmpeg", "-y", "-hide_banner", "-loglevel", "error", + "-f", "concat", "-safe", "0", "-i", str(list_path), + "-c:a", "pcm_s16le", str(destination), + ] + proc = subprocess.run(command, capture_output=True, text=True) + if proc.returncode != 0: + raise RuntimeError( + f"ffmpeg failed to concatenate audio parts: {proc.stderr[-500:]}") + finally: + try: + list_path.unlink(missing_ok=True) + except OSError: + pass + + +def concat_audio_files(sources: List[Path], destination: Path) -> None: + """Concatenate audio files into one file at ``destination``. + + Joins the audio returned by several TTS sub-requests for a single + chunk. Uses the stdlib wave module when every input is a WAV with + matching parameters (lossless, no external tools); otherwise falls + back to ffmpeg's concat demuxer with re-encoding. + """ + if not sources: + raise ValueError("No audio files to concatenate") + if _concat_wav_files(sources, destination): + return + _concat_with_ffmpeg(sources, destination) + + def _encode_args(output_format: str) -> List[str]: """Return ffmpeg output codec/bitrate args for the requested container.""" if output_format == "m4b": diff --git a/converter/chunking.py b/converter/chunking.py index 0c85adf..425765f 100644 --- a/converter/chunking.py +++ b/converter/chunking.py @@ -1,21 +1,40 @@ """Split extracted book text into TTS-sized chunks.""" +import logging import re from typing import List from . import config +logger = logging.getLogger(__name__) + def split_into_chunks(text: str, max_words: int = config.CHUNK_SIZE_WORDS) -> List[str]: """Split text into chunks of at most ``max_words`` words. - Splits on sentence boundaries. Sentences longer than the limit are split - further at clause punctuation (which is kept attached for TTS prosody). - Clause splits only happen at whitespace after punctuation, so tokens like - "1,000,000" or "12:30" are never broken apart or re-joined with added - spaces. A single sentence with no usable split point longer than the - limit is kept intact as one oversized chunk. + ``max_words`` is clamped to ``config.MAX_REQUEST_WORDS``: requests + beyond that ceiling are silently truncated by the TTS servers (no + error is reported), so chunks larger than the ceiling are never + produced regardless of configuration. + + Splits on sentence boundaries. Sentences longer than the limit are + split further at clause punctuation (which is kept attached for TTS + prosody). Clause splits only happen at whitespace after punctuation, + so tokens like "1,000,000" or "12:30" are never broken apart. A piece + with no usable punctuation split point longer than the limit is split + at word boundaries as a last resort: individual tokens stay intact, + but whitespace between them is normalized. """ + if max_words > config.MAX_REQUEST_WORDS: + logger.warning( + "Requested chunk size of %d words exceeds the %d-word request ceiling; " + "larger requests are silently truncated by the TTS servers, so the " + "size is clamped to %d words (see MAX_REQUEST_WORDS in converter/config.py)", + max_words, config.MAX_REQUEST_WORDS, config.MAX_REQUEST_WORDS) + max_words = config.MAX_REQUEST_WORDS + if max_words < 1: + max_words = 1 + if not text.strip(): return [] @@ -35,11 +54,23 @@ def split_into_chunks(text: str, max_words: int = config.CHUNK_SIZE_WORDS) -> Li # Split long sentences at clause boundaries, keeping punctuation. # Only split where whitespace already follows the punctuation so - # the reassembled text is byte-identical to the input (no spaces - # injected into "1,000,000" or "12:30"). + # tokens are never broken apart or re-joined with added spaces + # (no spaces are injected into "1,000,000" or "12:30"). parts = re.split(r"(?<=[,;:])\s+", sentence) for part in parts: part_words = len(part.split()) + if part_words > max_words: + # Last resort: no punctuation split point is available, + # so split at word boundaries. Tokens themselves (and + # therefore numbers like "1,000,000") stay intact. + if current_chunk: + chunks.append(current_chunk.strip()) + current_chunk = "" + current_words = 0 + words = part.split() + for start in range(0, len(words), max_words): + chunks.append(" ".join(words[start:start + max_words])) + continue if current_words + part_words <= max_words: current_chunk += part + " " current_words += part_words diff --git a/converter/config.py b/converter/config.py index eb01462..a19ac63 100644 --- a/converter/config.py +++ b/converter/config.py @@ -13,14 +13,35 @@ QWEN_API_URL = "http://127.0.0.1:7860" # CustomVoice demo VOICE_CLONE_API_URL = "http://127.0.0.1:7861" # Base-model demo FASTER_TTS_API_URL = "http://127.0.0.1:8000" # faster-qwen3-tts server -# Words per TTS generation request. Each API call is ONE model generation: -# long generations lose prosody and can degrade into garbled audio. -CHUNK_SIZE_WORDS = 40 +# Words per TTS generation request. Each request is ONE model generation: +# the voice is re-sampled per request (every chunk boundary can drift +# slightly), while over-long generations lose prosody and can turn garbled. +# ~250 words is ~1.5-2 minutes of speech: few voice boundaries while +# staying inside both servers' generation caps. +CHUNK_SIZE_WORDS = 250 + +# Hard ceiling on words per request, regardless of CHUNK_SIZE_WORDS. Both +# TTS servers silently truncate audio when a single generation exceeds its +# cap (~2.5 min for the faster backend's static KV cache, ~11 min for the +# Gradio demo) without reporting any error, so larger requests are always +# split client-side. Keep a margin below ~300 words to survive slow +# narration on the faster backend. +MAX_REQUEST_WORDS = 250 + +# Duration sanity check: a response whose audio is far shorter than its +# word count implies is treated as silently truncated, fails the request, +# and goes through the normal retry logic. 150 wpm is a typical spoken +# pace; the ratio is set low (0.5) so only gross truncation trips it. +ESTIMATED_WORDS_PER_MINUTE = 150 +MIN_AUDIO_DURATION_RATIO = 0.5 +MIN_WORDS_FOR_DURATION_CHECK = 10 + VOICE_CLONE_MAX_CHUNK_CHARS = 200 # Server-side re-chunking limit for clone requests VOICE_CLONE_CHUNK_GAP = 0 # Pause (seconds) between server-side clone chunks + MIN_DELAY_BETWEEN_CHUNKS = 0 # Pause between API calls (rate-limit protection; local servers need none) -API_TIMEOUT = 300 # Seconds before an API call times out +API_TIMEOUT = 600 # Seconds before an API call times out (a ~250-word request can take minutes on the Gradio demo) MAX_RETRIES = 3 # Attempts per chunk request HEARTBEAT_INTERVAL_SECONDS = 30 # Print "still working" this often during a chunk diff --git a/converter/tts.py b/converter/tts.py index 9930558..8a1667a 100644 --- a/converter/tts.py +++ b/converter/tts.py @@ -12,6 +12,7 @@ import json import logging import shutil import sys +import tempfile import threading import time import urllib.error @@ -21,6 +22,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple from . import config +from .audio import concat_audio_files, probe_duration_ms from .chunking import split_into_chunks logger = logging.getLogger(__name__) @@ -151,6 +153,49 @@ def transcribe_reference_audio(audio_path: str, model_name: str = "base") -> Opt return None +def check_for_truncation(text: str, actual_seconds: Optional[float], label: str) -> None: + """Raise RuntimeError when audio is far shorter than its text implies. + + Both backends silently truncate audio when a single generation hits an + internal cap (no error is reported to the client), so grossly short + audio must be detected client-side: failing the request lets the retry + logic re-run it, and persistent failures surface as failed chunks + instead of a "successful" run with missing audio. ``actual_seconds`` + is None when the duration could not be determined, in which case the + check is skipped. Requests shorter than + ``config.MIN_WORDS_FOR_DURATION_CHECK`` words are not checked (their + duration estimates are too noisy). + """ + words = len(text.split()) + if actual_seconds is None or words < config.MIN_WORDS_FOR_DURATION_CHECK: + return + expected_seconds = 60.0 * words / config.ESTIMATED_WORDS_PER_MINUTE + if actual_seconds < expected_seconds * config.MIN_AUDIO_DURATION_RATIO: + raise RuntimeError( + f"{label}: audio is far shorter than the text implies " + f"({actual_seconds:.1f}s of audio for {words} words, expected at " + f"least {expected_seconds * config.MIN_AUDIO_DURATION_RATIO:.0f}s); " + "the TTS server likely truncated the generation silently" + ) + + +def _audio_duration_seconds(path: Path) -> Optional[float]: + """Return an audio file's duration in seconds, or None when unknown.""" + try: + with wave.open(str(path), "rb") as wav_file: + framerate = wav_file.getframerate() + if framerate > 0: + return wav_file.getnframes() / float(framerate) + except (wave.Error, EOFError, OSError): + pass + if shutil.which("ffprobe") is None: + return None + milliseconds = probe_duration_ms(path) + if milliseconds <= 0: + return None + return milliseconds / 1000.0 + + class _BaseTTSClient: """Shared chunk retry logic, heartbeat, and chunk file bookkeeping.""" @@ -355,39 +400,76 @@ class QwenTTSClient(_BaseTTSClient): # ------------------------------------------------------------------ def generate_chunk(self, text: str, chunk_num: int) -> Optional[str]: - """Generate one audio chunk; returns its path in the chunks folder.""" + """Generate one audio chunk; returns its path in the chunks folder. + + The text is split into sub-requests of at most + ``config.MAX_REQUEST_WORDS`` words each (the book-level chunker + normally guarantees this already; the split is defense in depth + against pathological input such as a punctuation-free run of + text), and the audio files returned for the sub-requests are + concatenated into one chunk file. + """ try: - if self.voice_mode == VOICE_MODE_CUSTOM: - with self._chunk_heartbeat(chunk_num): - result = self._generate_custom_voice(text) - elif self.voice_mode == VOICE_MODE_CLONE: - with self._chunk_heartbeat(chunk_num): - result = self._generate_voice_clone(text) - else: - raise ValueError(f"Unknown voice mode: {self.voice_mode}") - - if not isinstance(result, (tuple, list)) or not result: - raise RuntimeError("Qwen API returned an invalid result") - - audio_path = result[0] # First element is the audio file path - if not isinstance(audio_path, (str, Path)) or not audio_path: - raise RuntimeError("Qwen API did not return an audio file path") - - source = Path(audio_path) - if not source.exists(): - raise RuntimeError(f"Generated audio file not found: {audio_path}") - - suffix = source.suffix or ".wav" - output_path = self._chunk_path(chunk_num, suffix) - shutil.copy2(source, output_path) + sub_texts = split_into_chunks(text, max_words=config.MAX_REQUEST_WORDS) + if not sub_texts: + raise RuntimeError("No text to synthesize") - logger.debug("Chunk %d generated successfully", chunk_num) + output_path: Optional[Path] = None + with tempfile.TemporaryDirectory(prefix="tts_parts_") as parts_dir, \ + self._chunk_heartbeat(chunk_num): + part_paths = [ + self._generate_sub_request(sub_text, parts_dir, sub_num, + len(sub_texts), chunk_num) + for sub_num, sub_text in enumerate(sub_texts, 1) + ] + if len(part_paths) == 1: + suffix = part_paths[0].suffix or ".wav" + output_path = self._chunk_path(chunk_num, suffix) + shutil.copy2(part_paths[0], output_path) + else: + output_path = self._chunk_path(chunk_num, ".wav") + concat_audio_files(part_paths, output_path) + + logger.debug("Chunk %d generated successfully (%d sub-request(s))", + chunk_num, len(sub_texts)) return str(output_path) except Exception as exc: logger.error("Qwen chunk processing failed for chunk %d: %s", chunk_num, exc) return None + def _generate_sub_request(self, text: str, parts_dir: str, sub_num: int, + sub_total: int, chunk_num: int) -> Path: + """Run one API generation for ``text``; returns the downloaded audio.""" + if sub_total > 1: + logger.info("Chunk %d: oversized input split into %d requests " + "(sub-request %d/%d)", chunk_num, sub_total, sub_num, sub_total) + if self.voice_mode == VOICE_MODE_CUSTOM: + result = self._generate_custom_voice(text) + elif self.voice_mode == VOICE_MODE_CLONE: + result = self._generate_voice_clone(text) + else: + raise ValueError(f"Unknown voice mode: {self.voice_mode}") + + if not isinstance(result, (tuple, list)) or not result: + raise RuntimeError("Qwen API returned an invalid result") + + audio_path = result[0] # First element is the audio file path + if not isinstance(audio_path, (str, Path)) or not audio_path: + raise RuntimeError("Qwen API did not return an audio file path") + + source = Path(audio_path) + if not source.exists(): + raise RuntimeError(f"Generated audio file not found: {audio_path}") + + destination = Path(parts_dir) / f"part_{sub_num:02d}{source.suffix or '.wav'}" + shutil.copy2(source, destination) + + check_for_truncation( + text, _audio_duration_seconds(destination), + f"Chunk {chunk_num} sub-request {sub_num}/{sub_total}") + return destination + # ------------------------------------------------------------------ # API payloads # ------------------------------------------------------------------ @@ -556,15 +638,19 @@ class FasterTTSClient(_BaseTTSClient): def generate_chunk(self, text: str, chunk_num: int) -> Optional[str]: """Generate one audio chunk; returns its path in the chunks folder.""" try: - sub_chunks = split_into_chunks(text, max_words=config.CHUNK_SIZE_WORDS) + sub_chunks = split_into_chunks(text, max_words=config.MAX_REQUEST_WORDS) if not sub_chunks: raise RuntimeError("No text to synthesize") pcm_parts: List[bytes] = [] with self._chunk_heartbeat(chunk_num): for sub_num, sub_text in enumerate(sub_chunks, 1): - pcm_parts.append(self._request_pcm_with_retry( - sub_text, chunk_num, sub_num, len(sub_chunks))) + pcm = self._request_pcm_with_retry( + sub_text, chunk_num, sub_num, len(sub_chunks)) + check_for_truncation( + sub_text, len(pcm) / (2 * SAMPLE_RATE), + f"Chunk {chunk_num} sub-chunk {sub_num}/{len(sub_chunks)}") + pcm_parts.append(pcm) output_path = self._chunk_path(chunk_num, ".wav") with wave.open(str(output_path), "wb") as wav_file: |