diff options
| author | historia <historiavg@proton.me> | 2026-08-20 16:17:57 -0400 |
|---|---|---|
| committer | historia <historiavg@proton.me> | 2026-08-20 16:17:57 -0400 |
| commit | 4d3530f63730b47870d25629802c0c41f0c9ffae (patch) | |
| tree | 319311a69f7ace225a4ef0f8040fc22857c304b8 /converter | |
| parent | c2afb9d01b854bb709345c1b33bdba741daceb25 (diff) | |
| download | tts-audiobook-generator-4d3530f63730b47870d25629802c0c41f0c9ffae.tar.gz | |
feat: audio.cpp backend support
Diffstat (limited to 'converter')
| -rw-r--r-- | converter/config.py | 86 | ||||
| -rw-r--r-- | converter/converter.py | 60 | ||||
| -rw-r--r-- | converter/tts.py | 281 |
3 files changed, 370 insertions, 57 deletions
diff --git a/converter/config.py b/converter/config.py index 9f4dd95..4cb5ab8 100644 --- a/converter/config.py +++ b/converter/config.py @@ -1,50 +1,58 @@ -"""Configuration for the audiobook converter. - -Edit these values to change voices and processing behavior. Everything -else (voice mode names, languages, speaker names, model ids, folders, -file formats) is fixed in the code where it is used. -""" - -# Server endpoints. Voice clone needs the Base-model demo, which is a -# separate server from the CustomVoice demo (that one only exposes -# /run_instruct); the faster backend is the OpenAI-compatible server from -# the faster-qwen3-tts repository. -QWEN_API_URL = "http://127.0.0.1:7860" # CustomVoice demo -CLONE_API_URL = "http://127.0.0.1:7861" # Base-model demo -FASTER_API_URL = "http://127.0.0.1:8000" # faster-qwen3-tts server - -# Words per TTS generation request. Each request is ONE model generation: -# the voice is re-sampled per request (every chunk boundary can drift -# slightly), while over-long generations lose prosody and can turn garbled. -# ~250 words is ~1.5-2 minutes of speech: few voice boundaries while -# staying inside both servers' generation caps. +# Default output options +AUDIO_FORMAT = "m4b" +AUDIO_BITRATE = "128k" +LANGUAGE = "English" + +API_TIMEOUT = 600 # Timeout per chunk request in seconds +MAX_RETRIES = 3 # Attempts per chunk request +HEARTBEAT_INTERVAL_SECONDS = 30 # Print "still working" in console logs every N seconds + +# Words per TTS generation request. +# Note that qwen-tts-demo does no chunking at all, but faster-qwen-tts and +# audio.cpp may do chunking as well, so you may be needlessly double-chunking. CHUNK_SIZE = 250 -API_TIMEOUT = 600 # Seconds before an API call times out (a ~250-word request can take minutes on the Gradio demo) -MAX_RETRIES = 3 # Attempts per chunk request -HEARTBEAT_INTERVAL_SECONDS = 30 # Print "still working" this often during a chunk +# Default TTS backend. +# gradio: qwen-tts-demo +# faster: faster-qwen-tts +# audiocpp: audiocpp_server +# The --backend CLI flag overrides this +BACKEND = "gradio" -LANGUAGE = "English" +############################################################################### +# BACKEND 1: qwen-tts-demo (gradio) options # +############################################################################### -# Seed sent to the TTS servers (only the endpoints that accept one; the -# primary /run_instruct and /run_voice_clone endpoints and the faster -# backend never receive a seed). -1 means "randomize per generation". -# With CONSTANT_SEED = True and SEED = -1, one random seed is drawn at -# startup and reused for every request of the run, keeping the voice -# consistent across chunk boundaries; set SEED to a fixed number to also -# reproduce the same voice across runs. -SEED = -1 -CONSTANT_SEED = True +# There are different API URLs for CustomVoice and Base models so you can run both at once +QWEN_API_URL = "http://127.0.0.1:7860" # CustomVoice model +CLONE_API_URL = "http://127.0.0.1:7861" # Base model -SPEAKER = "Vivian" +# Custom voice options +SPEAKER = "Vivian" #Vivian, Serena, Uncle_Fu, Dylan, Eric, Ryan, Aiden, Ono_Anna, Sohee INSTRUCT = "Speak naturally and clearly, as if reading a dramatic book to an adult audience." +# Don't clone with transcription, only use x-vector-only cloning. Generally "worse" XVECTOR_ONLY = False -# Must match a key in the faster server's voices.json ("default" when the -# server was launched with --ref-audio). The server silently falls back to -# its first voice for unknown names. +# Randomization seed. -1 means randomize with every generation +# With SEED = -1 and CONSTANT_SEED = True, one random seed will be used for the entire audiobook. +# This may keep the voice slightly more consistent across chunk boundaries +SEED = -1 +CONSTANT_SEED = False + +############################################################################### +# BACKEND 2: faster-qwen-tts options # +############################################################################### +FASTER_API_URL = "http://127.0.0.1:8000" # faster-qwen3-tts server (Base model only) + +# Default voice if no --voice is passed FASTER_VOICE = "default" -AUDIO_FORMAT = "m4b" # Default output container -AUDIO_BITRATE = "128k" +############################################################################### +# BACKEND 3: audio.cpp options # +############################################################################### +AUDIOCPP_API_URL = "http://127.0.0.1:8080" # audio.cpp audiocpp_server + +# Model ids in the audio.cpp server.json config. +AUDIOCPP_MODEL_ID = "qwen"https://github.com/0xShug0/audio.cpp +AUDIOCPP_CLONE_MODEL_ID = "qwen-clone" diff --git a/converter/converter.py b/converter/converter.py index 4422316..eb3e80e 100644 --- a/converter/converter.py +++ b/converter/converter.py @@ -15,10 +15,15 @@ from typing import Dict, List, Optional, Tuple from . import audio, chunking, config, cover, extractors from .audio import TrackMeta from .tts import ( + BACKENDS, + BACKEND_AUDIOCPP, + BACKEND_FASTER, + BACKEND_GRADIO, MODEL_SIZE, VOICE_MODE_CLONE, VOICE_MODE_CUSTOM, VOICE_MODES, + AudioCppTTSClient, FasterTTSClient, QwenTTSClient, normalize_language, @@ -128,12 +133,16 @@ class AudiobookConverter: def __init__(self, voice_mode: str = VOICE_MODE_CUSTOM, voice_clone_ref_audio: Optional[str] = None, voice_clone_ref_text: Optional[str] = None, skip_transcription: bool = False, speed: float = 1.0, single_file: bool = False, output_format: str = config.AUDIO_FORMAT, - language: Optional[str] = None, faster: bool = False, - faster_voice: Optional[str] = None, debug: bool = False): + language: Optional[str] = None, backend: str = BACKEND_GRADIO, + voice: Optional[str] = None, debug: bool = False): if speed <= 0: raise ValueError(f"Speed must be a positive number, got {speed}") if output_format not in AUDIO_FORMATS: raise ValueError(f"Unsupported output format: {output_format}") + if backend not in BACKENDS: + raise ValueError( + f"Unknown backend: {backend!r} (expected one of {BACKENDS})" + ) if language is None: language = config.LANGUAGE self.language = normalize_language(language) @@ -142,14 +151,18 @@ class AudiobookConverter: self.speed = speed self.single_file = single_file self.output_format = output_format - self.faster = faster - self.faster_voice = faster_voice + self.backend = backend + self.voice = voice self.debug = bool(debug) self._validate_configuration() - if faster: + if backend == BACKEND_FASTER: # The faster backend always voice-clones using a reference voice # configured on the server, so no local reference audio is needed. - self.tts = FasterTTSClient(voice=faster_voice) + self.tts = FasterTTSClient(voice=voice) + elif backend == BACKEND_AUDIOCPP: + # Speaker mode (no voice) uses a built-in CustomVoice speaker; + # an explicit voice selects a server-side preset (cloning). + self.tts = AudioCppTTSClient(voice=voice, language=self.language) else: self.tts = QwenTTSClient( voice_mode=voice_mode, @@ -166,7 +179,7 @@ class AudiobookConverter: f"Unknown voice mode: {self.voice_mode!r} " f"(expected one of {VOICE_MODES})" ) - if self.voice_mode == VOICE_MODE_CLONE and not self.faster: + if self.voice_mode == VOICE_MODE_CLONE and self.backend == BACKEND_GRADIO: if not self.voice_clone_ref_audio: raise ValueError( "Voice Clone mode requires a reference audio file. " @@ -189,12 +202,15 @@ class AudiobookConverter: """Narrator name used in output file names. Custom voice mode uses the built-in speaker's display name; voice - clone mode uses the reference audio file's stem; the faster backend - uses the server-side voice name. Spaces become underscores - (e.g. "Uncle Fu" -> "Uncle_Fu"). + clone mode uses the reference audio file's stem; the faster and + audiocpp backends use the server-side voice name (falling back to + the built-in speaker for the audiocpp backend's speaker mode). + Spaces become underscores (e.g. "Uncle Fu" -> "Uncle_Fu"). """ - if self.faster: - narrator = self.faster_voice or config.FASTER_VOICE + if self.backend == BACKEND_FASTER: + narrator = self.voice or config.FASTER_VOICE + elif self.backend == BACKEND_AUDIOCPP: + narrator = self.voice or speaker_display_name() elif self.voice_mode == VOICE_MODE_CLONE: narrator = Path(self.voice_clone_ref_audio).stem else: @@ -445,7 +461,11 @@ class AudiobookConverter: chunk_sizes = [len(chunk.split()) for chunk in chunks] avg_chunk_size = sum(chunk_sizes) / len(chunk_sizes) logger.info("Split into %d chunks (avg %.0f words per chunk)", total_chunks, avg_chunk_size) - backend = "faster TTS API" if self.faster else "Qwen API" + backend_labels = { + BACKEND_FASTER: "faster TTS API", + BACKEND_AUDIOCPP: "audio.cpp server", + } + backend = backend_labels.get(self.backend, "Qwen API") print(f"[INFO] Processing {total_chunks} chunks via {backend}...") results = self._synthesize_chunks(chunks, debug_dir=debug_dir) @@ -493,10 +513,20 @@ class AudiobookConverter: print("=" * 70) print(f"Books folder: {BOOKS_FOLDER}") print(f"Output folder: {AUDIOBOOKS_FOLDER}") - if self.faster: + if self.backend == BACKEND_FASTER: print(f"Faster TTS endpoint: {config.FASTER_API_URL}") print("Backend: faster (voice cloning, reference configured on server)") - print(f"Voice: {self.faster_voice or config.FASTER_VOICE}") + print(f"Voice: {self.voice or config.FASTER_VOICE}") + elif self.backend == BACKEND_AUDIOCPP: + print(f"audio.cpp endpoint: {config.AUDIOCPP_API_URL}") + print(f"Model id: {config.AUDIOCPP_MODEL_ID}") + if self.voice: + print("Backend: audio.cpp (voice cloning, reference configured on server)") + print(f"Voice: {self.voice}") + else: + print("Backend: audio.cpp (custom voice, built-in speaker)") + print(f"Speaker: {config.SPEAKER}") + print(f"Language: {self.language}") else: api_url = (config.CLONE_API_URL if self.voice_mode == VOICE_MODE_CLONE else config.QWEN_API_URL) diff --git a/converter/tts.py b/converter/tts.py index ff9da2b..741d9d3 100644 --- a/converter/tts.py +++ b/converter/tts.py @@ -4,6 +4,9 @@ QwenTTSClient talks to the Qwen3-TTS Gradio demos (custom voice / voice clone). FasterTTSClient talks to the OpenAI-compatible server from the faster-qwen3-tts repository (voice cloning only; the reference voice is configured server-side — see the "Faster backend" section of the README). +AudioCppTTSClient talks to the audiocpp_server from the audio.cpp +repository, which serves the same Qwen3-TTS models through an +OpenAI-style API (see the "audio.cpp backend" section of the README). """ import contextlib @@ -17,6 +20,7 @@ import tempfile import threading import time import urllib.error +import urllib.parse import urllib.request import wave from pathlib import Path @@ -33,6 +37,12 @@ VOICE_MODE_CUSTOM = "custom_voice" VOICE_MODE_CLONE = "voice_clone" VOICE_MODES = (VOICE_MODE_CUSTOM, VOICE_MODE_CLONE) +# TTS backends (re-exported for the CLI and the converter orchestrator). +BACKEND_GRADIO = "gradio" +BACKEND_FASTER = "faster" +BACKEND_AUDIOCPP = "audiocpp" +BACKENDS = (BACKEND_GRADIO, BACKEND_FASTER, BACKEND_AUDIOCPP) + # Languages understood by the Qwen3-TTS API. Display names must match the # demo dropdown exactly (the demo silently falls back to "Auto" for # unrecognized values, so languages are validated client-side first). @@ -92,6 +102,21 @@ SAMPLE_RATE = 24000 CHUNKS_FOLDER = Path(__file__).resolve().parent.parent / "chunks" +def _resolve_request_seed() -> int: + """Resolve the seed sent with every request. + + Returns config.SEED as-is, or (with CONSTANT_SEED and SEED < 0) one + random value drawn per run, meant to be reused for every request so + the voice stays consistent across chunk boundaries. Without + CONSTANT_SEED, -1 is returned so the server re-samples the voice on + every generation. + """ + seed = config.SEED + if config.CONSTANT_SEED and seed < 0: + seed = random.randrange(2 ** 31) + return seed + + def speaker_display_name() -> str: """Return the Gradio display name for the configured custom speaker.""" return SPEAKER_DISPLAY_NAMES.get( @@ -289,9 +314,7 @@ class QwenTTSClient(_BaseTTSClient): # reused for every request so the voice stays consistent across # chunk boundaries. Without CONSTANT_SEED, -1 is forwarded so the # server re-samples the voice on every generation. - self._seed = config.SEED - if config.CONSTANT_SEED and self._seed < 0: - self._seed = random.randrange(2 ** 31) + self._seed = _resolve_request_seed() if language is None: language = config.LANGUAGE # Validate before connecting so bad values fail fast without a server. @@ -677,3 +700,255 @@ class FasterTTSClient(_BaseTTSClient): except Exception as exc: logger.error("Faster chunk processing failed for chunk %d: %s", chunk_num, exc) return None + + +class AudioCppTTSClient(_BaseTTSClient): + """Generates audio chunks through an audio.cpp audiocpp_server. + + Talks to the OpenAI-style HTTP API of audiocpp_server, which serves + the same Qwen3-TTS models as the Gradio demos through a native + ggml runtime (GGUF weights, no Python serving stack). Two voice + modes, both resolved server-side from the request's "voice" field: + + - Speaker mode (no ``voice``): a built-in CustomVoice speaker name + (e.g. "Vivian") is passed through, plus the INSTRUCT style prompt. + The server must be configured with the CustomVoice model for this. + - Preset mode (``voice=NAME``): a voice configured on the server + (``voice_presets`` or ``voice_dir`` in its config, e.g. a cloning + reference). The name is validated against GET /v1/audio/voices at + startup because an unresolvable name would silently fall back to + plain TTS on the Base model instead of failing. When + AUDIOCPP_CLONE_MODEL_ID names a second server entry (typically the + Base model), preset requests are routed to it. + + Each response is a complete WAV file, so sub-request audio is + concatenated with the same lossless path used for the Gradio client. + """ + + def __init__(self, voice: Optional[str] = None, language: Optional[str] = None, + api_url: Optional[str] = None): + self.api_url = (api_url or config.AUDIOCPP_API_URL).rstrip("/") + self.model_id = config.AUDIOCPP_MODEL_ID + # Validate before connecting so bad values fail fast without a server. + self.language = normalize_language( + language if language is not None else config.LANGUAGE) + # Same seed convention as the Gradio client: one value per run, + # reused for every request (see _resolve_request_seed). + self._seed = _resolve_request_seed() + self.preset_mode = bool(voice) + self.voice = voice or speaker_display_name() + self._check_health() + model_ids = self._check_model() + if self.preset_mode: + self._select_model(model_ids) + self._check_voice() + print(f"[OK] Connected to audio.cpp server at {self.api_url} " + f"(model '{self.model_id}', voice '{self.voice}')") + else: + print(f"[OK] Connected to audio.cpp server at {self.api_url} " + f"(model '{self.model_id}', speaker '{self.voice}')") + print("[INFO] Speaker mode expects the server to be configured with the " + "CustomVoice model; with the Base model the speaker name is ignored " + "and a random default voice is used (see README).") + + # ------------------------------------------------------------------ + # Connection + # ------------------------------------------------------------------ + + def _get_json(self, path: str, timeout: int = 10) -> Dict[str, Any]: + """GET a JSON document from the server.""" + url = f"{self.api_url}{path}" + try: + with urllib.request.urlopen(url, timeout=timeout) as response: + return json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + detail = "" + try: + detail = exc.read().decode("utf-8", errors="replace")[:200] + except Exception: + pass + raise RuntimeError( + f"audio.cpp server returned HTTP {exc.code} for {path}: {detail}") from exc + except urllib.error.URLError as exc: + raise RuntimeError(f"audio.cpp request failed for {path}: {exc.reason}") from exc + + def _check_health(self) -> None: + """Verify the server is reachable and reports healthy.""" + try: + payload = self._get_json("/health") + except Exception as exc: + raise RuntimeError( + f"audio.cpp server not reachable at {self.api_url}: {exc}. " + "Start audiocpp_server first (see the 'audio.cpp backend' " + "section of the README)." + ) from exc + if payload.get("status") != "ok": + raise RuntimeError( + f"The audio.cpp server at {self.api_url} reports status " + f"{payload.get('status')!r} instead of 'ok'") + + def _check_model(self) -> List[str]: + """Verify the configured model id exists; return all server model ids.""" + try: + payload = self._get_json("/v1/models") + except Exception as exc: + raise RuntimeError( + f"The audio.cpp server at {self.api_url} did not answer " + f"/v1/models: {exc}") from exc + entries = payload.get("data") or [] + model_ids = [entry.get("id") for entry in entries if isinstance(entry, dict)] + if self.model_id not in model_ids: + configured = ", ".join(str(mid) for mid in model_ids if mid) or "none" + raise RuntimeError( + f"The audio.cpp server at {self.api_url} has no model id " + f"'{self.model_id}' (configured: {configured}). Add a qwen3_tts " + "model entry to the server config and match AUDIOCPP_MODEL_ID " + "in converter/config.py to its id (see README)." + ) + return [mid for mid in model_ids if mid] + + def _select_model(self, model_ids: List[str]) -> None: + """Pick the model for preset (cloning) requests. + + Defaults to the primary model id. When AUDIOCPP_CLONE_MODEL_ID is + configured (typically a Base-model entry, since only that variant + consumes reference audio) and present on the server, preset + requests are routed to it instead, so one server can host the + CustomVoice model for speaker mode and the Base model for + cloning. + """ + clone_model_id = config.AUDIOCPP_CLONE_MODEL_ID + if not clone_model_id or clone_model_id == self.model_id: + return + if clone_model_id in model_ids: + self.model_id = clone_model_id + else: + logger.warning( + "AUDIOCPP_CLONE_MODEL_ID %r is not configured on the audio.cpp " + "server; preset requests use '%s' instead", + clone_model_id, self.model_id) + + def _check_voice(self) -> None: + """Verify the requested voice is available on the server. + + A voice name that matches no server preset or voice-library wav + would be passed through to the model as a cached voice id; on the + Base (cloning) model that is silently ignored and plain TTS audio + comes back, so preset names are validated up front. When the + voices endpoint cannot be queried, validation is skipped with a + warning rather than blocking the run. + """ + query = urllib.parse.urlencode({"model": self.model_id}) + try: + payload = self._get_json(f"/v1/audio/voices?{query}") + except Exception as exc: + logger.warning("Could not list server voices; skipping voice " + "validation: %s", exc) + return + voices = payload.get("voices") or [] + if self.voice not in voices: + available = ", ".join(str(v) for v in voices) or "none" + raise RuntimeError( + f"Voice '{self.voice}' is not available on the audio.cpp server " + f"(available: {available}). Configure it as a voice_preset or " + "voice_dir entry in the server config, or pass a listed name " + "with --voice (see README)." + ) + + # ------------------------------------------------------------------ + # HTTP requests + # ------------------------------------------------------------------ + + def _request_wav(self, text: str) -> bytes: + """POST one sub-chunk and return the raw WAV bytes.""" + url = f"{self.api_url}/v1/audio/speech" + payload: Dict[str, Any] = { + "model": self.model_id, + "input": text, + "voice": self.voice, + "language": self.language, + "seed": self._seed, + } + if not self.preset_mode and config.INSTRUCT: + # Style instruction for the CustomVoice speakers; ignored by + # the Base (cloning) model. + payload["instructions"] = config.INSTRUCT + request = urllib.request.Request( + url, data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, method="POST") + try: + with urllib.request.urlopen(request, timeout=config.API_TIMEOUT) as response: + wav = response.read() + except urllib.error.HTTPError as exc: + detail = "" + try: + detail = exc.read().decode("utf-8", errors="replace")[:200] + except Exception: + pass + raise RuntimeError(f"audio.cpp server returned HTTP {exc.code}: {detail}") from exc + except urllib.error.URLError as exc: + raise RuntimeError(f"audio.cpp request failed: {exc.reason}") from exc + if len(wav) < 12 or wav[:4] != b"RIFF" or wav[8:12] != b"WAVE": + raise RuntimeError("audio.cpp server returned audio that is not a WAV file") + return wav + + def _request_wav_with_retry(self, text: str, chunk_num: int, sub_num: int, + sub_total: int) -> bytes: + """Request one sub-chunk, retrying transient failures.""" + for attempt in range(config.MAX_RETRIES): + try: + return self._request_wav(text) + except Exception as exc: + logger.warning("Chunk %d sub-chunk %d/%d attempt %d failed: %s", + chunk_num, sub_num, sub_total, attempt + 1, exc) + if attempt < config.MAX_RETRIES - 1: + time.sleep(2 + 2 * attempt) + raise RuntimeError(f"Sub-chunk {sub_num}/{sub_total} failed after " + f"{config.MAX_RETRIES} attempts") + + # ------------------------------------------------------------------ + # Chunk generation + # ------------------------------------------------------------------ + + def generate_chunk(self, text: str, chunk_num: int) -> Optional[str]: + """Generate one audio chunk; returns its path in the chunks folder. + + The text is split into sub-requests of at most + ``MAX_REQUEST_WORDS`` words each (defense in depth against + pathological input, matching the Gradio client), each sub-request + returns a complete WAV file, and the parts are concatenated into + one chunk file. + """ + try: + sub_texts = split_into_chunks(text, max_words=MAX_REQUEST_WORDS) + if not sub_texts: + raise RuntimeError("No text to synthesize") + + output_path: Optional[Path] = None + with tempfile.TemporaryDirectory(prefix="tts_parts_") as parts_dir, \ + self._chunk_heartbeat(chunk_num): + part_paths = [] + for sub_num, sub_text in enumerate(sub_texts, 1): + wav = self._request_wav_with_retry( + sub_text, chunk_num, sub_num, len(sub_texts)) + destination = Path(parts_dir) / f"part_{sub_num:02d}.wav" + destination.write_bytes(wav) + check_for_truncation( + sub_text, _audio_duration_seconds(destination), + f"Chunk {chunk_num} sub-request {sub_num}/{len(sub_texts)}") + part_paths.append(destination) + if len(part_paths) == 1: + output_path = self._chunk_path(chunk_num, ".wav") + shutil.copy2(part_paths[0], output_path) + else: + output_path = self._chunk_path(chunk_num, ".wav") + concat_audio_files(part_paths, output_path) + + logger.debug("Chunk %d generated successfully (%d sub-request(s))", + chunk_num, len(sub_texts)) + return str(output_path) + + except Exception as exc: + logger.error("audio.cpp chunk processing failed for chunk %d: %s", + chunk_num, exc) + return None |
