From 4d3530f63730b47870d25629802c0c41f0c9ffae Mon Sep 17 00:00:00 2001 From: historia Date: Thu, 20 Aug 2026 16:17:57 -0400 Subject: feat: audio.cpp backend support --- converter/tts.py | 281 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 278 insertions(+), 3 deletions(-) (limited to 'converter/tts.py') diff --git a/converter/tts.py b/converter/tts.py index ff9da2b..741d9d3 100644 --- a/converter/tts.py +++ b/converter/tts.py @@ -4,6 +4,9 @@ QwenTTSClient talks to the Qwen3-TTS Gradio demos (custom voice / voice clone). FasterTTSClient talks to the OpenAI-compatible server from the faster-qwen3-tts repository (voice cloning only; the reference voice is configured server-side — see the "Faster backend" section of the README). +AudioCppTTSClient talks to the audiocpp_server from the audio.cpp +repository, which serves the same Qwen3-TTS models through an +OpenAI-style API (see the "audio.cpp backend" section of the README). """ import contextlib @@ -17,6 +20,7 @@ import tempfile import threading import time import urllib.error +import urllib.parse import urllib.request import wave from pathlib import Path @@ -33,6 +37,12 @@ VOICE_MODE_CUSTOM = "custom_voice" VOICE_MODE_CLONE = "voice_clone" VOICE_MODES = (VOICE_MODE_CUSTOM, VOICE_MODE_CLONE) +# TTS backends (re-exported for the CLI and the converter orchestrator). +BACKEND_GRADIO = "gradio" +BACKEND_FASTER = "faster" +BACKEND_AUDIOCPP = "audiocpp" +BACKENDS = (BACKEND_GRADIO, BACKEND_FASTER, BACKEND_AUDIOCPP) + # Languages understood by the Qwen3-TTS API. Display names must match the # demo dropdown exactly (the demo silently falls back to "Auto" for # unrecognized values, so languages are validated client-side first). @@ -92,6 +102,21 @@ SAMPLE_RATE = 24000 CHUNKS_FOLDER = Path(__file__).resolve().parent.parent / "chunks" +def _resolve_request_seed() -> int: + """Resolve the seed sent with every request. + + Returns config.SEED as-is, or (with CONSTANT_SEED and SEED < 0) one + random value drawn per run, meant to be reused for every request so + the voice stays consistent across chunk boundaries. Without + CONSTANT_SEED, -1 is returned so the server re-samples the voice on + every generation. + """ + seed = config.SEED + if config.CONSTANT_SEED and seed < 0: + seed = random.randrange(2 ** 31) + return seed + + def speaker_display_name() -> str: """Return the Gradio display name for the configured custom speaker.""" return SPEAKER_DISPLAY_NAMES.get( @@ -289,9 +314,7 @@ class QwenTTSClient(_BaseTTSClient): # reused for every request so the voice stays consistent across # chunk boundaries. Without CONSTANT_SEED, -1 is forwarded so the # server re-samples the voice on every generation. - self._seed = config.SEED - if config.CONSTANT_SEED and self._seed < 0: - self._seed = random.randrange(2 ** 31) + self._seed = _resolve_request_seed() if language is None: language = config.LANGUAGE # Validate before connecting so bad values fail fast without a server. @@ -677,3 +700,255 @@ class FasterTTSClient(_BaseTTSClient): except Exception as exc: logger.error("Faster chunk processing failed for chunk %d: %s", chunk_num, exc) return None + + +class AudioCppTTSClient(_BaseTTSClient): + """Generates audio chunks through an audio.cpp audiocpp_server. + + Talks to the OpenAI-style HTTP API of audiocpp_server, which serves + the same Qwen3-TTS models as the Gradio demos through a native + ggml runtime (GGUF weights, no Python serving stack). Two voice + modes, both resolved server-side from the request's "voice" field: + + - Speaker mode (no ``voice``): a built-in CustomVoice speaker name + (e.g. "Vivian") is passed through, plus the INSTRUCT style prompt. + The server must be configured with the CustomVoice model for this. + - Preset mode (``voice=NAME``): a voice configured on the server + (``voice_presets`` or ``voice_dir`` in its config, e.g. a cloning + reference). The name is validated against GET /v1/audio/voices at + startup because an unresolvable name would silently fall back to + plain TTS on the Base model instead of failing. When + AUDIOCPP_CLONE_MODEL_ID names a second server entry (typically the + Base model), preset requests are routed to it. + + Each response is a complete WAV file, so sub-request audio is + concatenated with the same lossless path used for the Gradio client. + """ + + def __init__(self, voice: Optional[str] = None, language: Optional[str] = None, + api_url: Optional[str] = None): + self.api_url = (api_url or config.AUDIOCPP_API_URL).rstrip("/") + self.model_id = config.AUDIOCPP_MODEL_ID + # Validate before connecting so bad values fail fast without a server. + self.language = normalize_language( + language if language is not None else config.LANGUAGE) + # Same seed convention as the Gradio client: one value per run, + # reused for every request (see _resolve_request_seed). + self._seed = _resolve_request_seed() + self.preset_mode = bool(voice) + self.voice = voice or speaker_display_name() + self._check_health() + model_ids = self._check_model() + if self.preset_mode: + self._select_model(model_ids) + self._check_voice() + print(f"[OK] Connected to audio.cpp server at {self.api_url} " + f"(model '{self.model_id}', voice '{self.voice}')") + else: + print(f"[OK] Connected to audio.cpp server at {self.api_url} " + f"(model '{self.model_id}', speaker '{self.voice}')") + print("[INFO] Speaker mode expects the server to be configured with the " + "CustomVoice model; with the Base model the speaker name is ignored " + "and a random default voice is used (see README).") + + # ------------------------------------------------------------------ + # Connection + # ------------------------------------------------------------------ + + def _get_json(self, path: str, timeout: int = 10) -> Dict[str, Any]: + """GET a JSON document from the server.""" + url = f"{self.api_url}{path}" + try: + with urllib.request.urlopen(url, timeout=timeout) as response: + return json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + detail = "" + try: + detail = exc.read().decode("utf-8", errors="replace")[:200] + except Exception: + pass + raise RuntimeError( + f"audio.cpp server returned HTTP {exc.code} for {path}: {detail}") from exc + except urllib.error.URLError as exc: + raise RuntimeError(f"audio.cpp request failed for {path}: {exc.reason}") from exc + + def _check_health(self) -> None: + """Verify the server is reachable and reports healthy.""" + try: + payload = self._get_json("/health") + except Exception as exc: + raise RuntimeError( + f"audio.cpp server not reachable at {self.api_url}: {exc}. " + "Start audiocpp_server first (see the 'audio.cpp backend' " + "section of the README)." + ) from exc + if payload.get("status") != "ok": + raise RuntimeError( + f"The audio.cpp server at {self.api_url} reports status " + f"{payload.get('status')!r} instead of 'ok'") + + def _check_model(self) -> List[str]: + """Verify the configured model id exists; return all server model ids.""" + try: + payload = self._get_json("/v1/models") + except Exception as exc: + raise RuntimeError( + f"The audio.cpp server at {self.api_url} did not answer " + f"/v1/models: {exc}") from exc + entries = payload.get("data") or [] + model_ids = [entry.get("id") for entry in entries if isinstance(entry, dict)] + if self.model_id not in model_ids: + configured = ", ".join(str(mid) for mid in model_ids if mid) or "none" + raise RuntimeError( + f"The audio.cpp server at {self.api_url} has no model id " + f"'{self.model_id}' (configured: {configured}). Add a qwen3_tts " + "model entry to the server config and match AUDIOCPP_MODEL_ID " + "in converter/config.py to its id (see README)." + ) + return [mid for mid in model_ids if mid] + + def _select_model(self, model_ids: List[str]) -> None: + """Pick the model for preset (cloning) requests. + + Defaults to the primary model id. When AUDIOCPP_CLONE_MODEL_ID is + configured (typically a Base-model entry, since only that variant + consumes reference audio) and present on the server, preset + requests are routed to it instead, so one server can host the + CustomVoice model for speaker mode and the Base model for + cloning. + """ + clone_model_id = config.AUDIOCPP_CLONE_MODEL_ID + if not clone_model_id or clone_model_id == self.model_id: + return + if clone_model_id in model_ids: + self.model_id = clone_model_id + else: + logger.warning( + "AUDIOCPP_CLONE_MODEL_ID %r is not configured on the audio.cpp " + "server; preset requests use '%s' instead", + clone_model_id, self.model_id) + + def _check_voice(self) -> None: + """Verify the requested voice is available on the server. + + A voice name that matches no server preset or voice-library wav + would be passed through to the model as a cached voice id; on the + Base (cloning) model that is silently ignored and plain TTS audio + comes back, so preset names are validated up front. When the + voices endpoint cannot be queried, validation is skipped with a + warning rather than blocking the run. + """ + query = urllib.parse.urlencode({"model": self.model_id}) + try: + payload = self._get_json(f"/v1/audio/voices?{query}") + except Exception as exc: + logger.warning("Could not list server voices; skipping voice " + "validation: %s", exc) + return + voices = payload.get("voices") or [] + if self.voice not in voices: + available = ", ".join(str(v) for v in voices) or "none" + raise RuntimeError( + f"Voice '{self.voice}' is not available on the audio.cpp server " + f"(available: {available}). Configure it as a voice_preset or " + "voice_dir entry in the server config, or pass a listed name " + "with --voice (see README)." + ) + + # ------------------------------------------------------------------ + # HTTP requests + # ------------------------------------------------------------------ + + def _request_wav(self, text: str) -> bytes: + """POST one sub-chunk and return the raw WAV bytes.""" + url = f"{self.api_url}/v1/audio/speech" + payload: Dict[str, Any] = { + "model": self.model_id, + "input": text, + "voice": self.voice, + "language": self.language, + "seed": self._seed, + } + if not self.preset_mode and config.INSTRUCT: + # Style instruction for the CustomVoice speakers; ignored by + # the Base (cloning) model. + payload["instructions"] = config.INSTRUCT + request = urllib.request.Request( + url, data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, method="POST") + try: + with urllib.request.urlopen(request, timeout=config.API_TIMEOUT) as response: + wav = response.read() + except urllib.error.HTTPError as exc: + detail = "" + try: + detail = exc.read().decode("utf-8", errors="replace")[:200] + except Exception: + pass + raise RuntimeError(f"audio.cpp server returned HTTP {exc.code}: {detail}") from exc + except urllib.error.URLError as exc: + raise RuntimeError(f"audio.cpp request failed: {exc.reason}") from exc + if len(wav) < 12 or wav[:4] != b"RIFF" or wav[8:12] != b"WAVE": + raise RuntimeError("audio.cpp server returned audio that is not a WAV file") + return wav + + def _request_wav_with_retry(self, text: str, chunk_num: int, sub_num: int, + sub_total: int) -> bytes: + """Request one sub-chunk, retrying transient failures.""" + for attempt in range(config.MAX_RETRIES): + try: + return self._request_wav(text) + except Exception as exc: + logger.warning("Chunk %d sub-chunk %d/%d attempt %d failed: %s", + chunk_num, sub_num, sub_total, attempt + 1, exc) + if attempt < config.MAX_RETRIES - 1: + time.sleep(2 + 2 * attempt) + raise RuntimeError(f"Sub-chunk {sub_num}/{sub_total} failed after " + f"{config.MAX_RETRIES} attempts") + + # ------------------------------------------------------------------ + # Chunk generation + # ------------------------------------------------------------------ + + def generate_chunk(self, text: str, chunk_num: int) -> Optional[str]: + """Generate one audio chunk; returns its path in the chunks folder. + + The text is split into sub-requests of at most + ``MAX_REQUEST_WORDS`` words each (defense in depth against + pathological input, matching the Gradio client), each sub-request + returns a complete WAV file, and the parts are concatenated into + one chunk file. + """ + try: + sub_texts = split_into_chunks(text, max_words=MAX_REQUEST_WORDS) + if not sub_texts: + raise RuntimeError("No text to synthesize") + + output_path: Optional[Path] = None + with tempfile.TemporaryDirectory(prefix="tts_parts_") as parts_dir, \ + self._chunk_heartbeat(chunk_num): + part_paths = [] + for sub_num, sub_text in enumerate(sub_texts, 1): + wav = self._request_wav_with_retry( + sub_text, chunk_num, sub_num, len(sub_texts)) + destination = Path(parts_dir) / f"part_{sub_num:02d}.wav" + destination.write_bytes(wav) + check_for_truncation( + sub_text, _audio_duration_seconds(destination), + f"Chunk {chunk_num} sub-request {sub_num}/{len(sub_texts)}") + part_paths.append(destination) + if len(part_paths) == 1: + output_path = self._chunk_path(chunk_num, ".wav") + shutil.copy2(part_paths[0], output_path) + else: + output_path = self._chunk_path(chunk_num, ".wav") + concat_audio_files(part_paths, output_path) + + logger.debug("Chunk %d generated successfully (%d sub-request(s))", + chunk_num, len(sub_texts)) + return str(output_path) + + except Exception as exc: + logger.error("audio.cpp chunk processing failed for chunk %d: %s", + chunk_num, exc) + return None -- cgit v1.2.3