feat: audio.cpp backend support

1 files changed, 278 insertions, 3 deletions

diff --git a/converter/tts.py b/converter/tts.py
index ff9da2b..741d9d3 100644
--- a/converter/tts.py
+++ b/converter/tts.py
@@ -4,6 +4,9 @@ QwenTTSClient talks to the Qwen3-TTS Gradio demos (custom voice / voice clone).
 FasterTTSClient talks to the OpenAI-compatible server from the
 faster-qwen3-tts repository (voice cloning only; the reference voice is
 configured server-side — see the "Faster backend" section of the README).
+AudioCppTTSClient talks to the audiocpp_server from the audio.cpp
+repository, which serves the same Qwen3-TTS models through an
+OpenAI-style API (see the "audio.cpp backend" section of the README).
 """
 
 import contextlib
@@ -17,6 +20,7 @@ import tempfile
 import threading
 import time
 import urllib.error
+import urllib.parse
 import urllib.request
 import wave
 from pathlib import Path
@@ -33,6 +37,12 @@ VOICE_MODE_CUSTOM = "custom_voice"
 VOICE_MODE_CLONE = "voice_clone"
 VOICE_MODES = (VOICE_MODE_CUSTOM, VOICE_MODE_CLONE)
 
+# TTS backends (re-exported for the CLI and the converter orchestrator).
+BACKEND_GRADIO = "gradio"
+BACKEND_FASTER = "faster"
+BACKEND_AUDIOCPP = "audiocpp"
+BACKENDS = (BACKEND_GRADIO, BACKEND_FASTER, BACKEND_AUDIOCPP)
+
 # Languages understood by the Qwen3-TTS API. Display names must match the
 # demo dropdown exactly (the demo silently falls back to "Auto" for
 # unrecognized values, so languages are validated client-side first).
@@ -92,6 +102,21 @@ SAMPLE_RATE = 24000
 CHUNKS_FOLDER = Path(__file__).resolve().parent.parent / "chunks"
 
 
+def _resolve_request_seed() -> int:
+    """Resolve the seed sent with every request.
+
+    Returns config.SEED as-is, or (with CONSTANT_SEED and SEED < 0) one
+    random value drawn per run, meant to be reused for every request so
+    the voice stays consistent across chunk boundaries. Without
+    CONSTANT_SEED, -1 is returned so the server re-samples the voice on
+    every generation.
+    """
+    seed = config.SEED
+    if config.CONSTANT_SEED and seed < 0:
+        seed = random.randrange(2 ** 31)
+    return seed
+
+
 def speaker_display_name() -> str:
     """Return the Gradio display name for the configured custom speaker."""
     return SPEAKER_DISPLAY_NAMES.get(
@@ -289,9 +314,7 @@ class QwenTTSClient(_BaseTTSClient):
         # reused for every request so the voice stays consistent across
         # chunk boundaries. Without CONSTANT_SEED, -1 is forwarded so the
         # server re-samples the voice on every generation.
-        self._seed = config.SEED
-        if config.CONSTANT_SEED and self._seed < 0:
-            self._seed = random.randrange(2 ** 31)
+        self._seed = _resolve_request_seed()
         if language is None:
             language = config.LANGUAGE
         # Validate before connecting so bad values fail fast without a server.
@@ -677,3 +700,255 @@ class FasterTTSClient(_BaseTTSClient):
         except Exception as exc:
             logger.error("Faster chunk processing failed for chunk %d: %s", chunk_num, exc)
             return None
+
+
+class AudioCppTTSClient(_BaseTTSClient):
+    """Generates audio chunks through an audio.cpp audiocpp_server.
+
+    Talks to the OpenAI-style HTTP API of audiocpp_server, which serves
+    the same Qwen3-TTS models as the Gradio demos through a native
+    ggml runtime (GGUF weights, no Python serving stack). Two voice
+    modes, both resolved server-side from the request's "voice" field:
+
+    - Speaker mode (no ``voice``): a built-in CustomVoice speaker name
+      (e.g. "Vivian") is passed through, plus the INSTRUCT style prompt.
+      The server must be configured with the CustomVoice model for this.
+    - Preset mode (``voice=NAME``): a voice configured on the server
+      (``voice_presets`` or ``voice_dir`` in its config, e.g. a cloning
+      reference). The name is validated against GET /v1/audio/voices at
+      startup because an unresolvable name would silently fall back to
+      plain TTS on the Base model instead of failing. When
+      AUDIOCPP_CLONE_MODEL_ID names a second server entry (typically the
+      Base model), preset requests are routed to it.
+
+    Each response is a complete WAV file, so sub-request audio is
+    concatenated with the same lossless path used for the Gradio client.
+    """
+
+    def __init__(self, voice: Optional[str] = None, language: Optional[str] = None,
+                 api_url: Optional[str] = None):
+        self.api_url = (api_url or config.AUDIOCPP_API_URL).rstrip("/")
+        self.model_id = config.AUDIOCPP_MODEL_ID
+        # Validate before connecting so bad values fail fast without a server.
+        self.language = normalize_language(
+            language if language is not None else config.LANGUAGE)
+        # Same seed convention as the Gradio client: one value per run,
+        # reused for every request (see _resolve_request_seed).
+        self._seed = _resolve_request_seed()
+        self.preset_mode = bool(voice)
+        self.voice = voice or speaker_display_name()
+        self._check_health()
+        model_ids = self._check_model()
+        if self.preset_mode:
+            self._select_model(model_ids)
+            self._check_voice()
+            print(f"[OK] Connected to audio.cpp server at {self.api_url} "
+                  f"(model '{self.model_id}', voice '{self.voice}')")
+        else:
+            print(f"[OK] Connected to audio.cpp server at {self.api_url} "
+                  f"(model '{self.model_id}', speaker '{self.voice}')")
+            print("[INFO] Speaker mode expects the server to be configured with the "
+                  "CustomVoice model; with the Base model the speaker name is ignored "
+                  "and a random default voice is used (see README).")
+
+    # ------------------------------------------------------------------
+    # Connection
+    # ------------------------------------------------------------------
+
+    def _get_json(self, path: str, timeout: int = 10) -> Dict[str, Any]:
+        """GET a JSON document from the server."""
+        url = f"{self.api_url}{path}"
+        try:
+            with urllib.request.urlopen(url, timeout=timeout) as response:
+                return json.loads(response.read().decode("utf-8"))
+        except urllib.error.HTTPError as exc:
+            detail = ""
+            try:
+                detail = exc.read().decode("utf-8", errors="replace")[:200]
+            except Exception:
+                pass
+            raise RuntimeError(
+                f"audio.cpp server returned HTTP {exc.code} for {path}: {detail}") from exc
+        except urllib.error.URLError as exc:
+            raise RuntimeError(f"audio.cpp request failed for {path}: {exc.reason}") from exc
+
+    def _check_health(self) -> None:
+        """Verify the server is reachable and reports healthy."""
+        try:
+            payload = self._get_json("/health")
+        except Exception as exc:
+            raise RuntimeError(
+                f"audio.cpp server not reachable at {self.api_url}: {exc}. "
+                "Start audiocpp_server first (see the 'audio.cpp backend' "
+                "section of the README)."
+            ) from exc
+        if payload.get("status") != "ok":
+            raise RuntimeError(
+                f"The audio.cpp server at {self.api_url} reports status "
+                f"{payload.get('status')!r} instead of 'ok'")
+
+    def _check_model(self) -> List[str]:
+        """Verify the configured model id exists; return all server model ids."""
+        try:
+            payload = self._get_json("/v1/models")
+        except Exception as exc:
+            raise RuntimeError(
+                f"The audio.cpp server at {self.api_url} did not answer "
+                f"/v1/models: {exc}") from exc
+        entries = payload.get("data") or []
+        model_ids = [entry.get("id") for entry in entries if isinstance(entry, dict)]
+        if self.model_id not in model_ids:
+            configured = ", ".join(str(mid) for mid in model_ids if mid) or "none"
+            raise RuntimeError(
+                f"The audio.cpp server at {self.api_url} has no model id "
+                f"'{self.model_id}' (configured: {configured}). Add a qwen3_tts "
+                "model entry to the server config and match AUDIOCPP_MODEL_ID "
+                "in converter/config.py to its id (see README)."
+            )
+        return [mid for mid in model_ids if mid]
+
+    def _select_model(self, model_ids: List[str]) -> None:
+        """Pick the model for preset (cloning) requests.
+
+        Defaults to the primary model id. When AUDIOCPP_CLONE_MODEL_ID is
+        configured (typically a Base-model entry, since only that variant
+        consumes reference audio) and present on the server, preset
+        requests are routed to it instead, so one server can host the
+        CustomVoice model for speaker mode and the Base model for
+        cloning.
+        """
+        clone_model_id = config.AUDIOCPP_CLONE_MODEL_ID
+        if not clone_model_id or clone_model_id == self.model_id:
+            return
+        if clone_model_id in model_ids:
+            self.model_id = clone_model_id
+        else:
+            logger.warning(
+                "AUDIOCPP_CLONE_MODEL_ID %r is not configured on the audio.cpp "
+                "server; preset requests use '%s' instead",
+                clone_model_id, self.model_id)
+
+    def _check_voice(self) -> None:
+        """Verify the requested voice is available on the server.
+
+        A voice name that matches no server preset or voice-library wav
+        would be passed through to the model as a cached voice id; on the
+        Base (cloning) model that is silently ignored and plain TTS audio
+        comes back, so preset names are validated up front. When the
+        voices endpoint cannot be queried, validation is skipped with a
+        warning rather than blocking the run.
+        """
+        query = urllib.parse.urlencode({"model": self.model_id})
+        try:
+            payload = self._get_json(f"/v1/audio/voices?{query}")
+        except Exception as exc:
+            logger.warning("Could not list server voices; skipping voice "
+                           "validation: %s", exc)
+            return
+        voices = payload.get("voices") or []
+        if self.voice not in voices:
+            available = ", ".join(str(v) for v in voices) or "none"
+            raise RuntimeError(
+                f"Voice '{self.voice}' is not available on the audio.cpp server "
+                f"(available: {available}). Configure it as a voice_preset or "
+                "voice_dir entry in the server config, or pass a listed name "
+                "with --voice (see README)."
+            )
+
+    # ------------------------------------------------------------------
+    # HTTP requests
+    # ------------------------------------------------------------------
+
+    def _request_wav(self, text: str) -> bytes:
+        """POST one sub-chunk and return the raw WAV bytes."""
+        url = f"{self.api_url}/v1/audio/speech"
+        payload: Dict[str, Any] = {
+            "model": self.model_id,
+            "input": text,
+            "voice": self.voice,
+            "language": self.language,
+            "seed": self._seed,
+        }
+        if not self.preset_mode and config.INSTRUCT:
+            # Style instruction for the CustomVoice speakers; ignored by
+            # the Base (cloning) model.
+            payload["instructions"] = config.INSTRUCT
+        request = urllib.request.Request(
+            url, data=json.dumps(payload).encode("utf-8"),
+            headers={"Content-Type": "application/json"}, method="POST")
+        try:
+            with urllib.request.urlopen(request, timeout=config.API_TIMEOUT) as response:
+                wav = response.read()
+        except urllib.error.HTTPError as exc:
+            detail = ""
+            try:
+                detail = exc.read().decode("utf-8", errors="replace")[:200]
+            except Exception:
+                pass
+            raise RuntimeError(f"audio.cpp server returned HTTP {exc.code}: {detail}") from exc
+        except urllib.error.URLError as exc:
+            raise RuntimeError(f"audio.cpp request failed: {exc.reason}") from exc
+        if len(wav) < 12 or wav[:4] != b"RIFF" or wav[8:12] != b"WAVE":
+            raise RuntimeError("audio.cpp server returned audio that is not a WAV file")
+        return wav
+
+    def _request_wav_with_retry(self, text: str, chunk_num: int, sub_num: int,
+                                sub_total: int) -> bytes:
+        """Request one sub-chunk, retrying transient failures."""
+        for attempt in range(config.MAX_RETRIES):
+            try:
+                return self._request_wav(text)
+            except Exception as exc:
+                logger.warning("Chunk %d sub-chunk %d/%d attempt %d failed: %s",
+                               chunk_num, sub_num, sub_total, attempt + 1, exc)
+                if attempt < config.MAX_RETRIES - 1:
+                    time.sleep(2 + 2 * attempt)
+        raise RuntimeError(f"Sub-chunk {sub_num}/{sub_total} failed after "
+                           f"{config.MAX_RETRIES} attempts")
+
+    # ------------------------------------------------------------------
+    # Chunk generation
+    # ------------------------------------------------------------------
+
+    def generate_chunk(self, text: str, chunk_num: int) -> Optional[str]:
+        """Generate one audio chunk; returns its path in the chunks folder.
+
+        The text is split into sub-requests of at most
+        ``MAX_REQUEST_WORDS`` words each (defense in depth against
+        pathological input, matching the Gradio client), each sub-request
+        returns a complete WAV file, and the parts are concatenated into
+        one chunk file.
+        """
+        try:
+            sub_texts = split_into_chunks(text, max_words=MAX_REQUEST_WORDS)
+            if not sub_texts:
+                raise RuntimeError("No text to synthesize")
+
+            output_path: Optional[Path] = None
+            with tempfile.TemporaryDirectory(prefix="tts_parts_") as parts_dir, \
+                    self._chunk_heartbeat(chunk_num):
+                part_paths = []
+                for sub_num, sub_text in enumerate(sub_texts, 1):
+                    wav = self._request_wav_with_retry(
+                        sub_text, chunk_num, sub_num, len(sub_texts))
+                    destination = Path(parts_dir) / f"part_{sub_num:02d}.wav"
+                    destination.write_bytes(wav)
+                    check_for_truncation(
+                        sub_text, _audio_duration_seconds(destination),
+                        f"Chunk {chunk_num} sub-request {sub_num}/{len(sub_texts)}")
+                    part_paths.append(destination)
+                if len(part_paths) == 1:
+                    output_path = self._chunk_path(chunk_num, ".wav")
+                    shutil.copy2(part_paths[0], output_path)
+                else:
+                    output_path = self._chunk_path(chunk_num, ".wav")
+                    concat_audio_files(part_paths, output_path)
+
+            logger.debug("Chunk %d generated successfully (%d sub-request(s))",
+                         chunk_num, len(sub_texts))
+            return str(output_path)
+
+        except Exception as exc:
+            logger.error("audio.cpp chunk processing failed for chunk %d: %s",
+                         chunk_num, exc)
+            return None