feat: experimental support for non-qwen models

1 files changed, 231 insertions, 65 deletions

diff --git a/converter/tts.py b/converter/tts.py
index 142cf6d..1a41643 100644
--- a/converter/tts.py
+++ b/converter/tts.py
@@ -5,8 +5,10 @@ FasterTTSClient talks to the OpenAI-compatible server from the
 faster-qwen3-tts repository (voice cloning only; the reference voice is
 configured server-side — see the "Faster backend" section of the README).
 AudioCppTTSClient talks to the audiocpp_server from the audio.cpp
-repository, which serves the same Qwen3-TTS models through an
-OpenAI-style API (see the "audio.cpp backend" section of the README).
+repository, which can host any TTS model family audio.cpp supports
+(Qwen3-TTS, Higgs Audio, VoxCPM2, IndexTTS2, ...) through one OpenAI-style
+API; the family is detected from the server at startup (see the
+"audio.cpp backend" sections of the README).
 """
 
 import contextlib
@@ -80,6 +82,74 @@ TTS_LANGUAGE_ALIASES = {
     "en-gb": "English",
 }
 
+# Qwen display names -> ISO 639-1 codes, for audio.cpp families whose
+# language request option takes a code instead of a display name. "Auto"
+# has no code and maps to None so the field is omitted and the server
+# applies its own default.
+LANGUAGE_ISO_CODES = {
+    "Chinese": "zh",
+    "English": "en",
+    "German": "de",
+    "Italian": "it",
+    "Portuguese": "pt",
+    "Spanish": "es",
+    "Japanese": "ja",
+    "Korean": "ko",
+    "French": "fr",
+    "Russian": "ru",
+}
+
+# --- audio.cpp model families ---------------------------------------------
+#
+# audiocpp_server exposes the same OpenAI-style API for every TTS family it
+# hosts; families only differ in a few request conventions, captured here as
+# profiles. Families that are not listed use the default profile below.
+
+# How the "language" request field is expressed by a family.
+AUDIOCPP_LANG_DISPLAY = "display"  # Qwen display names, e.g. "English"
+AUDIOCPP_LANG_ISO = "iso"          # ISO 639-1 codes, e.g. "en"
+AUDIOCPP_LANG_OMIT = "omit"        # no language field; the model detects it
+
+# The only family with a built-in speaker mode (CustomVoice speaker names
+# plus the INSTRUCT style prompt). Every other family is clone-only: the
+# voice comes from a server-side preset requested with --voice.
+AUDIOCPP_FAMILY_QWEN3_TTS = "qwen3_tts"
+
+
+class AudioCppFamilyProfile:
+    """Request conventions of one audio.cpp model family."""
+
+    def __init__(self, language_style: str = AUDIOCPP_LANG_OMIT,
+                 sends_instructions: bool = False,
+                 builtin_speakers: bool = False):
+        self.language_style = language_style
+        self.sends_instructions = sends_instructions
+        self.builtin_speakers = builtin_speakers
+
+
+# Generic profile for families not listed in AUDIOCPP_FAMILY_PROFILES:
+# clone-only, no style instructions, and no language field (the model
+# detects the language itself). Describes higgs_audio_tts, voxcpm2,
+# fish_audio, dots_tts, dramabox, omnivoice, outetts, glm_tts, miotts,
+# moss_tts_*, pocket_tts, vibevoice, ... as well as families added to
+# audio.cpp after this table was written.
+AUDIOCPP_DEFAULT_FAMILY_PROFILE = AudioCppFamilyProfile()
+
+AUDIOCPP_FAMILY_PROFILES = {
+    AUDIOCPP_FAMILY_QWEN3_TTS: AudioCppFamilyProfile(
+        language_style=AUDIOCPP_LANG_DISPLAY,
+        sends_instructions=True,
+        builtin_speakers=True,
+    ),
+    # Families whose language option takes a code (e.g. "en") instead of
+    # a Qwen display name; otherwise clone-only like the default profile.
+    "chatterbox": AudioCppFamilyProfile(language_style=AUDIOCPP_LANG_ISO),
+    "confucius4_tts": AudioCppFamilyProfile(language_style=AUDIOCPP_LANG_ISO),
+    "index_tts2": AudioCppFamilyProfile(language_style=AUDIOCPP_LANG_ISO),
+    "magpie_tts": AudioCppFamilyProfile(language_style=AUDIOCPP_LANG_ISO),
+    "supertonic": AudioCppFamilyProfile(language_style=AUDIOCPP_LANG_ISO),
+}
+
 # Canonical speaker names -> display names used by the qwen-tts demo.
 SPEAKER_DISPLAY_NAMES = {
     "ryan": "Ryan",
@@ -723,31 +793,37 @@ class FasterTTSClient(_BaseTTSClient):
 class AudioCppTTSClient(_BaseTTSClient):
     """Generates audio chunks through an audio.cpp audiocpp_server.
 
-    Talks to the OpenAI-style HTTP API of audiocpp_server, which serves
-    the same Qwen3-TTS models as the Gradio demos through a native
-    ggml runtime (GGUF weights, no Python serving stack). Two voice
-    modes, both resolved server-side from the request's "voice" field:
-
-    - Speaker mode (no ``voice``): a built-in CustomVoice speaker name
-      (e.g. "Vivian") is passed through, plus the INSTRUCT style prompt.
-      The server must be configured with the CustomVoice model for this.
+    Talks to the OpenAI-style HTTP API of audiocpp_server, which hosts TTS
+    model families through a native ggml runtime (GGUF weights, no Python
+    serving stack). The server API is family-agnostic; the family of the
+    configured model entry is read from GET /v1/models at startup and
+    adapts the request payload (language field style, style instructions)
+    through AUDIOCPP_FAMILY_PROFILES. Two voice modes, both resolved
+    server-side from the request's "voice" field:
+
+    - Speaker mode (no ``voice``): Qwen3-TTS only. A built-in CustomVoice
+      speaker name (e.g. "Vivian") is passed through, plus the INSTRUCT
+      style prompt. The server must be configured with the CustomVoice
+      model for this. Families without built-in speakers reject this mode
+      with a hint to pass --voice.
     - Preset mode (``voice=NAME``): a voice configured on the server
       (``voice_presets`` or ``voice_dir`` in its config, e.g. a cloning
       reference). The name is validated against GET /v1/audio/voices at
       startup because an unresolvable name would silently fall back to
-      plain TTS on the Base model instead of failing. When
-      AUDIOCPP_CLONE_MODEL_ID names a second server entry (typically the
-      Base model), preset requests are routed to it. Only the entry
-      actually used needs to exist on the server: a clone-only (Base)
-      server works for --voice runs, while speaker mode on such a server
-      fails with a hint to pass --voice.
-
-    Chunking: the server does its own long-form text chunking (its
-    ``text_chunk_size`` option, 8192 chars by default for qwen3_tts), so by
-    default each chapter is sent as a single request and the audio comes
-    back already stitched. With ``chunk_text=True`` (the --chunk CLI flag),
-    text is instead split client-side into CHUNK_SIZE-word sub-requests,
-    which may needlessly double-chunk — the warning is printed by the CLI.
+      plain TTS on a clone-based model instead of failing. When
+      AUDIOCPP_CLONE_MODEL_ID names a second server entry of the same
+      family (typically the Qwen Base model), preset requests are routed
+      to it. Only the entry actually used needs to exist on the server: a
+      clone-only (Base) server works for --voice runs, while speaker mode
+      on such a server fails with a hint to pass --voice.
+
+    Chunking: the server does its own long-form text chunking for every
+    family (its ``text_chunk_size`` option, with a per-family default), so
+    by default each chapter is sent as a single request and the audio
+    comes back already stitched. With ``chunk_text=True`` (the --chunk CLI
+    flag), text is instead split client-side into CHUNK_SIZE-word
+    sub-requests, which may needlessly double-chunk — the warning is
+    printed by the CLI.
 
     Each response is a complete WAV file, so sub-request audio is
     concatenated with the same lossless path used for the Gradio client.
@@ -771,27 +847,49 @@ class AudioCppTTSClient(_BaseTTSClient):
         # server does its own long-form chunking (text_chunk_size); when True,
         # text is split client-side into CHUNK_SIZE-word sub-requests first.
         self.chunk_text = bool(chunk_text)
+        # Family of the selected model entry and its request profile; both
+        # are resolved from GET /v1/models during _connect.
+        self.family = ""
+        self.profile = AUDIOCPP_DEFAULT_FAMILY_PROFILE
+        self._connect()
+
+    # ------------------------------------------------------------------
+    # Connection
+    # ------------------------------------------------------------------
+
+    def _connect(self) -> None:
+        """Health-check the server and resolve the model, family, and voice.
+
+        Speaker mode is only offered to families with built-in speakers
+        (Qwen3-TTS); every other family must select a server-side voice
+        with --voice, so it fails fast with a hint instead of silently
+        synthesizing with a random default voice.
+        """
         self._check_health()
-        model_ids = self._list_model_ids()
+        models = self._list_models()
+        if self.preset_mode:
+            self._select_model(models)
+        self._require_model_id(models)
+        self._resolve_family(models)
         if self.preset_mode:
-            # Resolve the model before validating so the check covers the
-            # id actually used; a clone-only server works for --voice runs.
-            self._select_model(model_ids)
-            self._require_model_id(model_ids)
             self._check_voice()
             print(f"[OK] Connected to audio.cpp server at {self.api_url} "
-                  f"(model '{self.model_id}', voice '{self.voice}')")
-        else:
-            self._require_model_id(model_ids)
+                  f"(model '{self.model_id}', family '{self.family}', "
+                  f"voice '{self.voice}')")
+        elif self.profile.builtin_speakers:
             print(f"[OK] Connected to audio.cpp server at {self.api_url} "
-                  f"(model '{self.model_id}', speaker '{self.voice}')")
+                  f"(model '{self.model_id}', family '{self.family}', "
+                  f"speaker '{self.voice}')")
             print("[INFO] Speaker mode expects the server to be configured with the "
                   "CustomVoice model; with the Base model the speaker name is ignored "
                   "and a random default voice is used (see README).")
-
-    # ------------------------------------------------------------------
-    # Connection
-    # ------------------------------------------------------------------
+        else:
+            raise RuntimeError(
+                f"The audio.cpp model '{self.model_id}' (family "
+                f"'{self.family}') has no built-in speakers, so its voice "
+                "must come from the server: rerun with --voice NAME "
+                "matching a voice_preset or voice_dir entry in the server "
+                "config (see README).")
 
     def _get_json(self, path: str, timeout: int = 10) -> Dict[str, Any]:
         """GET a JSON document from the server."""
@@ -825,8 +923,8 @@ class AudioCppTTSClient(_BaseTTSClient):
                 f"The audio.cpp server at {self.api_url} reports status "
                 f"{payload.get('status')!r} instead of 'ok'")
 
-    def _list_model_ids(self) -> List[str]:
-        """Fetch the model ids reported by the server."""
+    def _list_models(self) -> List[Dict[str, str]]:
+        """Fetch the (id, family) pairs reported by the server."""
         try:
             payload = self._get_json("/v1/models")
         except Exception as exc:
@@ -834,16 +932,23 @@ class AudioCppTTSClient(_BaseTTSClient):
                 f"The audio.cpp server at {self.api_url} did not answer "
                 f"/v1/models: {exc}") from exc
         entries = payload.get("data") or []
-        model_ids = [entry.get("id") for entry in entries if isinstance(entry, dict)]
-        return [mid for mid in model_ids if mid]
-
-    def _require_model_id(self, model_ids: List[str]) -> None:
+        models: List[Dict[str, str]] = []
+        for entry in entries:
+            if isinstance(entry, dict) and entry.get("id"):
+                models.append({
+                    "id": entry["id"],
+                    "family": entry.get("family") or "",
+                })
+        return models
+
+    def _require_model_id(self, models: List[Dict[str, str]]) -> None:
         """Verify the model id chosen for this run exists on the server.
 
         Speaker mode needs AUDIOCPP_MODEL_ID (the CustomVoice entry).
         Preset mode validates whichever id _select_model resolved, so a
-        server hosting only the Base (cloning) model works for --voice.
+        server hosting only a cloning model works for --voice.
         """
+        model_ids = [model["id"] for model in models]
         if self.model_id in model_ids:
             return
         configured = ", ".join(model_ids) or "none"
@@ -852,39 +957,90 @@ class AudioCppTTSClient(_BaseTTSClient):
                 f"The audio.cpp server at {self.api_url} has no model id "
                 f"'{self.model_id}' or clone model id "
                 f"'{config.AUDIOCPP_CLONE_MODEL_ID}' (configured: {configured}). "
-                "Add a qwen3_tts model entry to the server config and match "
-                "AUDIOCPP_MODEL_ID / AUDIOCPP_CLONE_MODEL_ID in "
-                "converter/config.py to its id (see README)."
+                "Add a TTS model entry for the family you want to the server "
+                "config and match AUDIOCPP_MODEL_ID / AUDIOCPP_CLONE_MODEL_ID "
+                "in converter/config.py to its id (see README)."
             )
         raise RuntimeError(
             f"The audio.cpp server at {self.api_url} has no model id "
             f"'{self.model_id}' (configured: {configured}). Speaker mode needs "
-            "the CustomVoice model: add a qwen3_tts model entry to the server "
-            "config and match AUDIOCPP_MODEL_ID in converter/config.py to its "
-            "id, or rerun with --voice to use a cloning preset on the Base "
+            "the Qwen3-TTS CustomVoice model: add a qwen3_tts model entry to "
+            "the server config and match AUDIOCPP_MODEL_ID in converter/config.py to its "
+            "id, or rerun with --voice to use a voice preset on any TTS "
             "model (see README)."
         )
 
-    def _select_model(self, model_ids: List[str]) -> None:
+    def _select_model(self, models: List[Dict[str, str]]) -> None:
         """Pick the model for preset (cloning) requests.
 
         Defaults to the primary model id. When AUDIOCPP_CLONE_MODEL_ID is
-        configured (typically a Base-model entry, since only that variant
-        consumes reference audio) and present on the server, preset
-        requests are routed to it instead, so one server can host the
-        CustomVoice model for speaker mode and the Base model for
-        cloning.
+        configured and present on the server, preset requests are routed
+        to it instead, so one server can host the CustomVoice model for
+        speaker mode and the Base model for cloning (Qwen3-TTS setups).
+        A clone id that names a model of a different family is ignored
+        with a warning, since preset requests must synthesize with the
+        family the run is configured for.
         """
         clone_model_id = config.AUDIOCPP_CLONE_MODEL_ID
         if not clone_model_id or clone_model_id == self.model_id:
             return
-        if clone_model_id in model_ids:
-            self.model_id = clone_model_id
-        else:
+        families = {model["id"]: model["family"] for model in models}
+        if clone_model_id not in families:
+            # A qwen3_tts primary without its clone entry silently degrades
+            # (presets are ignored on the CustomVoice model), so that case
+            # keeps the warning; single-model servers of other families are
+            # the normal configuration and only get a debug note.
+            primary_is_qwen = (families.get(self.model_id)
+                               or AUDIOCPP_FAMILY_QWEN3_TTS) \
+                == AUDIOCPP_FAMILY_QWEN3_TTS
+            if primary_is_qwen:
+                logger.warning(
+                    "AUDIOCPP_CLONE_MODEL_ID %r is not configured on the audio.cpp "
+                    "server; preset requests use '%s' instead",
+                    clone_model_id, self.model_id)
+            else:
+                logger.debug(
+                    "AUDIOCPP_CLONE_MODEL_ID %r is not configured on the audio.cpp "
+                    "server; preset requests use '%s' instead",
+                    clone_model_id, self.model_id)
+            return
+        primary_family = families.get(self.model_id)
+        clone_family = families[clone_model_id]
+        if primary_family and clone_family and primary_family != clone_family:
             logger.warning(
-                "AUDIOCPP_CLONE_MODEL_ID %r is not configured on the audio.cpp "
-                "server; preset requests use '%s' instead",
-                clone_model_id, self.model_id)
+                "AUDIOCPP_CLONE_MODEL_ID %r hosts family %r, but "
+                "AUDIOCPP_MODEL_ID %r hosts %r; preset requests stay on "
+                "'%s'. Point both ids at the same model entry in "
+                "converter/config.py (single-model servers use the same id "
+                "for both)",
+                clone_model_id, clone_family, self.model_id, primary_family,
+                self.model_id)
+            return
+        self.model_id = clone_model_id
+
+    def _resolve_family(self, models: List[Dict[str, str]]) -> None:
+        """Resolve the selected model's family and its request profile.
+
+        The family comes from GET /v1/models. Servers that predate the
+        family field served Qwen3-TTS only, so a missing family is treated
+        as qwen3_tts, which also preserves this client's legacy behavior
+        against those versions.
+        """
+        entry = next(
+            (model for model in models if model["id"] == self.model_id), None)
+        family = (entry["family"] if entry is not None else "") or ""
+        if not family:
+            family = AUDIOCPP_FAMILY_QWEN3_TTS
+            logger.debug("Model '%s' reported no family; assuming qwen3_tts",
+                         self.model_id)
+        self.family = family
+        self.profile = AUDIOCPP_FAMILY_PROFILES.get(
+            family, AUDIOCPP_DEFAULT_FAMILY_PROFILE)
+        if family not in AUDIOCPP_FAMILY_PROFILES:
+            logger.info(
+                "audio.cpp family '%s' has no dedicated profile; using the "
+                "generic profile (voice cloning via --voice, model-detected "
+                "language)", family)
 
     def _check_voice(self) -> None:
         """Verify the requested voice is available on the server.
@@ -929,15 +1085,25 @@ class AudioCppTTSClient(_BaseTTSClient):
             "model": self.model_id,
             "input": text,
             "voice": self.voice,
-            "language": self.language,
         }
+        if self.profile.language_style == AUDIOCPP_LANG_DISPLAY:
+            payload["language"] = self.language
+        elif self.profile.language_style == AUDIOCPP_LANG_ISO:
+            iso_code = LANGUAGE_ISO_CODES.get(self.language)
+            if iso_code:
+                payload["language"] = iso_code
+            else:
+                # "Auto": no code to send, so let the server pick its default.
+                logger.debug("%s: no language code for %r; omitted from request",
+                             self.family, self.language)
         if self._seed >= 0:
             # audio.cpp has no negative "randomize" seed; a negative seed
             # means "let the server randomize", so the field is omitted.
             payload["seed"] = self._seed
-        if not self.preset_mode and config.INSTRUCT:
-            # Style instruction for the CustomVoice speakers; ignored by
-            # the Base (cloning) model.
+        if not self.preset_mode and config.INSTRUCT \
+                and self.profile.sends_instructions:
+            # Style instruction for the Qwen3-TTS CustomVoice speakers;
+            # ignored by the Base (cloning) model and other families.
             payload["instructions"] = config.INSTRUCT
         request = urllib.request.Request(
             url, data=json.dumps(payload).encode("utf-8"),