diff options
Diffstat (limited to 'converter')
| -rw-r--r-- | converter/config.py | 10 | ||||
| -rw-r--r-- | converter/converter.py | 1 | ||||
| -rw-r--r-- | converter/tts.py | 296 |
3 files changed, 241 insertions, 66 deletions
diff --git a/converter/config.py b/converter/config.py index 83778ab..5cf4a8f 100644 --- a/converter/config.py +++ b/converter/config.py @@ -54,6 +54,14 @@ FASTER_VOICE = "default" ############################################################################### AUDIOCPP_API_URL = "http://127.0.0.1:8080" # audio.cpp audiocpp_server -# Model ids in the audio.cpp server.json config. +# Model ids in the audio.cpp server.json config. AUDIOCPP_MODEL_ID may point +# at any TTS model entry the server hosts (qwen3_tts, higgs_audio_tts, +# voxcpm2, index_tts2, ...); the family is detected from the server at +# startup and adapts the request automatically. Only qwen3_tts has built-in +# speakers (speaker mode); every other family needs --voice with a +# server-side voice preset. For single-model servers, set +# AUDIOCPP_CLONE_MODEL_ID to the same id as AUDIOCPP_MODEL_ID (or leave it +# empty); for Qwen3-TTS it typically names a second entry with the Base +# (cloning) model. AUDIOCPP_MODEL_ID = "qwen" AUDIOCPP_CLONE_MODEL_ID = "qwen-clone" diff --git a/converter/converter.py b/converter/converter.py index 84ac766..de302ad 100644 --- a/converter/converter.py +++ b/converter/converter.py @@ -562,6 +562,7 @@ class AudiobookConverter: elif self.backend == BACKEND_AUDIOCPP: print(f"audio.cpp endpoint: {config.AUDIOCPP_API_URL}") print(f"Model id: {self.tts.model_id}") + print(f"Model family: {getattr(self.tts, 'family', 'unknown')}") if self.voice: print("Backend: audio.cpp (voice cloning, reference configured on server)") print(f"Voice: {self.voice}") diff --git a/converter/tts.py b/converter/tts.py index 142cf6d..1a41643 100644 --- a/converter/tts.py +++ b/converter/tts.py @@ -5,8 +5,10 @@ FasterTTSClient talks to the OpenAI-compatible server from the faster-qwen3-tts repository (voice cloning only; the reference voice is configured server-side — see the "Faster backend" section of the README). AudioCppTTSClient talks to the audiocpp_server from the audio.cpp -repository, which serves the same Qwen3-TTS models through an -OpenAI-style API (see the "audio.cpp backend" section of the README). +repository, which can host any TTS model family audio.cpp supports +(Qwen3-TTS, Higgs Audio, VoxCPM2, IndexTTS2, ...) through one OpenAI-style +API; the family is detected from the server at startup (see the +"audio.cpp backend" sections of the README). """ import contextlib @@ -80,6 +82,74 @@ TTS_LANGUAGE_ALIASES = { "en-gb": "English", } +# Qwen display names -> ISO 639-1 codes, for audio.cpp families whose +# language request option takes a code instead of a display name. "Auto" +# has no code and maps to None so the field is omitted and the server +# applies its own default. +LANGUAGE_ISO_CODES = { + "Chinese": "zh", + "English": "en", + "German": "de", + "Italian": "it", + "Portuguese": "pt", + "Spanish": "es", + "Japanese": "ja", + "Korean": "ko", + "French": "fr", + "Russian": "ru", +} + +# --- audio.cpp model families --------------------------------------------- +# +# audiocpp_server exposes the same OpenAI-style API for every TTS family it +# hosts; families only differ in a few request conventions, captured here as +# profiles. Families that are not listed use the default profile below. + +# How the "language" request field is expressed by a family. +AUDIOCPP_LANG_DISPLAY = "display" # Qwen display names, e.g. "English" +AUDIOCPP_LANG_ISO = "iso" # ISO 639-1 codes, e.g. "en" +AUDIOCPP_LANG_OMIT = "omit" # no language field; the model detects it + +# The only family with a built-in speaker mode (CustomVoice speaker names +# plus the INSTRUCT style prompt). Every other family is clone-only: the +# voice comes from a server-side preset requested with --voice. +AUDIOCPP_FAMILY_QWEN3_TTS = "qwen3_tts" + + +class AudioCppFamilyProfile: + """Request conventions of one audio.cpp model family.""" + + def __init__(self, language_style: str = AUDIOCPP_LANG_OMIT, + sends_instructions: bool = False, + builtin_speakers: bool = False): + self.language_style = language_style + self.sends_instructions = sends_instructions + self.builtin_speakers = builtin_speakers + + +# Generic profile for families not listed in AUDIOCPP_FAMILY_PROFILES: +# clone-only, no style instructions, and no language field (the model +# detects the language itself). Describes higgs_audio_tts, voxcpm2, +# fish_audio, dots_tts, dramabox, omnivoice, outetts, glm_tts, miotts, +# moss_tts_*, pocket_tts, vibevoice, ... as well as families added to +# audio.cpp after this table was written. +AUDIOCPP_DEFAULT_FAMILY_PROFILE = AudioCppFamilyProfile() + +AUDIOCPP_FAMILY_PROFILES = { + AUDIOCPP_FAMILY_QWEN3_TTS: AudioCppFamilyProfile( + language_style=AUDIOCPP_LANG_DISPLAY, + sends_instructions=True, + builtin_speakers=True, + ), + # Families whose language option takes a code (e.g. "en") instead of + # a Qwen display name; otherwise clone-only like the default profile. + "chatterbox": AudioCppFamilyProfile(language_style=AUDIOCPP_LANG_ISO), + "confucius4_tts": AudioCppFamilyProfile(language_style=AUDIOCPP_LANG_ISO), + "index_tts2": AudioCppFamilyProfile(language_style=AUDIOCPP_LANG_ISO), + "magpie_tts": AudioCppFamilyProfile(language_style=AUDIOCPP_LANG_ISO), + "supertonic": AudioCppFamilyProfile(language_style=AUDIOCPP_LANG_ISO), +} + # Canonical speaker names -> display names used by the qwen-tts demo. SPEAKER_DISPLAY_NAMES = { "ryan": "Ryan", @@ -723,31 +793,37 @@ class FasterTTSClient(_BaseTTSClient): class AudioCppTTSClient(_BaseTTSClient): """Generates audio chunks through an audio.cpp audiocpp_server. - Talks to the OpenAI-style HTTP API of audiocpp_server, which serves - the same Qwen3-TTS models as the Gradio demos through a native - ggml runtime (GGUF weights, no Python serving stack). Two voice - modes, both resolved server-side from the request's "voice" field: - - - Speaker mode (no ``voice``): a built-in CustomVoice speaker name - (e.g. "Vivian") is passed through, plus the INSTRUCT style prompt. - The server must be configured with the CustomVoice model for this. + Talks to the OpenAI-style HTTP API of audiocpp_server, which hosts TTS + model families through a native ggml runtime (GGUF weights, no Python + serving stack). The server API is family-agnostic; the family of the + configured model entry is read from GET /v1/models at startup and + adapts the request payload (language field style, style instructions) + through AUDIOCPP_FAMILY_PROFILES. Two voice modes, both resolved + server-side from the request's "voice" field: + + - Speaker mode (no ``voice``): Qwen3-TTS only. A built-in CustomVoice + speaker name (e.g. "Vivian") is passed through, plus the INSTRUCT + style prompt. The server must be configured with the CustomVoice + model for this. Families without built-in speakers reject this mode + with a hint to pass --voice. - Preset mode (``voice=NAME``): a voice configured on the server (``voice_presets`` or ``voice_dir`` in its config, e.g. a cloning reference). The name is validated against GET /v1/audio/voices at startup because an unresolvable name would silently fall back to - plain TTS on the Base model instead of failing. When - AUDIOCPP_CLONE_MODEL_ID names a second server entry (typically the - Base model), preset requests are routed to it. Only the entry - actually used needs to exist on the server: a clone-only (Base) - server works for --voice runs, while speaker mode on such a server - fails with a hint to pass --voice. - - Chunking: the server does its own long-form text chunking (its - ``text_chunk_size`` option, 8192 chars by default for qwen3_tts), so by - default each chapter is sent as a single request and the audio comes - back already stitched. With ``chunk_text=True`` (the --chunk CLI flag), - text is instead split client-side into CHUNK_SIZE-word sub-requests, - which may needlessly double-chunk — the warning is printed by the CLI. + plain TTS on a clone-based model instead of failing. When + AUDIOCPP_CLONE_MODEL_ID names a second server entry of the same + family (typically the Qwen Base model), preset requests are routed + to it. Only the entry actually used needs to exist on the server: a + clone-only (Base) server works for --voice runs, while speaker mode + on such a server fails with a hint to pass --voice. + + Chunking: the server does its own long-form text chunking for every + family (its ``text_chunk_size`` option, with a per-family default), so + by default each chapter is sent as a single request and the audio + comes back already stitched. With ``chunk_text=True`` (the --chunk CLI + flag), text is instead split client-side into CHUNK_SIZE-word + sub-requests, which may needlessly double-chunk — the warning is + printed by the CLI. Each response is a complete WAV file, so sub-request audio is concatenated with the same lossless path used for the Gradio client. @@ -771,27 +847,49 @@ class AudioCppTTSClient(_BaseTTSClient): # server does its own long-form chunking (text_chunk_size); when True, # text is split client-side into CHUNK_SIZE-word sub-requests first. self.chunk_text = bool(chunk_text) + # Family of the selected model entry and its request profile; both + # are resolved from GET /v1/models during _connect. + self.family = "" + self.profile = AUDIOCPP_DEFAULT_FAMILY_PROFILE + self._connect() + + # ------------------------------------------------------------------ + # Connection + # ------------------------------------------------------------------ + + def _connect(self) -> None: + """Health-check the server and resolve the model, family, and voice. + + Speaker mode is only offered to families with built-in speakers + (Qwen3-TTS); every other family must select a server-side voice + with --voice, so it fails fast with a hint instead of silently + synthesizing with a random default voice. + """ self._check_health() - model_ids = self._list_model_ids() + models = self._list_models() + if self.preset_mode: + self._select_model(models) + self._require_model_id(models) + self._resolve_family(models) if self.preset_mode: - # Resolve the model before validating so the check covers the - # id actually used; a clone-only server works for --voice runs. - self._select_model(model_ids) - self._require_model_id(model_ids) self._check_voice() print(f"[OK] Connected to audio.cpp server at {self.api_url} " - f"(model '{self.model_id}', voice '{self.voice}')") - else: - self._require_model_id(model_ids) + f"(model '{self.model_id}', family '{self.family}', " + f"voice '{self.voice}')") + elif self.profile.builtin_speakers: print(f"[OK] Connected to audio.cpp server at {self.api_url} " - f"(model '{self.model_id}', speaker '{self.voice}')") + f"(model '{self.model_id}', family '{self.family}', " + f"speaker '{self.voice}')") print("[INFO] Speaker mode expects the server to be configured with the " "CustomVoice model; with the Base model the speaker name is ignored " "and a random default voice is used (see README).") - - # ------------------------------------------------------------------ - # Connection - # ------------------------------------------------------------------ + else: + raise RuntimeError( + f"The audio.cpp model '{self.model_id}' (family " + f"'{self.family}') has no built-in speakers, so its voice " + "must come from the server: rerun with --voice NAME " + "matching a voice_preset or voice_dir entry in the server " + "config (see README).") def _get_json(self, path: str, timeout: int = 10) -> Dict[str, Any]: """GET a JSON document from the server.""" @@ -825,8 +923,8 @@ class AudioCppTTSClient(_BaseTTSClient): f"The audio.cpp server at {self.api_url} reports status " f"{payload.get('status')!r} instead of 'ok'") - def _list_model_ids(self) -> List[str]: - """Fetch the model ids reported by the server.""" + def _list_models(self) -> List[Dict[str, str]]: + """Fetch the (id, family) pairs reported by the server.""" try: payload = self._get_json("/v1/models") except Exception as exc: @@ -834,16 +932,23 @@ class AudioCppTTSClient(_BaseTTSClient): f"The audio.cpp server at {self.api_url} did not answer " f"/v1/models: {exc}") from exc entries = payload.get("data") or [] - model_ids = [entry.get("id") for entry in entries if isinstance(entry, dict)] - return [mid for mid in model_ids if mid] - - def _require_model_id(self, model_ids: List[str]) -> None: + models: List[Dict[str, str]] = [] + for entry in entries: + if isinstance(entry, dict) and entry.get("id"): + models.append({ + "id": entry["id"], + "family": entry.get("family") or "", + }) + return models + + def _require_model_id(self, models: List[Dict[str, str]]) -> None: """Verify the model id chosen for this run exists on the server. Speaker mode needs AUDIOCPP_MODEL_ID (the CustomVoice entry). Preset mode validates whichever id _select_model resolved, so a - server hosting only the Base (cloning) model works for --voice. + server hosting only a cloning model works for --voice. """ + model_ids = [model["id"] for model in models] if self.model_id in model_ids: return configured = ", ".join(model_ids) or "none" @@ -852,39 +957,90 @@ class AudioCppTTSClient(_BaseTTSClient): f"The audio.cpp server at {self.api_url} has no model id " f"'{self.model_id}' or clone model id " f"'{config.AUDIOCPP_CLONE_MODEL_ID}' (configured: {configured}). " - "Add a qwen3_tts model entry to the server config and match " - "AUDIOCPP_MODEL_ID / AUDIOCPP_CLONE_MODEL_ID in " - "converter/config.py to its id (see README)." + "Add a TTS model entry for the family you want to the server " + "config and match AUDIOCPP_MODEL_ID / AUDIOCPP_CLONE_MODEL_ID " + "in converter/config.py to its id (see README)." ) raise RuntimeError( f"The audio.cpp server at {self.api_url} has no model id " f"'{self.model_id}' (configured: {configured}). Speaker mode needs " - "the CustomVoice model: add a qwen3_tts model entry to the server " - "config and match AUDIOCPP_MODEL_ID in converter/config.py to its " - "id, or rerun with --voice to use a cloning preset on the Base " + "the Qwen3-TTS CustomVoice model: add a qwen3_tts model entry to " + "the server config and match AUDIOCPP_MODEL_ID in converter/config.py to its " + "id, or rerun with --voice to use a voice preset on any TTS " "model (see README)." ) - def _select_model(self, model_ids: List[str]) -> None: + def _select_model(self, models: List[Dict[str, str]]) -> None: """Pick the model for preset (cloning) requests. Defaults to the primary model id. When AUDIOCPP_CLONE_MODEL_ID is - configured (typically a Base-model entry, since only that variant - consumes reference audio) and present on the server, preset - requests are routed to it instead, so one server can host the - CustomVoice model for speaker mode and the Base model for - cloning. + configured and present on the server, preset requests are routed + to it instead, so one server can host the CustomVoice model for + speaker mode and the Base model for cloning (Qwen3-TTS setups). + A clone id that names a model of a different family is ignored + with a warning, since preset requests must synthesize with the + family the run is configured for. """ clone_model_id = config.AUDIOCPP_CLONE_MODEL_ID if not clone_model_id or clone_model_id == self.model_id: return - if clone_model_id in model_ids: - self.model_id = clone_model_id - else: + families = {model["id"]: model["family"] for model in models} + if clone_model_id not in families: + # A qwen3_tts primary without its clone entry silently degrades + # (presets are ignored on the CustomVoice model), so that case + # keeps the warning; single-model servers of other families are + # the normal configuration and only get a debug note. + primary_is_qwen = (families.get(self.model_id) + or AUDIOCPP_FAMILY_QWEN3_TTS) \ + == AUDIOCPP_FAMILY_QWEN3_TTS + if primary_is_qwen: + logger.warning( + "AUDIOCPP_CLONE_MODEL_ID %r is not configured on the audio.cpp " + "server; preset requests use '%s' instead", + clone_model_id, self.model_id) + else: + logger.debug( + "AUDIOCPP_CLONE_MODEL_ID %r is not configured on the audio.cpp " + "server; preset requests use '%s' instead", + clone_model_id, self.model_id) + return + primary_family = families.get(self.model_id) + clone_family = families[clone_model_id] + if primary_family and clone_family and primary_family != clone_family: logger.warning( - "AUDIOCPP_CLONE_MODEL_ID %r is not configured on the audio.cpp " - "server; preset requests use '%s' instead", - clone_model_id, self.model_id) + "AUDIOCPP_CLONE_MODEL_ID %r hosts family %r, but " + "AUDIOCPP_MODEL_ID %r hosts %r; preset requests stay on " + "'%s'. Point both ids at the same model entry in " + "converter/config.py (single-model servers use the same id " + "for both)", + clone_model_id, clone_family, self.model_id, primary_family, + self.model_id) + return + self.model_id = clone_model_id + + def _resolve_family(self, models: List[Dict[str, str]]) -> None: + """Resolve the selected model's family and its request profile. + + The family comes from GET /v1/models. Servers that predate the + family field served Qwen3-TTS only, so a missing family is treated + as qwen3_tts, which also preserves this client's legacy behavior + against those versions. + """ + entry = next( + (model for model in models if model["id"] == self.model_id), None) + family = (entry["family"] if entry is not None else "") or "" + if not family: + family = AUDIOCPP_FAMILY_QWEN3_TTS + logger.debug("Model '%s' reported no family; assuming qwen3_tts", + self.model_id) + self.family = family + self.profile = AUDIOCPP_FAMILY_PROFILES.get( + family, AUDIOCPP_DEFAULT_FAMILY_PROFILE) + if family not in AUDIOCPP_FAMILY_PROFILES: + logger.info( + "audio.cpp family '%s' has no dedicated profile; using the " + "generic profile (voice cloning via --voice, model-detected " + "language)", family) def _check_voice(self) -> None: """Verify the requested voice is available on the server. @@ -929,15 +1085,25 @@ class AudioCppTTSClient(_BaseTTSClient): "model": self.model_id, "input": text, "voice": self.voice, - "language": self.language, } + if self.profile.language_style == AUDIOCPP_LANG_DISPLAY: + payload["language"] = self.language + elif self.profile.language_style == AUDIOCPP_LANG_ISO: + iso_code = LANGUAGE_ISO_CODES.get(self.language) + if iso_code: + payload["language"] = iso_code + else: + # "Auto": no code to send, so let the server pick its default. + logger.debug("%s: no language code for %r; omitted from request", + self.family, self.language) if self._seed >= 0: # audio.cpp has no negative "randomize" seed; a negative seed # means "let the server randomize", so the field is omitted. payload["seed"] = self._seed - if not self.preset_mode and config.INSTRUCT: - # Style instruction for the CustomVoice speakers; ignored by - # the Base (cloning) model. + if not self.preset_mode and config.INSTRUCT \ + and self.profile.sends_instructions: + # Style instruction for the Qwen3-TTS CustomVoice speakers; + # ignored by the Base (cloning) model and other families. payload["instructions"] = config.INSTRUCT request = urllib.request.Request( url, data=json.dumps(payload).encode("utf-8"), |