From df57cf2733e398473a58d788cd97fea3a618f892 Mon Sep 17 00:00:00 2001 From: historia Date: Sun, 23 Aug 2026 14:01:56 -0400 Subject: feat: tui for make_audiocpp_server_json --- converter/tts.py | 150 ++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 133 insertions(+), 17 deletions(-) (limited to 'converter/tts.py') diff --git a/converter/tts.py b/converter/tts.py index 9b54cf4..f5d54e2 100644 --- a/converter/tts.py +++ b/converter/tts.py @@ -115,6 +115,17 @@ AUDIOCPP_LANG_OMIT = "omit" # no language field; the model detects it # voice comes from a server-side preset requested with --voice. AUDIOCPP_FAMILY_QWEN3_TTS = "qwen3_tts" +# Server model entry tasks this client can synthesize audiobooks with, +# taken from GET /v1/models (the "task" field of each entry; servers that +# predate the field reported TTS models only, so a missing task is treated +# as "tts"). "vdes" entries are voice design models: the voice is described +# with --instructions instead of coming from a speaker or a reference clip. +# Entries with any other task (asr, vc, diar, ...) are rejected at connect +# time with a hint to pick a synthesis entry. +AUDIOCPP_TASK_TTS = "tts" +AUDIOCPP_TASK_VDES = "vdes" +AUDIOCPP_SYNTHESIS_TASKS = (AUDIOCPP_TASK_TTS, "clon", AUDIOCPP_TASK_VDES) + class AudioCppFamilyProfile: """Request conventions of one audio.cpp model family.""" @@ -748,17 +759,17 @@ class AudioCppTTSClient(_BaseTTSClient): Talks to the OpenAI-style HTTP API of audiocpp_server, which hosts TTS model families through a native ggml runtime (GGUF weights, no Python - serving stack). The server API is family-agnostic; the family of the - configured model entry is read from GET /v1/models at startup and - adapts the request payload (language field style, style instructions) - through AUDIOCPP_FAMILY_PROFILES. Two voice modes, both resolved - server-side from the request's "voice" field: + serving stack). The server API is family-agnostic; the family and task + of the configured model entry are read from GET /v1/models at startup + and adapt the request payload (language field style, style instructions) + through AUDIOCPP_FAMILY_PROFILES. Three voice modes, all resolved + server-side from the request's "voice"/"instructions" fields: - Speaker mode (no ``voice``): Qwen3-TTS only. A built-in CustomVoice speaker name (e.g. "Vivian") is passed through, plus the INSTRUCT style prompt. The server must be configured with the CustomVoice model for this. Families without built-in speakers reject this mode - with a hint to pass --voice. + with a hint to pass --voice (or --instructions, see below). - Preset mode (``voice=NAME``): a voice configured on the server (``voice_presets`` or ``voice_dir`` in its config, e.g. a cloning reference). The name is validated against GET /v1/audio/voices at @@ -769,6 +780,21 @@ class AudioCppTTSClient(_BaseTTSClient): to it. Only the entry actually used needs to exist on the server: a clone-only (Base) server works for --voice runs, while speaker mode on such a server fails with a hint to pass --voice. + - Voice design (task "vdes" entries, e.g. Qwen3-TTS VoiceDesign): the + voice is described in natural language through ``instructions``, + which is required and sent with every request (no ``voice`` field). + A constant per-run seed keeps the designed voice consistent across + chunk boundaries. + + ``instructions`` also works on non-design entries, where it acts as a + generic style/delivery instruction (voice control): families that read + it (OmniVoice, Qwen3-TTS CustomVoice, ...) shape the voice or delivery + accordingly, and others ignore it. On instruction-conditioned families + without built-in speakers it may replace --voice entirely (the + instruction defines the voice). Extra request options (``--option + KEY=VALUE``, e.g. emotion, voice_id, speed) are forwarded verbatim in + the request's "options" object, which is the server's generic + pass-through for per-model controls. Chunking: the server does its own long-form text chunking for every family (its ``text_chunk_size`` option, with a per-family default), so @@ -784,7 +810,9 @@ class AudioCppTTSClient(_BaseTTSClient): def __init__(self, voice: Optional[str] = None, language: Optional[str] = None, api_url: Optional[str] = None, chunk_text: bool = False, - model_id: Optional[str] = None): + model_id: Optional[str] = None, + instructions: Optional[str] = None, + request_options: Optional[Dict[str, str]] = None): self.api_url = (api_url or config.AUDIOCPP_API_URL).rstrip("/") # Per-run model selection: the --model CLI flag overrides config; an # empty value is resolved at connect time when the server hosts exactly @@ -802,13 +830,28 @@ class AudioCppTTSClient(_BaseTTSClient): self._seed = _resolve_request_seed() self.preset_mode = bool(voice) self.voice = voice or speaker_display_name() + # Style/voice-design instruction sent with every request (the CLI + # --instructions flag overrides AUDIOCPP_INSTRUCTIONS in config.py). + # For task "vdes" entries it describes the voice to design; for other + # families it is a generic style instruction when the model reads one. + self.instructions = (instructions if instructions is not None + else config.AUDIOCPP_INSTRUCTIONS or "").strip() + # Free-form per-request options (--option KEY=VALUE) forwarded in the + # request's "options" object; models ignore keys they don't know. + self.request_options: Dict[str, str] = dict(request_options or {}) + # Both set during _connect once the entry's task is known: design_mode + # for "vdes" entries, instruction_voice when a family without built-in + # speakers gets its voice from the instruction alone (no voice field). + self.design_mode = False + self.instruction_voice = False # When False (default), each chapter is sent as one request and the # server does its own long-form chunking (text_chunk_size); when True, # text is split client-side into CHUNK_SIZE-word sub-requests first. self.chunk_text = bool(chunk_text) - # Family of the selected model entry and its request profile; both - # are resolved from GET /v1/models during _connect. + # Family and task of the selected model entry and the family's request + # profile; all are resolved from GET /v1/models during _connect. self.family = "" + self.task = AUDIOCPP_TASK_TTS self.profile = AUDIOCPP_DEFAULT_FAMILY_PROFILE self._connect() @@ -817,12 +860,14 @@ class AudioCppTTSClient(_BaseTTSClient): # ------------------------------------------------------------------ def _connect(self) -> None: - """Health-check the server and resolve the model, family, and voice. + """Health-check the server and resolve the model, family, task, and voice. Speaker mode is only offered to families with built-in speakers (Qwen3-TTS); every other family must select a server-side voice - with --voice, so it fails fast with a hint instead of silently - synthesizing with a random default voice. + with --voice or describe one with --instructions, so it fails fast + with a hint instead of silently synthesizing with a random default + voice. Voice design entries (task "vdes") require --instructions + and reject --voice. """ self._check_health() models = self._list_models() @@ -831,7 +876,33 @@ class AudioCppTTSClient(_BaseTTSClient): self._select_model(models) self._require_model_id(models) self._resolve_family(models) - if self.preset_mode: + self._resolve_task(models) + if self.task not in AUDIOCPP_SYNTHESIS_TASKS: + available = ", ".join(model["id"] for model in models) or "none" + raise RuntimeError( + f"The audio.cpp model '{self.model_id}' has task " + f"'{self.task}'; audiobook.py can only synthesize with TTS " + f"model entries (tasks {', '.join(AUDIOCPP_SYNTHESIS_TASKS)}). " + f"Pick a synthesis entry with --model (available: {available})." + ) + if self.design_mode: + if self.preset_mode: + raise RuntimeError( + f"--voice cannot be used with the voice design model " + f"'{self.model_id}': the voice is described by the " + "--instructions text instead (see README).") + if not self.instructions: + raise RuntimeError( + f"The audio.cpp model '{self.model_id}' (family " + f"'{self.family}') is a voice design model: pass a " + "description of the voice to synthesize with, e.g. " + '--instructions "A warm adult female narrator with a ' + 'British accent" (see README).') + print(f"[OK] Connected to audio.cpp server at {self.api_url} " + f"(model '{self.model_id}', family '{self.family}', " + "voice design)") + print(f"[INFO] Designing the voice from: {self.instructions}") + elif self.preset_mode: self._check_voice() print(f"[OK] Connected to audio.cpp server at {self.api_url} " f"(model '{self.model_id}', family '{self.family}', " @@ -843,13 +914,26 @@ class AudioCppTTSClient(_BaseTTSClient): print("[INFO] Speaker mode expects the server to be configured with the " "CustomVoice model; with the Base model the speaker name is ignored " "and a random default voice is used (see README).") + elif self.instructions: + # Families without built-in speakers can still get their voice + # from the instruction alone (e.g. OmniVoice voice design). + self.instruction_voice = True + print(f"[OK] Connected to audio.cpp server at {self.api_url} " + f"(model '{self.model_id}', family '{self.family}', " + "instruction voice)") + print(f"[INFO] Designing the voice from: {self.instructions}") else: raise RuntimeError( f"The audio.cpp model '{self.model_id}' (family " f"'{self.family}') has no built-in speakers, so its voice " "must come from the server: rerun with --voice NAME " "matching a voice_preset or voice_dir entry in the server " - "config (see README).") + "config, or describe a voice with --instructions for " + "families that support it (see README).") + if self.instructions and not self.design_mode and not self.instruction_voice: + print(f"[INFO] Sending instruction with every request: {self.instructions}") + print("[INFO] Its effect (style, emotion, delivery) depends on the " + "model family; models without instruction support ignore it.") def _get_json(self, path: str, timeout: int = 10) -> Dict[str, Any]: """GET a JSON document from the server.""" @@ -884,7 +968,7 @@ class AudioCppTTSClient(_BaseTTSClient): f"{payload.get('status')!r} instead of 'ok'") def _list_models(self) -> List[Dict[str, str]]: - """Fetch the (id, family) pairs reported by the server.""" + """Fetch the (id, family, task) triples reported by the server.""" try: payload = self._get_json("/v1/models") except Exception as exc: @@ -898,6 +982,7 @@ class AudioCppTTSClient(_BaseTTSClient): models.append({ "id": entry["id"], "family": entry.get("family") or "", + "task": entry.get("task") or "", }) return models @@ -1034,6 +1119,25 @@ class AudioCppTTSClient(_BaseTTSClient): "generic profile (voice cloning via --voice, model-detected " "language)", family) + def _resolve_task(self, models: List[Dict[str, str]]) -> None: + """Resolve the selected model's task (tts, clon, vdes, ...) and set + design mode for voice design entries. + + The task comes from GET /v1/models and is fixed per server entry by + its server.json config (a VoiceDesign model must be hosted with + "task": "vdes"). Servers that predate the task field hosted plain + TTS models, so a missing task is treated as tts. + """ + entry = next( + (model for model in models if model["id"] == self.model_id), None) + task = (entry["task"] if entry is not None else "") or "" + if not task: + task = AUDIOCPP_TASK_TTS + logger.debug("Model '%s' reported no task; assuming tts", + self.model_id) + self.task = task + self.design_mode = task == AUDIOCPP_TASK_VDES + def _check_voice(self) -> None: """Verify the requested voice is available on the server. @@ -1076,8 +1180,12 @@ class AudioCppTTSClient(_BaseTTSClient): payload: Dict[str, Any] = { "model": self.model_id, "input": text, - "voice": self.voice, } + # Design models take no voice field (the voice comes from the + # instruction); instruction-voice runs on families without built-in + # speakers omit it too, since no speaker or preset was requested. + if not self.design_mode and not self.instruction_voice: + payload["voice"] = self.voice if self.profile.language_style == AUDIOCPP_LANG_DISPLAY: payload["language"] = self.language elif self.profile.language_style == AUDIOCPP_LANG_ISO: @@ -1092,11 +1200,19 @@ class AudioCppTTSClient(_BaseTTSClient): # audio.cpp has no negative "randomize" seed; a negative seed # means "let the server randomize", so the field is omitted. payload["seed"] = self._seed - if not self.preset_mode and config.INSTRUCT \ + if self.instructions: + # Explicit voice-design or style instruction (required for task + # "vdes" entries; a Ctrl/style control on families that read it). + payload["instructions"] = self.instructions + elif not self.preset_mode and config.INSTRUCT \ and self.profile.sends_instructions: # Style instruction for the Qwen3-TTS CustomVoice speakers; # ignored by the Base (cloning) model and other families. payload["instructions"] = config.INSTRUCT + if self.request_options: + # Generic per-model controls (--option KEY=VALUE): forwarded + # verbatim; the model ignores keys it does not know. + payload["options"] = dict(self.request_options) request = urllib.request.Request( url, data=json.dumps(payload).encode("utf-8"), headers={"Content-Type": "application/json"}, method="POST") -- cgit v1.2.3