From 130dcd988e0554a6343c92fd45d808fd508789b3 Mon Sep 17 00:00:00 2001 From: historia Date: Wed, 9 Sep 2026 17:02:41 -0400 Subject: fix: model-specific instructions support and set cfg-scale 4 for breeze automatically --- app/converter/clients/__init__.py | 2 + app/converter/clients/audiocpp.py | 178 +++++++++++++++++++++++++++++++++++++- app/converter/clients/qwen.py | 18 +++- app/converter/clients/sglomni.py | 15 ++++ app/converter/converter.py | 18 +++- 5 files changed, 225 insertions(+), 6 deletions(-) (limited to 'app/converter') diff --git a/app/converter/clients/__init__.py b/app/converter/clients/__init__.py index 5f38786..44f423b 100644 --- a/app/converter/clients/__init__.py +++ b/app/converter/clients/__init__.py @@ -47,6 +47,7 @@ from .audiocpp import ( AudioCppFamilyProfile, AudioCppTTSClient, audiocpp_entry_supports_design, + audiocpp_entry_supports_instructions, audiocpp_entry_voice_capability, audiocpp_family_narrates, audiocpp_family_spec_tasks, @@ -89,6 +90,7 @@ __all__ = [ "AudioCppFamilyProfile", "AUDIOCPP_DEFAULT_FAMILY_PROFILE", "AUDIOCPP_FAMILY_PROFILES", "audiocpp_entry_voice_capability", "audiocpp_entry_supports_design", "audiocpp_voice_for_run", + "audiocpp_entry_supports_instructions", "audiocpp_family_narrates", "audiocpp_family_spec_tasks", "audiocpp_family_voice_policy", "audiocpp_request_error", "audiocpp_script_input", diff --git a/app/converter/clients/audiocpp.py b/app/converter/clients/audiocpp.py index 578a13b..9f52c40 100644 --- a/app/converter/clients/audiocpp.py +++ b/app/converter/clients/audiocpp.py @@ -62,10 +62,14 @@ AUDIOCPP_VOICE_DESIGN = "design" # voice described by --instructions (vdes) # top-level "instructions" field (the default, read by most design/style # implementations), or the "instruction" request option inside the # "options" object (BreezeTTS 2: --request-option instruction=... and the -# endpoint example send it there; its loader ignores the top-level field). +# endpoint example send it there). audio.cpp's adapter translates the +# top-level "instructions" field into that same "instruction" request +# option for every family, so both channels reach the model. AUDIOCPP_INSTRUCTION_FIELD = "field" AUDIOCPP_INSTRUCTION_OPTION = "option" +AUDIOCPP_FAMILY_BREEZE_TTS = "breeze_tts" + # HTTP error body fragments identifying deterministic request-configuration # problems: the identical request will fail on every retry, so the chunk # loop must give up immediately instead of burning its attempt budget. @@ -76,6 +80,10 @@ AUDIOCPP_NON_RETRYABLE_ERRORS = ( # Cloning without the reference transcript (Qwen3-TTS Base ICL mode): # the server-side voice has reference audio but no transcript for it. "requires reference text", + # A CustomVoice entry given a voice preset that resolves to reference + # audio rather than a built-in speaker id: the model takes built-in + # speakers only. + "custom voice prefill requires speaker", # The server cannot resolve a model contract for the family (its own # hint text about model_specs/--model-spec-override follows the fragment). "model contract spec not found for family", @@ -168,6 +176,15 @@ AUDIOCPP_HINTED_ERRORS = ( "task and cannot generate audiobooks. Consider deleting the model " "from the server configuration (re-run Configure Backends → " "audio.cpp and unselect it)."), + # A CustomVoice entry whose selected server voice resolved to + # reference audio rather than a built-in speaker id (the flat voice + # list cannot distinguish them, so the connect-time check only + # warns): the model cannot clone reference audio. + ("custom voice prefill requires speaker", + "The CustomVoice model serves built-in speakers only: pick a " + "built-in speaker named on the entry (e.g. Vivian, Ryan, Uncle Fu) " + "with --voice, or select the Base model entry to clone a reference " + "voice."), # Graph allocation failures: mostly device memory. The server's log # carries the exact size the failed allocation attempted. ("failed to allocate", _ALLOCATION_HINT_TEXT()), @@ -323,6 +340,11 @@ def audiocpp_family_voice_policy(family: str) -> str: timbre reference despite the spec not declaring a clone task (Vevo2's zero-shot TTS route). Unknown families (no local specs) keep the conservative clone-only default the client has always applied. + + _AUDIOCPP_KNOWN_FAMILY_POLICIES carries verified policies for + *specific* families, consulted before the local spec lookup: a family + whose local spec is absent (older audio.cpp checkout against a newer + server) or incomplete must not be misread as reference-required. """ if family == AUDIOCPP_FAMILY_QWEN3_TTS \ or family in AUDIOCPP_CLONE_ONLY_FAMILIES \ @@ -330,6 +352,8 @@ def audiocpp_family_voice_policy(family: str) -> str: # Qwen3-TTS is entry-typed (speaker/clone/design capability per # model id), so the family policy stays out of its way. return AUDIOCPP_VOICE_REQUIRED + if family in _AUDIOCPP_KNOWN_FAMILY_POLICIES: + return _AUDIOCPP_KNOWN_FAMILY_POLICIES[family] tasks = audiocpp_family_spec_tasks(family) if not tasks: return AUDIOCPP_VOICE_REQUIRED @@ -585,6 +609,66 @@ AUDIOCPP_FAMILY_PROFILES = { instruction_channel=AUDIOCPP_INSTRUCTION_OPTION), } +# Voice-policy overrides for families whose evidence contradicts their +# local model spec (or whose spec predates audio.cpp's task declarations). +# Kept as data so the spec lookup can stay the single generic path. +_AUDIOCPP_KNOWN_FAMILY_POLICIES = { + # BreezeTTS 2 (upstream spec / implementation: tts, clone ("voice + # direction"), design). Older local checkouts carry no breeze_tts + # spec at all — without this fallback a remote Breeze entry reads as + # clone-only and an instructions-only run (voice direction without a + # reference) is refused at connect time. + AUDIOCPP_FAMILY_BREEZE_TTS: AUDIOCPP_VOICE_OPTIONAL, + # VibeVoice's spec declares only "tts", but its implementation + # explicitly accepts reference audio (the session uses a speaker + # reference when the request carries one): mixed tts+clone. The + # pick-falls-back rules then stop silently dropping the selected + # voice for it. + "vibevoice": AUDIOCPP_VOICE_OPTIONAL, +} + + +def audiocpp_entry_supports_instructions(family: str, task: str, + model_id: str) -> Optional[bool]: + """Whether an entry's model consumes a style instruction, if known. + + Three-valued on purpose: True (the model's implementation reads the + instruction), False (proven not to), None (unknown — a spec the local + checkout does not describe or one without instruction markers). + Distinguishing unknown from unsupported keeps the TUI behind honest + wording instead of implying every family follows instructions. + + Verified cases: + - task "vdes" entries: the voice comes from the instruction. + - BreezeTTS 2: the instruction conditions synthesis with or without a + reference (voice design / voice direction). + - Qwen3-TTS: the CustomVoice and VoiceDesign implementations read + "instruction"; the Base (cloning) implementation does not — a + variant-specific split, which is why Qwen is resolved per entry. + - Other families: True only when their local model spec declares an + "instruction"/"instruct" request option; None otherwise. + """ + if task == AUDIOCPP_TASK_VDES: + return True + if family == AUDIOCPP_FAMILY_BREEZE_TTS: + return True + if family == AUDIOCPP_FAMILY_QWEN3_TTS: + lowered = (model_id or "").lower() + if "customvoice" in lowered: + return True + if "base" in lowered: + return False + return None + spec = _family_spec(family) + if spec is None: + return None + names = {str(option.get("name") or "") + for option in (spec.get("request_options") or []) + if isinstance(option, dict)} + if names & {"instruction", "instruct"}: + return True + return None + def audiocpp_script_input(prefix: str, text: str) -> str: """TEXT formatted as one ": text" script line. @@ -758,6 +842,15 @@ class AudioCppTTSClient(BaseTTSClient): # Free-form per-request options (--option KEY=VALUE) forwarded in the # request's "options" object; models ignore keys they don't know. self.request_options: Dict[str, str] = dict(request_options or {}) + # The instruction can also arrive as --option instruction=... The + # server folds both sources into one request option (the + # top-level field overwrites the option), so two *different* + # instructions would silently drop one — refuse instead, before a + # server is even contacted. + self._check_instruction_conflict() + # A guidance value recommended for instructed requests on specific + # families (resolved in _connect; None = no automatic guidance). + self._auto_guidance_scale: Optional[float] = None # Set during _connect: design_mode for "vdes" entries, instruction_voice # when a family without built-in speakers gets its voice from the # instruction alone (no voice field), and plain_mode for plain-TTS @@ -778,6 +871,51 @@ class AudioCppTTSClient(BaseTTSClient): # Connection # ------------------------------------------------------------------ + def _resolve_auto_guidance(self) -> None: + """Select the request-strengthening guidance for instructed runs. + + Breeze-TTS 2's model card recommends guidance scale 4 to + strengthen instruction-following (its own SDK examples carry + --cfg-scale 4 for voice direction and design); audio.cpp's + default is 1.0, which leaves the clone's reference delivery + dominant. When this run carries an instruction (the field or the + request option) and the user did not set their own guidance + value, the request gets the recommended one. + """ + if self.family == AUDIOCPP_FAMILY_BREEZE_TTS \ + and self.instructions \ + and "guidance_scale" not in (getattr( + self, "request_options", {}) or {}): + self._auto_guidance_scale = 4.0 + self._report("[INFO] Sending guidance_scale 4 with the " + "instruction (strengthens Breeze instruction " + "following; set --option guidance_scale=... to " + "override)") + + def _check_instruction_conflict(self) -> None: + """Refuse two different instructions given at once. + + The server normalizes both the top-level "instructions" field and + the "instruction" request option into the same request option, + with the field winning — so "Instructions: calm narration" plus + "--option instruction=screaming" would silently drop the field. + Identical values are allowed (same effective instruction). + """ + option_instruction = (self.request_options.get("instruction") + or "").strip() + if self.instructions and option_instruction \ + and self.instructions != option_instruction: + raise RuntimeError( + "Two conflicting instructions were given: the Instructions " + f"text ({self.instructions!r}) and the request option " + f"instruction={option_instruction!r}. Keep one source only: " + "an --option instruction=... overrides --instructions " + "silently, so use whichever you intend.") + if not self.instructions and option_instruction: + # Make the forwarded option visible where --instructions + # would be reported, for the run log's completeness. + self.instructions = option_instruction + def _connected(self, mode: str) -> None: """Report the resolved connection (MODE: speaker/voice/... label).""" self._report(f"[OK] Connected to audio.cpp server at {self.api_url} " @@ -833,6 +971,17 @@ class AudioCppTTSClient(BaseTTSClient): f"--voice cannot be used with the voice design model " f"'{self.model_id}': the voice is described by the " "--instructions text instead (see README).") + if audiocpp_entry_voice_capability( + self.family, self.task, self.model_id) \ + == AUDIOCPP_VOICE_SPEAKER \ + and not is_builtin_speaker(self.voice): + self._report( + "[WARNING] The selected CustomVoice entry serves " + f"built-in speakers, and '{self.voice}' is not one " + "of them: unless this server preset maps to a " + "built-in speaker id, synthesis will fail with " + "'custom voice prefill requires speaker'. To clone " + "reference audio, select the Base model entry.") self._check_voice() self._connected(f"voice '{self.voice}'") else: @@ -892,6 +1041,21 @@ class AudioCppTTSClient(BaseTTSClient): self._report(f"[INFO] Sending instruction with every request: {self.instructions}") self._report("[INFO] Its effect (style, emotion, delivery) depends on the " "model family; models without instruction support ignore it.") + # Breeze-TTS 2's model card recommends guidance scale 4 to + # strengthen instruction-following (its SDK default --cfg-scale + # 4 for voice direction); audio.cpp's default is 1.0, which + # leaves the clone's reference delivery dominant. When the run + # carries an instruction and the user did not set their own + # guidance value, send the recommended one. + self._resolve_auto_guidance() + logger.info( + "audio.cpp run settings: model=%s family=%s task=%s voice=%r " + "instruction=%r request_options=%s auto_guidance_scale=%s seed=%r", + self.model_id, self.family or "", self.task, + self.voice, self.instructions or None, + getattr(self, "request_options", {}) or {}, + getattr(self, "_auto_guidance_scale", None), + None if self._seed < 0 else self._seed) unload = (config.AUDIOCPP_UNLOAD_MODELS if self._unload_models_override is None else self._unload_models_override) @@ -1210,11 +1374,21 @@ class AudioCppTTSClient(BaseTTSClient): if self._seed >= 0: # audio.cpp has no negative "randomize" seed; a negative seed # means "let the server randomize", so the field is omitted. - payload["seed"] = self._seed + # Above 2^53 a JSON number loses precision, so the full-range + # seeds audio.cpp documents (send uint64 as a decimal string) + # are sent as strings. + if self._seed > 2 ** 53: + payload["seed"] = str(self._seed) + else: + payload["seed"] = self._seed if self.request_options: # Generic per-model controls (--option KEY=VALUE): forwarded # verbatim; the model ignores keys it does not know. payload["options"] = dict(self.request_options) + if getattr(self, "_auto_guidance_scale", None) is not None: + # Instructed Breeze request without an explicit guidance + # value: the model card's recommended instruction strength. + payload["guidance_scale"] = self._auto_guidance_scale if self.instructions: # Explicit voice-design or style instruction (required for task # "vdes" entries; a voice/style control on families that read diff --git a/app/converter/clients/qwen.py b/app/converter/clients/qwen.py index dd1c8ba..3f4025c 100644 --- a/app/converter/clients/qwen.py +++ b/app/converter/clients/qwen.py @@ -268,7 +268,16 @@ class QwenTTSClient(BaseTTSClient): # ------------------------------------------------------------------ def _generate_custom_voice(self, text: str) -> Tuple: - """Generate audio using CustomVoice mode with the run's speaker.""" + """Generate audio using CustomVoice mode with the run's speaker. + + When the endpoint exposes an instruction parameter and the run + carries one, it is sent as a delivery/style control (the demo's + run_instruct accepts an ``instruct`` argument alongside the + speaker; the model voices the text with that delivery instead of + the speaker's default). With no instruction the request is + unchanged. + """ + instructions = getattr(self, "instructions", "") or "" custom_api = self._resolve_api_name("/run_instruct", "/run_custom_voice", "/generate_custom_voice") if custom_api == "/run_instruct": payload = dict( @@ -276,6 +285,9 @@ class QwenTTSClient(BaseTTSClient): lang_disp=self.language, spk_disp=speaker_display_name_for(self.speaker), ) + if instructions \ + and self._endpoint_accepts_param(custom_api, "instruct"): + payload["instruct"] = instructions else: payload = dict( text=text, @@ -290,6 +302,10 @@ class QwenTTSClient(BaseTTSClient): if self._endpoint_accepts_param(custom_api, "seed"): payload["seed"] = self._seed + if instructions \ + and self._endpoint_accepts_param(custom_api, "instruct"): + payload["instruct"] = instructions + return self.client.predict(**payload, api_name=custom_api) def _generate_voice_design(self, text: str) -> Tuple: diff --git a/app/converter/clients/sglomni.py b/app/converter/clients/sglomni.py index 6b8493b..13b98d7 100644 --- a/app/converter/clients/sglomni.py +++ b/app/converter/clients/sglomni.py @@ -175,6 +175,11 @@ class SgOmniTTSClient(BaseTTSClient): raise RuntimeError( f"{entry.label} designs the voice from an instruction: " 'pass --instructions "..." describing the voice.') + if self.instructions and not entry.supports_instructions: + raise RuntimeError( + f"{entry.label} does not consume style instructions: the " + "server would silently ignore them. Remove --instructions, " + "or pick a model that supports them (Qwen3-TTS, MOSS-TTS).") if entry.capability == "clone" and entry.requires_reference \ and not self.ref_audio: raise RuntimeError( @@ -377,6 +382,16 @@ class SgOmniTTSClient(BaseTTSClient): payload["ref_audio"] = self._ref_audio_value() if self.ref_text: payload["ref_text"] = self.ref_text + if entry.supports_instructions and self.instructions: + # Reference + separate style instruction (Qwen3-TTS Base + # instruction conditioning, MOSS v1.5 user message). NOT + # the design task_type: the reference stays the voice. + payload["instructions"] = self.instructions + elif entry.supports_instructions and self.instructions: + # Speaker models (Qwen3-TTS CustomVoice) also shape the + # delivery with an instruction; unsupported models never get + # one (checked at connect). + payload["instructions"] = self.instructions return payload def _request_wav(self, text: str) -> bytes: diff --git a/app/converter/converter.py b/app/converter/converter.py index 3b878c8..2b849a2 100644 --- a/app/converter/converter.py +++ b/app/converter/converter.py @@ -137,9 +137,10 @@ def voice_mode_for(backend: str, voice: Optional[str] = None, sglomni resolves from the selected model's capability — a design model takes instructions, a clone-capable model clones when a reference .wav is given and otherwise synthesizes its default voice, and a - speaker-capable model takes a preset name; qwen designs with - instructions, clones only with a reference .wav, and uses a built-in - speaker otherwise), so the hub can run the pre-flight overwrite checks + speaker-capable model takes a preset name; qwen clones with a + reference .wav, takes a built-in speaker otherwise (instructions + sent as that model's delivery control), and designs with + instructions alone), so the hub can run the pre-flight overwrite checks against exactly the output names the conversion will produce. """ if backend == BACKEND_FASTER: @@ -157,6 +158,17 @@ def voice_mode_for(backend: str, voice: Optional[str] = None, return VOICE_MODE_CUSTOM # Unresolved model (the caller resolves it later): the qwen-style # heuristic is the closest pre-flight approximation. + if backend == BACKEND_QWEN: + # Mirrors the CLI routing (audiobook.convert): a reference clone + # wins, then a built-in speaker (with instructions as that + # model's delivery control), then instructions alone (VoiceDesign). + if clone: + return VOICE_MODE_CLONE + if (voice or "").strip(): + return VOICE_MODE_CUSTOM + if (instructions or "").strip(): + return VOICE_MODE_DESIGN + return VOICE_MODE_CUSTOM if (instructions or "").strip(): return VOICE_MODE_DESIGN return VOICE_MODE_CLONE if clone else VOICE_MODE_CUSTOM -- cgit v1.2.3