aboutsummaryrefslogtreecommitdiff
path: root/app
diff options
context:
space:
mode:
Diffstat (limited to 'app')
-rw-r--r--app/backends/probe.py11
-rw-r--r--app/backends/sglomni/catalog.py16
-rw-r--r--app/converter/clients/__init__.py2
-rw-r--r--app/converter/clients/audiocpp.py178
-rw-r--r--app/converter/clients/qwen.py18
-rw-r--r--app/converter/clients/sglomni.py15
-rw-r--r--app/converter/converter.py18
-rw-r--r--app/tests/test_converter_progress.py11
-rw-r--r--app/tests/test_hub.py14
-rw-r--r--app/tests/test_instruction_capabilities.py419
-rw-r--r--app/ui/hub.py32
11 files changed, 715 insertions, 19 deletions
diff --git a/app/backends/probe.py b/app/backends/probe.py
index d54a9f3..c84c88d 100644
--- a/app/backends/probe.py
+++ b/app/backends/probe.py
@@ -107,8 +107,15 @@ def _identify_health(base: str, timeout: float) -> Optional[str]:
def _identify_gradio(base: str, timeout: float) -> Optional[str]:
- """Identify a qwen-tts Gradio demo from its ``/info`` named endpoints."""
- payload = _get_json(f"{base}/info", timeout)
+ """Identify a qwen-tts Gradio demo from its ``/info`` named endpoints.
+
+ Modern Gradio (>= 4.x / 5.x) routes its API under ``/gradio_api`` —
+ its ``/info`` lives at ``/gradio_api/info`` with the legacy ``/info``
+ path gone or deprecated — so both prefixes are probed.
+ """
+ payload = _get_json(f"{base}/gradio_api/info", timeout)
+ if payload is None:
+ payload = _get_json(f"{base}/info", timeout)
if payload is None:
return None
endpoints = payload.get("named_endpoints")
diff --git a/app/backends/sglomni/catalog.py b/app/backends/sglomni/catalog.py
index 8bfb1fa..2452d82 100644
--- a/app/backends/sglomni/catalog.py
+++ b/app/backends/sglomni/catalog.py
@@ -52,6 +52,16 @@ class ModelEntry:
system_hint: Optional[str] = None # remediation when the binary is absent
speakers: Optional[Tuple[str, ...]] = None # preset voices (speaker)
supports_seed: bool = False # request-scoped seed accepted (Qwen3-TTS Base)
+ # Whether the model's serving pipeline consumes a separate
+ # "instructions" field alongside its normal voice conditioning
+ # (verified against the installed sglang_omni code, not assumed from
+ # the HTTP schema): Qwen3-TTS Base (clone + instruction conditioning
+ # in request_builders.py), Qwen3-TTS CustomVoice and VoiceDesign, and
+ # the MOSS v1.5 pair (reference + instruction in the user message).
+ # False would mean an instructions field is silently ignored (Higgs,
+ # Voxtral, fish, dots, ZONOS2 take none; fish's inline event tags
+ # belong in the text, not this field).
+ supports_instructions: bool = False
# NOTE(unverified upstream): only the two Base entries are known to
# accept a request-scoped seed (Voxtral rejects one outright); qwen's
# demo client does send seeds to the CustomVoice/VoiceDesign models,
@@ -161,6 +171,7 @@ ENTRIES: Tuple[ModelEntry, ...] = (
requires_reference=False,
extras=_QWEN_EXTRAS, system_dep="sox", system_hint=_SOX_HINT,
speakers=QWEN_CUSTOMVOICE_SPEAKERS,
+ supports_instructions=True,
notes="built-in speakers, lightest model",
),
ModelEntry(
@@ -172,6 +183,7 @@ ENTRIES: Tuple[ModelEntry, ...] = (
requires_reference=True,
extras=_QWEN_EXTRAS, system_dep="sox", system_hint=_SOX_HINT,
supports_seed=True,
+ supports_instructions=True,
notes="voice cloning from a reference clip",
),
ModelEntry(
@@ -183,6 +195,7 @@ ENTRIES: Tuple[ModelEntry, ...] = (
requires_reference=True,
extras=_QWEN_EXTRAS, system_dep="sox", system_hint=_SOX_HINT,
supports_seed=True,
+ supports_instructions=True,
notes="voice cloning, higher quality",
),
ModelEntry(
@@ -193,6 +206,7 @@ ENTRIES: Tuple[ModelEntry, ...] = (
capability=CAPABILITY_DESIGN,
requires_reference=False,
extras=_QWEN_EXTRAS, system_dep="sox", system_hint=_SOX_HINT,
+ supports_instructions=True,
notes="voice described by instructions",
),
ModelEntry(
@@ -229,6 +243,7 @@ ENTRIES: Tuple[ModelEntry, ...] = (
config="moss_tts.yaml",
capability=CAPABILITY_CLONE,
requires_reference=True,
+ supports_instructions=True,
notes="voice cloning from a reference clip",
),
ModelEntry(
@@ -238,6 +253,7 @@ ENTRIES: Tuple[ModelEntry, ...] = (
config="moss_tts_local.yaml",
capability=CAPABILITY_CLONE,
requires_reference=False,
+ supports_instructions=True,
notes="48 kHz, narration without a reference or cloning",
),
ModelEntry(
diff --git a/app/converter/clients/__init__.py b/app/converter/clients/__init__.py
index 5f38786..44f423b 100644
--- a/app/converter/clients/__init__.py
+++ b/app/converter/clients/__init__.py
@@ -47,6 +47,7 @@ from .audiocpp import (
AudioCppFamilyProfile,
AudioCppTTSClient,
audiocpp_entry_supports_design,
+ audiocpp_entry_supports_instructions,
audiocpp_entry_voice_capability,
audiocpp_family_narrates,
audiocpp_family_spec_tasks,
@@ -89,6 +90,7 @@ __all__ = [
"AudioCppFamilyProfile", "AUDIOCPP_DEFAULT_FAMILY_PROFILE",
"AUDIOCPP_FAMILY_PROFILES", "audiocpp_entry_voice_capability",
"audiocpp_entry_supports_design", "audiocpp_voice_for_run",
+ "audiocpp_entry_supports_instructions",
"audiocpp_family_narrates",
"audiocpp_family_spec_tasks", "audiocpp_family_voice_policy",
"audiocpp_request_error", "audiocpp_script_input",
diff --git a/app/converter/clients/audiocpp.py b/app/converter/clients/audiocpp.py
index 578a13b..9f52c40 100644
--- a/app/converter/clients/audiocpp.py
+++ b/app/converter/clients/audiocpp.py
@@ -62,10 +62,14 @@ AUDIOCPP_VOICE_DESIGN = "design" # voice described by --instructions (vdes)
# top-level "instructions" field (the default, read by most design/style
# implementations), or the "instruction" request option inside the
# "options" object (BreezeTTS 2: --request-option instruction=... and the
-# endpoint example send it there; its loader ignores the top-level field).
+# endpoint example send it there). audio.cpp's adapter translates the
+# top-level "instructions" field into that same "instruction" request
+# option for every family, so both channels reach the model.
AUDIOCPP_INSTRUCTION_FIELD = "field"
AUDIOCPP_INSTRUCTION_OPTION = "option"
+AUDIOCPP_FAMILY_BREEZE_TTS = "breeze_tts"
+
# HTTP error body fragments identifying deterministic request-configuration
# problems: the identical request will fail on every retry, so the chunk
# loop must give up immediately instead of burning its attempt budget.
@@ -76,6 +80,10 @@ AUDIOCPP_NON_RETRYABLE_ERRORS = (
# Cloning without the reference transcript (Qwen3-TTS Base ICL mode):
# the server-side voice has reference audio but no transcript for it.
"requires reference text",
+ # A CustomVoice entry given a voice preset that resolves to reference
+ # audio rather than a built-in speaker id: the model takes built-in
+ # speakers only.
+ "custom voice prefill requires speaker",
# The server cannot resolve a model contract for the family (its own
# hint text about model_specs/--model-spec-override follows the fragment).
"model contract spec not found for family",
@@ -168,6 +176,15 @@ AUDIOCPP_HINTED_ERRORS = (
"task and cannot generate audiobooks. Consider deleting the model "
"from the server configuration (re-run Configure Backends → "
"audio.cpp and unselect it)."),
+ # A CustomVoice entry whose selected server voice resolved to
+ # reference audio rather than a built-in speaker id (the flat voice
+ # list cannot distinguish them, so the connect-time check only
+ # warns): the model cannot clone reference audio.
+ ("custom voice prefill requires speaker",
+ "The CustomVoice model serves built-in speakers only: pick a "
+ "built-in speaker named on the entry (e.g. Vivian, Ryan, Uncle Fu) "
+ "with --voice, or select the Base model entry to clone a reference "
+ "voice."),
# Graph allocation failures: mostly device memory. The server's log
# carries the exact size the failed allocation attempted.
("failed to allocate", _ALLOCATION_HINT_TEXT()),
@@ -323,6 +340,11 @@ def audiocpp_family_voice_policy(family: str) -> str:
timbre reference despite the spec not declaring a clone task (Vevo2's
zero-shot TTS route). Unknown families (no local specs) keep the
conservative clone-only default the client has always applied.
+
+ _AUDIOCPP_KNOWN_FAMILY_POLICIES carries verified policies for
+ *specific* families, consulted before the local spec lookup: a family
+ whose local spec is absent (older audio.cpp checkout against a newer
+ server) or incomplete must not be misread as reference-required.
"""
if family == AUDIOCPP_FAMILY_QWEN3_TTS \
or family in AUDIOCPP_CLONE_ONLY_FAMILIES \
@@ -330,6 +352,8 @@ def audiocpp_family_voice_policy(family: str) -> str:
# Qwen3-TTS is entry-typed (speaker/clone/design capability per
# model id), so the family policy stays out of its way.
return AUDIOCPP_VOICE_REQUIRED
+ if family in _AUDIOCPP_KNOWN_FAMILY_POLICIES:
+ return _AUDIOCPP_KNOWN_FAMILY_POLICIES[family]
tasks = audiocpp_family_spec_tasks(family)
if not tasks:
return AUDIOCPP_VOICE_REQUIRED
@@ -585,6 +609,66 @@ AUDIOCPP_FAMILY_PROFILES = {
instruction_channel=AUDIOCPP_INSTRUCTION_OPTION),
}
+# Voice-policy overrides for families whose evidence contradicts their
+# local model spec (or whose spec predates audio.cpp's task declarations).
+# Kept as data so the spec lookup can stay the single generic path.
+_AUDIOCPP_KNOWN_FAMILY_POLICIES = {
+ # BreezeTTS 2 (upstream spec / implementation: tts, clone ("voice
+ # direction"), design). Older local checkouts carry no breeze_tts
+ # spec at all — without this fallback a remote Breeze entry reads as
+ # clone-only and an instructions-only run (voice direction without a
+ # reference) is refused at connect time.
+ AUDIOCPP_FAMILY_BREEZE_TTS: AUDIOCPP_VOICE_OPTIONAL,
+ # VibeVoice's spec declares only "tts", but its implementation
+ # explicitly accepts reference audio (the session uses a speaker
+ # reference when the request carries one): mixed tts+clone. The
+ # pick-falls-back rules then stop silently dropping the selected
+ # voice for it.
+ "vibevoice": AUDIOCPP_VOICE_OPTIONAL,
+}
+
+
+def audiocpp_entry_supports_instructions(family: str, task: str,
+ model_id: str) -> Optional[bool]:
+ """Whether an entry's model consumes a style instruction, if known.
+
+ Three-valued on purpose: True (the model's implementation reads the
+ instruction), False (proven not to), None (unknown — a spec the local
+ checkout does not describe or one without instruction markers).
+ Distinguishing unknown from unsupported keeps the TUI behind honest
+ wording instead of implying every family follows instructions.
+
+ Verified cases:
+ - task "vdes" entries: the voice comes from the instruction.
+ - BreezeTTS 2: the instruction conditions synthesis with or without a
+ reference (voice design / voice direction).
+ - Qwen3-TTS: the CustomVoice and VoiceDesign implementations read
+ "instruction"; the Base (cloning) implementation does not — a
+ variant-specific split, which is why Qwen is resolved per entry.
+ - Other families: True only when their local model spec declares an
+ "instruction"/"instruct" request option; None otherwise.
+ """
+ if task == AUDIOCPP_TASK_VDES:
+ return True
+ if family == AUDIOCPP_FAMILY_BREEZE_TTS:
+ return True
+ if family == AUDIOCPP_FAMILY_QWEN3_TTS:
+ lowered = (model_id or "").lower()
+ if "customvoice" in lowered:
+ return True
+ if "base" in lowered:
+ return False
+ return None
+ spec = _family_spec(family)
+ if spec is None:
+ return None
+ names = {str(option.get("name") or "")
+ for option in (spec.get("request_options") or [])
+ if isinstance(option, dict)}
+ if names & {"instruction", "instruct"}:
+ return True
+ return None
+
def audiocpp_script_input(prefix: str, text: str) -> str:
"""TEXT formatted as one "<PREFIX>: text" script line.
@@ -758,6 +842,15 @@ class AudioCppTTSClient(BaseTTSClient):
# Free-form per-request options (--option KEY=VALUE) forwarded in the
# request's "options" object; models ignore keys they don't know.
self.request_options: Dict[str, str] = dict(request_options or {})
+ # The instruction can also arrive as --option instruction=... The
+ # server folds both sources into one request option (the
+ # top-level field overwrites the option), so two *different*
+ # instructions would silently drop one — refuse instead, before a
+ # server is even contacted.
+ self._check_instruction_conflict()
+ # A guidance value recommended for instructed requests on specific
+ # families (resolved in _connect; None = no automatic guidance).
+ self._auto_guidance_scale: Optional[float] = None
# Set during _connect: design_mode for "vdes" entries, instruction_voice
# when a family without built-in speakers gets its voice from the
# instruction alone (no voice field), and plain_mode for plain-TTS
@@ -778,6 +871,51 @@ class AudioCppTTSClient(BaseTTSClient):
# Connection
# ------------------------------------------------------------------
+ def _resolve_auto_guidance(self) -> None:
+ """Select the request-strengthening guidance for instructed runs.
+
+ Breeze-TTS 2's model card recommends guidance scale 4 to
+ strengthen instruction-following (its own SDK examples carry
+ --cfg-scale 4 for voice direction and design); audio.cpp's
+ default is 1.0, which leaves the clone's reference delivery
+ dominant. When this run carries an instruction (the field or the
+ request option) and the user did not set their own guidance
+ value, the request gets the recommended one.
+ """
+ if self.family == AUDIOCPP_FAMILY_BREEZE_TTS \
+ and self.instructions \
+ and "guidance_scale" not in (getattr(
+ self, "request_options", {}) or {}):
+ self._auto_guidance_scale = 4.0
+ self._report("[INFO] Sending guidance_scale 4 with the "
+ "instruction (strengthens Breeze instruction "
+ "following; set --option guidance_scale=... to "
+ "override)")
+
+ def _check_instruction_conflict(self) -> None:
+ """Refuse two different instructions given at once.
+
+ The server normalizes both the top-level "instructions" field and
+ the "instruction" request option into the same request option,
+ with the field winning — so "Instructions: calm narration" plus
+ "--option instruction=screaming" would silently drop the field.
+ Identical values are allowed (same effective instruction).
+ """
+ option_instruction = (self.request_options.get("instruction")
+ or "").strip()
+ if self.instructions and option_instruction \
+ and self.instructions != option_instruction:
+ raise RuntimeError(
+ "Two conflicting instructions were given: the Instructions "
+ f"text ({self.instructions!r}) and the request option "
+ f"instruction={option_instruction!r}. Keep one source only: "
+ "an --option instruction=... overrides --instructions "
+ "silently, so use whichever you intend.")
+ if not self.instructions and option_instruction:
+ # Make the forwarded option visible where --instructions
+ # would be reported, for the run log's completeness.
+ self.instructions = option_instruction
+
def _connected(self, mode: str) -> None:
"""Report the resolved connection (MODE: speaker/voice/... label)."""
self._report(f"[OK] Connected to audio.cpp server at {self.api_url} "
@@ -833,6 +971,17 @@ class AudioCppTTSClient(BaseTTSClient):
f"--voice cannot be used with the voice design model "
f"'{self.model_id}': the voice is described by the "
"--instructions text instead (see README).")
+ if audiocpp_entry_voice_capability(
+ self.family, self.task, self.model_id) \
+ == AUDIOCPP_VOICE_SPEAKER \
+ and not is_builtin_speaker(self.voice):
+ self._report(
+ "[WARNING] The selected CustomVoice entry serves "
+ f"built-in speakers, and '{self.voice}' is not one "
+ "of them: unless this server preset maps to a "
+ "built-in speaker id, synthesis will fail with "
+ "'custom voice prefill requires speaker'. To clone "
+ "reference audio, select the Base model entry.")
self._check_voice()
self._connected(f"voice '{self.voice}'")
else:
@@ -892,6 +1041,21 @@ class AudioCppTTSClient(BaseTTSClient):
self._report(f"[INFO] Sending instruction with every request: {self.instructions}")
self._report("[INFO] Its effect (style, emotion, delivery) depends on the "
"model family; models without instruction support ignore it.")
+ # Breeze-TTS 2's model card recommends guidance scale 4 to
+ # strengthen instruction-following (its SDK default --cfg-scale
+ # 4 for voice direction); audio.cpp's default is 1.0, which
+ # leaves the clone's reference delivery dominant. When the run
+ # carries an instruction and the user did not set their own
+ # guidance value, send the recommended one.
+ self._resolve_auto_guidance()
+ logger.info(
+ "audio.cpp run settings: model=%s family=%s task=%s voice=%r "
+ "instruction=%r request_options=%s auto_guidance_scale=%s seed=%r",
+ self.model_id, self.family or "<unresolved>", self.task,
+ self.voice, self.instructions or None,
+ getattr(self, "request_options", {}) or {},
+ getattr(self, "_auto_guidance_scale", None),
+ None if self._seed < 0 else self._seed)
unload = (config.AUDIOCPP_UNLOAD_MODELS
if self._unload_models_override is None
else self._unload_models_override)
@@ -1210,11 +1374,21 @@ class AudioCppTTSClient(BaseTTSClient):
if self._seed >= 0:
# audio.cpp has no negative "randomize" seed; a negative seed
# means "let the server randomize", so the field is omitted.
- payload["seed"] = self._seed
+ # Above 2^53 a JSON number loses precision, so the full-range
+ # seeds audio.cpp documents (send uint64 as a decimal string)
+ # are sent as strings.
+ if self._seed > 2 ** 53:
+ payload["seed"] = str(self._seed)
+ else:
+ payload["seed"] = self._seed
if self.request_options:
# Generic per-model controls (--option KEY=VALUE): forwarded
# verbatim; the model ignores keys it does not know.
payload["options"] = dict(self.request_options)
+ if getattr(self, "_auto_guidance_scale", None) is not None:
+ # Instructed Breeze request without an explicit guidance
+ # value: the model card's recommended instruction strength.
+ payload["guidance_scale"] = self._auto_guidance_scale
if self.instructions:
# Explicit voice-design or style instruction (required for task
# "vdes" entries; a voice/style control on families that read
diff --git a/app/converter/clients/qwen.py b/app/converter/clients/qwen.py
index dd1c8ba..3f4025c 100644
--- a/app/converter/clients/qwen.py
+++ b/app/converter/clients/qwen.py
@@ -268,7 +268,16 @@ class QwenTTSClient(BaseTTSClient):
# ------------------------------------------------------------------
def _generate_custom_voice(self, text: str) -> Tuple:
- """Generate audio using CustomVoice mode with the run's speaker."""
+ """Generate audio using CustomVoice mode with the run's speaker.
+
+ When the endpoint exposes an instruction parameter and the run
+ carries one, it is sent as a delivery/style control (the demo's
+ run_instruct accepts an ``instruct`` argument alongside the
+ speaker; the model voices the text with that delivery instead of
+ the speaker's default). With no instruction the request is
+ unchanged.
+ """
+ instructions = getattr(self, "instructions", "") or ""
custom_api = self._resolve_api_name("/run_instruct", "/run_custom_voice", "/generate_custom_voice")
if custom_api == "/run_instruct":
payload = dict(
@@ -276,6 +285,9 @@ class QwenTTSClient(BaseTTSClient):
lang_disp=self.language,
spk_disp=speaker_display_name_for(self.speaker),
)
+ if instructions \
+ and self._endpoint_accepts_param(custom_api, "instruct"):
+ payload["instruct"] = instructions
else:
payload = dict(
text=text,
@@ -290,6 +302,10 @@ class QwenTTSClient(BaseTTSClient):
if self._endpoint_accepts_param(custom_api, "seed"):
payload["seed"] = self._seed
+ if instructions \
+ and self._endpoint_accepts_param(custom_api, "instruct"):
+ payload["instruct"] = instructions
+
return self.client.predict(**payload, api_name=custom_api)
def _generate_voice_design(self, text: str) -> Tuple:
diff --git a/app/converter/clients/sglomni.py b/app/converter/clients/sglomni.py
index 6b8493b..13b98d7 100644
--- a/app/converter/clients/sglomni.py
+++ b/app/converter/clients/sglomni.py
@@ -175,6 +175,11 @@ class SgOmniTTSClient(BaseTTSClient):
raise RuntimeError(
f"{entry.label} designs the voice from an instruction: "
'pass --instructions "..." describing the voice.')
+ if self.instructions and not entry.supports_instructions:
+ raise RuntimeError(
+ f"{entry.label} does not consume style instructions: the "
+ "server would silently ignore them. Remove --instructions, "
+ "or pick a model that supports them (Qwen3-TTS, MOSS-TTS).")
if entry.capability == "clone" and entry.requires_reference \
and not self.ref_audio:
raise RuntimeError(
@@ -377,6 +382,16 @@ class SgOmniTTSClient(BaseTTSClient):
payload["ref_audio"] = self._ref_audio_value()
if self.ref_text:
payload["ref_text"] = self.ref_text
+ if entry.supports_instructions and self.instructions:
+ # Reference + separate style instruction (Qwen3-TTS Base
+ # instruction conditioning, MOSS v1.5 user message). NOT
+ # the design task_type: the reference stays the voice.
+ payload["instructions"] = self.instructions
+ elif entry.supports_instructions and self.instructions:
+ # Speaker models (Qwen3-TTS CustomVoice) also shape the
+ # delivery with an instruction; unsupported models never get
+ # one (checked at connect).
+ payload["instructions"] = self.instructions
return payload
def _request_wav(self, text: str) -> bytes:
diff --git a/app/converter/converter.py b/app/converter/converter.py
index 3b878c8..2b849a2 100644
--- a/app/converter/converter.py
+++ b/app/converter/converter.py
@@ -137,9 +137,10 @@ def voice_mode_for(backend: str, voice: Optional[str] = None,
sglomni resolves from the selected model's capability — a design model
takes instructions, a clone-capable model clones when a reference .wav
is given and otherwise synthesizes its default voice, and a
- speaker-capable model takes a preset name; qwen designs with
- instructions, clones only with a reference .wav, and uses a built-in
- speaker otherwise), so the hub can run the pre-flight overwrite checks
+ speaker-capable model takes a preset name; qwen clones with a
+ reference .wav, takes a built-in speaker otherwise (instructions
+ sent as that model's delivery control), and designs with
+ instructions alone), so the hub can run the pre-flight overwrite checks
against exactly the output names the conversion will produce.
"""
if backend == BACKEND_FASTER:
@@ -157,6 +158,17 @@ def voice_mode_for(backend: str, voice: Optional[str] = None,
return VOICE_MODE_CUSTOM
# Unresolved model (the caller resolves it later): the qwen-style
# heuristic is the closest pre-flight approximation.
+ if backend == BACKEND_QWEN:
+ # Mirrors the CLI routing (audiobook.convert): a reference clone
+ # wins, then a built-in speaker (with instructions as that
+ # model's delivery control), then instructions alone (VoiceDesign).
+ if clone:
+ return VOICE_MODE_CLONE
+ if (voice or "").strip():
+ return VOICE_MODE_CUSTOM
+ if (instructions or "").strip():
+ return VOICE_MODE_DESIGN
+ return VOICE_MODE_CUSTOM
if (instructions or "").strip():
return VOICE_MODE_DESIGN
return VOICE_MODE_CLONE if clone else VOICE_MODE_CUSTOM
diff --git a/app/tests/test_converter_progress.py b/app/tests/test_converter_progress.py
index 0a44725..7fae974 100644
--- a/app/tests/test_converter_progress.py
+++ b/app/tests/test_converter_progress.py
@@ -53,14 +53,19 @@ class VoiceModeForTests(unittest.TestCase):
VOICE_MODE_CUSTOM)
def test_qwen_instructions_design(self):
- # Qwen: instructions alone select the VoiceDesign model, taking
- # precedence over a clone reference.
+ # Qwen: instructions alone select the VoiceDesign model. With a
+ # clone reference or a built-in speaker, the reference/speaker
+ # wins and the instructions only ride along (a directed run on
+ # models that support it).
self.assertEqual(voice_mode_for(BACKEND_QWEN,
instructions="A warm narrator"),
VOICE_MODE_DESIGN)
self.assertEqual(voice_mode_for(BACKEND_QWEN, clone="x.wav",
instructions="A warm narrator"),
- VOICE_MODE_DESIGN)
+ VOICE_MODE_CLONE)
+ self.assertEqual(voice_mode_for(BACKEND_QWEN, voice="Vivian",
+ instructions="A warm narrator"),
+ VOICE_MODE_CUSTOM)
self.assertEqual(voice_mode_for(BACKEND_QWEN, instructions=" "),
VOICE_MODE_CUSTOM)
diff --git a/app/tests/test_hub.py b/app/tests/test_hub.py
index 84796ee..90ed2d4 100644
--- a/app/tests/test_hub.py
+++ b/app/tests/test_hub.py
@@ -2873,7 +2873,10 @@ class ConvertFlowTests(unittest.TestCase):
elif entry.capability == "clone":
if entry.requires_reference:
overrides["voice"] = str(ref)
- else: # design
+ if entry.supports_instructions \
+ or entry.capability == "design":
+ # Supported entries show the optional
+ # delivery/style field; design requires it.
overrides["instructions"] = "A warm narrator."
self._answer_form(**overrides)
if hub.converter_mod.chunk_clamp_needed(entry):
@@ -2924,10 +2927,15 @@ class ConvertFlowTests(unittest.TestCase):
else: # design
expected = {"sglomni.model_id",
"sglomni.instructions"}
- self.assertEqual(kwargs.get("instructions"),
- "A warm narrator.")
self.assertNotIn("voice", kwargs)
self.assertNotIn("clone", kwargs)
+ if entry.supports_instructions:
+ # Design requires it; supported entries forward
+ # it as an optional delivery/style control.
+ expected = set(expected) | \
+ {"sglomni.instructions"}
+ self.assertEqual(kwargs.get("instructions"),
+ "A warm narrator.")
self.assertEqual(shown, expected)
diff --git a/app/tests/test_instruction_capabilities.py b/app/tests/test_instruction_capabilities.py
new file mode 100644
index 0000000..440ee08
--- /dev/null
+++ b/app/tests/test_instruction_capabilities.py
@@ -0,0 +1,419 @@
+"""Instruction-support and guidance regressions across the TTS backends.
+
+Covers the capabilities the models actually implement (verified against
+each backend's serving code) and what the clients send for them:
+Breeze-TTS 2's recommended guidance strength with instructions, the
+audio.cpp Qwen3-TTS variant split (CustomVoice reads instructions, the
+Base cloner does not), the SGLang models that consume a separate style
+instruction alongside their voice conditioning, and the Qwen demo's
+CustomVoice instruction parameter.
+"""
+
+import io
+import json
+import tempfile
+import unittest
+import wave
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+from converter.clients import (
+ BACKEND_QWEN, AudioCppTTSClient, QwenTTSClient,
+ VOICE_MODE_CLONE, VOICE_MODE_CUSTOM, VOICE_MODE_DESIGN,
+)
+from converter.clients.audiocpp import (
+ AUDIOCPP_FAMILY_BREEZE_TTS,
+ AUDIOCPP_FAMILY_PROFILES,
+ AUDIOCPP_VOICE_OPTIONAL,
+ audiocpp_entry_supports_instructions,
+ audiocpp_family_voice_policy,
+)
+
+
+_WAV_BYTES = b"RIFF\x18\x00\x00\x00WAVEfmt \x10\x00\x00\x00"
+
+
+# ---------------------------------------------------------------------------
+# Pure helpers
+# ---------------------------------------------------------------------------
+
+class VoicePolicyKnownFamiliesTests(unittest.TestCase):
+ """Families whose verified policy must survive a stale local spec."""
+
+ def test_breeze_is_tts_plus_clone_even_without_a_local_spec(self):
+ # Remote Breeze entries against older local checkouts carry no
+ # breeze_tts spec at all: the fallback keeps instructions-only
+ # voice direction connectable instead of demanding a reference.
+ self.assertEqual(
+ audiocpp_family_voice_policy(AUDIOCPP_FAMILY_BREEZE_TTS),
+ AUDIOCPP_VOICE_OPTIONAL)
+
+ def test_vibevoice_accepts_reference_audio_despite_its_spec(self):
+ # vibevoice.json declares only "tts", but the implementation
+ # accepts reference audio: a mixed tts+clone family, so a picked
+ # voice must not be silently dropped.
+ self.assertEqual(audiocpp_family_voice_policy("vibevoice"),
+ AUDIOCPP_VOICE_OPTIONAL)
+
+
+class EntryInstructionSupportTests(unittest.TestCase):
+ """audiocpp_entry_supports_instructions: True/False/None per entry."""
+
+ def test_breeze_supports_instructions(self):
+ self.assertIs(
+ audiocpp_entry_supports_instructions(
+ AUDIOCPP_FAMILY_BREEZE_TTS, "tts", "Breeze-TTS-2-GGUF"),
+ True)
+
+ def test_qwen_customvoice_and_design_support_instructions(self):
+ self.assertIs(
+ audiocpp_entry_supports_instructions(
+ "qwen3_tts", "tts", "Qwen3-TTS-12Hz-1.7B-CustomVoice-GGUF"),
+ True)
+ self.assertIs(
+ audiocpp_entry_supports_instructions(
+ "qwen3_tts", "vdes", "Qwen3-TTS-12Hz-1.7B-VoiceDesign"),
+ True)
+
+ def test_qwen_base_cloner_provably_does_not(self):
+ self.assertIs(
+ audiocpp_entry_supports_instructions(
+ "qwen3_tts", "tts", "Qwen3-TTS-12Hz-1.7B-Base-GGUF"),
+ False)
+
+ def test_unknown_families_are_unknown_not_unsupported(self):
+ self.assertIsNone(
+ audiocpp_entry_supports_instructions("chatterbox",
+ "tts", "x"))
+
+
+# ---------------------------------------------------------------------------
+# audio.cpp request payloads (instructed Breeze runs)
+# ---------------------------------------------------------------------------
+
+def _breeze_client(instructions=None, request_options=None,
+ voice="narrator", seed=-1):
+ """A fully-initialized Breeze client (no HTTP machinery touched)."""
+ with patch.object(AudioCppTTSClient, "_connect"):
+ client = AudioCppTTSClient(
+ Path("."), voice=voice, instructions=instructions,
+ request_options=request_options)
+ client.api_url = "http://127.0.0.1:8080"
+ client.model_id = "Breeze-TTS-2-GGUF"
+ client.family = AUDIOCPP_FAMILY_BREEZE_TTS
+ client.task = "tts"
+ client.profile = AUDIOCPP_FAMILY_PROFILES[AUDIOCPP_FAMILY_BREEZE_TTS]
+ client.design_mode = False
+ client.instruction_voice = False
+ client.plain_mode = False
+ client.preset_mode = True
+ client.speaker_mode = False
+ client._seed = seed
+ client._resolve_auto_guidance()
+ return client
+
+
+def _captured_payload(client):
+ """The JSON body _request_wav sends, via a stubbed urlopen."""
+ response = MagicMock()
+ response.read.return_value = _WAV_BYTES
+ response.__enter__ = lambda self: response
+ response.__exit__ = lambda self, *exc: None
+ with patch("converter.clients.audiocpp.urllib.request.urlopen") \
+ as urlopen:
+ urlopen.return_value = response
+ client._request_wav("Hello there.")
+ request = urlopen.call_args[0][0]
+ return json.loads(request.data.decode("utf-8"))
+
+
+class BreezeGuidanceDefaultTests(unittest.TestCase):
+ """Breeze guidance: recommended 4 with instructions, otherwise none."""
+
+ def test_instructed_clone_carries_guidance_4_and_the_instruction(self):
+ client = _breeze_client(instructions="Screaming, crazed, yelling")
+ self.assertEqual(client._auto_guidance_scale, 4.0)
+ payload = _captured_payload(client)
+ self.assertEqual(payload["guidance_scale"], 4.0)
+ self.assertEqual(payload["voice"], "narrator")
+ self.assertEqual(
+ payload["options"],
+ {"instruction": "Screaming, crazed, yelling"})
+ self.assertNotIn("instructions", payload)
+
+ def test_option_instruction_also_gets_the_guidance_default(self):
+ client = _breeze_client(request_options={
+ "instruction": "Read slowly and warmly."})
+ self.assertEqual(client._auto_guidance_scale, 4.0)
+ payload = _captured_payload(client)
+ self.assertEqual(payload["guidance_scale"], 4.0)
+ self.assertEqual(payload["options"]["instruction"],
+ "Read slowly and warmly.")
+
+ def test_explicit_guidance_option_is_preserved(self):
+ client = _breeze_client(
+ instructions="Screaming, crazed, yelling",
+ request_options={"guidance_scale": "2.5"})
+ self.assertIsNone(client._auto_guidance_scale)
+ payload = _captured_payload(client)
+ self.assertNotIn("guidance_scale", payload)
+ self.assertEqual(payload["options"]["guidance_scale"], "2.5")
+
+ def test_guidance_0_override_still_counts_as_explicit(self):
+ # 0 selects the instruction-free branch: a deliberate setting.
+ client = _breeze_client(
+ instructions="Screaming",
+ request_options={"guidance_scale": "0"})
+ payload = _captured_payload(client)
+ self.assertNotIn("guidance_scale", payload)
+
+ def test_plain_clone_without_instructions_uses_the_backend_default(self):
+ client = _breeze_client()
+ self.assertIsNone(client._auto_guidance_scale)
+ payload = _captured_payload(client)
+ self.assertNotIn("guidance_scale", payload)
+ self.assertNotIn("options", payload)
+ self.assertEqual(payload["voice"], "narrator")
+
+
+class InstructionConflictTests(unittest.TestCase):
+ """Two different instruction sources are refused before connecting."""
+
+ def test_conflicting_instructions_and_option_raise_without_a_server(self):
+ with self.assertRaises(RuntimeError) as ctx:
+ _breeze_client(instructions="calm narration",
+ request_options={"instruction": "screaming"})
+ self.assertIn("Two conflicting instructions",
+ str(ctx.exception))
+
+ def test_identical_instructions_from_both_sources_are_accepted(self):
+ client = _breeze_client(
+ instructions="calm narration",
+ request_options={"instruction": "calm narration"})
+ self.assertEqual(client.instructions, "calm narration")
+
+ def test_option_only_instruction_is_folded_into_the_reports(self):
+ client = _breeze_client(
+ request_options={"instruction": "calm narration"})
+ self.assertEqual(client.instructions, "calm narration")
+
+
+class SeedPrecisionTests(unittest.TestCase):
+ """Full-range uint64 seeds travel as decimal strings (audio.cpp docs)."""
+
+ def test_seed_above_2_pow_53_is_sent_as_a_string(self):
+ seed = 2 ** 53 + 3 # beyond the exact JSON-number integer range
+ client = _breeze_client(seed=seed)
+ payload = _captured_payload(client)
+ self.assertEqual(payload["seed"], str(seed))
+
+ def test_ordinary_seeds_stay_numbers(self):
+ client = _breeze_client(seed=42)
+ payload = _captured_payload(client)
+ self.assertEqual(payload["seed"], 42)
+
+
+# ---------------------------------------------------------------------------
+# SGLang-Omni: instructions on supported pipelines
+# ---------------------------------------------------------------------------
+
+class SgOmniInstructionTests(unittest.TestCase):
+ """instructions reach the payload only where the serving code reads it."""
+
+ _tmp_dir = None
+ _REF = None
+
+ @classmethod
+ def setUpClass(cls):
+ buffer = io.BytesIO()
+ with wave.open(buffer, "wb") as wav_file:
+ wav_file.setnchannels(1)
+ wav_file.setsampwidth(2)
+ wav_file.setframerate(24000)
+ wav_file.writeframes(b"\x01\x00" * 16)
+ cls._tmp_dir = tempfile.TemporaryDirectory()
+ cls._REF = Path(cls._tmp_dir.name) / "narrator.wav"
+ cls._REF.write_bytes(buffer.getvalue())
+ cls.addClassCleanup(cls._tmp_dir.cleanup)
+
+ def _client(self, model, **kwargs):
+ from converter.clients import SgOmniTTSClient
+ with patch.object(SgOmniTTSClient, "_connect"):
+ client = SgOmniTTSClient(
+ Path("."), model=model, ref_audio=str(self._REF),
+ ref_text="Hello transcript.", instructions="screaming",
+ **kwargs)
+ entry = client.entry
+ payload = client._request_payload("Hello there.")
+ return entry, payload
+
+ def test_qwen_base_clone_carries_ref_and_instruction(self):
+ entry, payload = self._client("qwen3_tts_1_7b_base")
+ self.assertIn("ref_audio", payload)
+ self.assertNotEqual(payload.get("task_type"), "VoiceDesign")
+ self.assertEqual(payload["instructions"], "screaming")
+
+ def test_moss_clone_carries_ref_and_instruction(self):
+ entry, payload = self._client("moss_tts")
+ self.assertIn("ref_audio", payload)
+ self.assertEqual(payload["instructions"], "screaming")
+
+ def test_customvoice_speaker_with_instruction(self):
+ entry, payload = self._client("qwen3_tts_0_6b_customvoice",
+ voice="Vivian")
+ self.assertEqual(payload["voice"], "Vivian")
+ self.assertEqual(payload["instructions"], "screaming")
+ self.assertNotIn("task_type", payload)
+
+ def test_design_remains_voice_design_with_instruction(self):
+ from converter.clients import SgOmniTTSClient
+ with patch.object(SgOmniTTSClient, "_connect"):
+ client = SgOmniTTSClient(
+ Path("."), model="qwen3_tts_1_7b_voicedesign",
+ instructions="a warm narrator")
+ payload = client._request_payload("Hello there.")
+ self.assertEqual(payload["task_type"], "VoiceDesign")
+ self.assertEqual(payload["instructions"], "a warm narrator")
+ self.assertNotIn("ref_audio", payload)
+
+ def test_unsupported_model_refuses_instructions_at_connect(self):
+ with self.assertRaises(RuntimeError) as ctx:
+ self._client("higgs_audio_v3_tts")
+ self.assertIn("does not consume style instructions",
+ str(ctx.exception))
+
+
+# ---------------------------------------------------------------------------
+# Qwen demo: CustomVoice instruction parameter
+# ---------------------------------------------------------------------------
+
+class QwenCustomVoiceInstructionTests(unittest.TestCase):
+ """The run_instruct endpoint takes an ``instruct`` delivery control."""
+
+ def test_run_instruct_sends_instruct_alongside_the_speaker(self):
+ client = QwenTTSClient.__new__(QwenTTSClient)
+ client.voice_mode = VOICE_MODE_CUSTOM
+ client.speaker = "Vivian"
+ client.language = "Auto"
+ client.instructions = "screaming, crazed"
+ client._seed = -1
+ client.client = MagicMock()
+ client._resolve_api_name = lambda *names: names[0]
+ client._endpoint_accepts_param = MagicMock(return_value=True)
+ client._generate_custom_voice("Hello there.")
+ predict = client.client.predict
+ predict.assert_called_once_with(
+ text="Hello there.", lang_disp="Auto",
+ spk_disp="Vivian", instruct="screaming, crazed",
+ api_name="/run_instruct")
+
+ def test_custom_voice_without_instructions_is_unchanged(self):
+ client = QwenTTSClient.__new__(QwenTTSClient)
+ client.voice_mode = VOICE_MODE_CUSTOM
+ client.speaker = "Vivian"
+ client.language = "Auto"
+ client.instructions = ""
+ client._seed = -1
+ client.client = MagicMock()
+ client._resolve_api_name = lambda *names: names[0]
+ client._endpoint_accepts_param = MagicMock(return_value=True)
+ client._generate_custom_voice("Hello there.")
+ _, kwargs = client.client.predict.call_args
+ self.assertNotIn("instruct", kwargs)
+
+
+# ---------------------------------------------------------------------------
+# audiobook voice-mode routing (qwen)
+# ---------------------------------------------------------------------------
+
+class CatalogInstructionFlagsTests(unittest.TestCase):
+ """Only the verified pipelines carry supports_instructions."""
+
+ def test_catalog_marks_only_the_verified_pipelines(self):
+ from backends.sglomni.catalog import ENTRIES
+ supported = {"qwen3_tts_0_6b_customvoice", "qwen3_tts_0_6b_base",
+ "qwen3_tts_1_7b_base", "qwen3_tts_1_7b_voicedesign",
+ "moss_tts", "moss_tts_local"}
+ for entry in ENTRIES:
+ with self.subTest(entry=entry.key):
+ self.assertEqual(entry.supports_instructions,
+ entry.key in supported)
+
+
+class GradioPrefixProbeTests(unittest.TestCase):
+ """Qwen demos under modern Gradio sit behind /gradio_api."""
+
+ def _identify(self, modern_payload, legacy_payload=None):
+ import backends.probe as probe
+ seen = []
+
+ def fake_get_json(url, timeout):
+ seen.append(url)
+ if url == "http://x/gradio_api/info":
+ return modern_payload
+ if url == "http://x/info":
+ return legacy_payload
+ return None
+
+ with patch.object(probe, "_get_json", side_effect=fake_get_json):
+ with patch.object(probe.common, "server_running",
+ return_value=True):
+ identity = probe._identify_gradio("http://x", 1.0)
+ return identity, seen
+
+ def test_modern_prefix_is_probed_first_and_identifies(self):
+ payload = {"named_endpoints": {"/run_instruct": {}}}
+ identity, seen = self._identify(payload)
+ self.assertEqual(identity, probe_identity("qwen-custom"))
+ self.assertEqual(seen, ["http://x/gradio_api/info"])
+
+ def test_legacy_info_still_identifies_older_gradio(self):
+ payload = {"named_endpoints": {"/run_voice_clone": {}}}
+ identity, seen = self._identify(None, payload)
+ self.assertEqual(identity, probe_identity("qwen-clone"))
+ self.assertEqual(seen, ["http://x/gradio_api/info",
+ "http://x/info"])
+
+ def test_neither_prefix_answers_none(self):
+ identity, _ = self._identify(None)
+ self.assertIsNone(identity)
+
+
+def probe_identity(name):
+ """The probe's IDENTITY_* constant for a backend NAME (local import)."""
+ import backends.probe as probe
+ return {"qwen-custom": probe.IDENTITY_QWEN_CUSTOM,
+ "qwen-clone": probe.IDENTITY_QWEN_CLONE,
+ }[name]
+
+
+# ---------------------------------------------------------------------------
+# audiobook voice-mode routing (qwen)
+# ---------------------------------------------------------------------------
+
+class QwenVoiceModeRoutingTests(unittest.TestCase):
+ """speaker + instructions is a directed CustomVoice run, not Design."""
+
+ def test_voice_mode_for_qwen_combinations(self):
+ from converter.converter import voice_mode_for
+ cases = [
+ (dict(voice=None, clone=None, instructions=None),
+ VOICE_MODE_CUSTOM),
+ (dict(voice=None, clone=None, instructions="screaming"),
+ VOICE_MODE_DESIGN),
+ (dict(voice="Vivian", clone=None, instructions="screaming"),
+ VOICE_MODE_CUSTOM),
+ (dict(voice=None, clone="ref.wav", instructions=None),
+ VOICE_MODE_CLONE),
+ ]
+ for kwargs, expected in cases:
+ with self.subTest(**kwargs):
+ self.assertEqual(
+ voice_mode_for(BACKEND_QWEN, voice=kwargs["voice"],
+ clone=kwargs["clone"],
+ instructions=kwargs["instructions"]),
+ expected)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/app/ui/hub.py b/app/ui/hub.py
index 7a6bfcc..cccf3bb 100644
--- a/app/ui/hub.py
+++ b/app/ui/hub.py
@@ -2294,9 +2294,25 @@ def _sglomni_fields(stdscr, api_url: Optional[str] = None,
voice_field["value"] = ""
def instructions_validate(value) -> Optional[str]:
- if model_capability(fields) != "design" or str(value).strip():
+ """Required on design entries (the voice comes from it); an
+ optional delivery/style control on entries that support one."""
+ if str(value or "").strip():
return None
- return "Describe the voice, e.g. 'A warm female narrator'"
+ if model_capability(fields) == "design":
+ return "Describe the voice, e.g. 'A warm female narrator'"
+ return None
+
+ def instructions_help(entry) -> list:
+ if entry.capability == "design":
+ return ["Describe the voice to design, e.g.",
+ '"A warm adult female narrator with a British accent".']
+ return ["Delivery/style instruction (supported by this model), e.g.",
+ '"Speak in a calm, soothing, and happy tone."']
+
+ def instructions_visible(fs) -> bool:
+ entry = model_entry(fs)
+ return (entry.capability == "design"
+ or getattr(entry, "supports_instructions", False))
# The Model picker reads as a table, like the audio.cpp one: pad every
# label to the widest one, then render each entry's capabilities as
@@ -2357,10 +2373,9 @@ def _sglomni_fields(stdscr, api_url: Optional[str] = None,
"clone")},
{"key": prefix + "instructions", "label": "Instructions",
"kind": "text", "value": "",
- "help": ["Describe the voice to design, e.g.",
- '"A warm adult female narrator with a British accent".'],
+ "help": lambda fs: instructions_help(model_entry(fs)),
"validate": instructions_validate,
- "visible": lambda fs: model_capability(fs) == "design"},
+ "visible": instructions_visible},
]
def mapper(result) -> Optional[tuple]:
@@ -2380,6 +2395,13 @@ def _sglomni_fields(stdscr, api_url: Optional[str] = None,
kwargs["voice"] = pick
else:
kwargs["instructions"] = result[prefix + "instructions"]
+ # Style/delivery instructions forward on every entry that takes
+ # them (design required, supported entries optional — the client
+ # rejects unsupported combinations instead of dropping the text).
+ instructions = ((result.get(prefix + "instructions")
+ or "").strip() or None)
+ if instructions:
+ kwargs["instructions"] = instructions
# The run view's Model/Voice rows: the catalog label, and the
# pick — or the clone reference's file name (the .wav stems,
# like the narrator tags). Design models describe the voice,
generated by cgit v1.2.3 (git 2.25.1) at 2026-09-09 23:07:00 +0000