diff options
Diffstat (limited to 'app/converter/clients')
| -rw-r--r-- | app/converter/clients/__init__.py | 3 | ||||
| -rw-r--r-- | app/converter/clients/audiocpp.py | 215 | ||||
| -rw-r--r-- | app/converter/clients/qwen.py | 18 | ||||
| -rw-r--r-- | app/converter/clients/sglomni.py | 162 |
4 files changed, 344 insertions, 54 deletions
diff --git a/app/converter/clients/__init__.py b/app/converter/clients/__init__.py index d2a7f8d..44f423b 100644 --- a/app/converter/clients/__init__.py +++ b/app/converter/clients/__init__.py @@ -47,6 +47,7 @@ from .audiocpp import ( AudioCppFamilyProfile, AudioCppTTSClient, audiocpp_entry_supports_design, + audiocpp_entry_supports_instructions, audiocpp_entry_voice_capability, audiocpp_family_narrates, audiocpp_family_spec_tasks, @@ -83,11 +84,13 @@ __all__ = [ "AUDIOCPP_FAMILY_QWEN3_TTS", "AUDIOCPP_TASK_TTS", "AUDIOCPP_TASK_VDES", "AUDIOCPP_SYNTHESIS_TASKS", "AUDIOCPP_VOICE_SPEAKER", "AUDIOCPP_VOICE_CLONE", "AUDIOCPP_VOICE_DESIGN", + "AUDIOCPP_INSTRUCTION_FIELD", "AUDIOCPP_INSTRUCTION_OPTION", "AUDIOCPP_CLONE_ONLY_FAMILIES", "AUDIOCPP_VOICE_REQUIRED", "AUDIOCPP_VOICE_OPTIONAL", "AUDIOCPP_VOICE_NONE", "AudioCppFamilyProfile", "AUDIOCPP_DEFAULT_FAMILY_PROFILE", "AUDIOCPP_FAMILY_PROFILES", "audiocpp_entry_voice_capability", "audiocpp_entry_supports_design", "audiocpp_voice_for_run", + "audiocpp_entry_supports_instructions", "audiocpp_family_narrates", "audiocpp_family_spec_tasks", "audiocpp_family_voice_policy", "audiocpp_request_error", "audiocpp_script_input", diff --git a/app/converter/clients/audiocpp.py b/app/converter/clients/audiocpp.py index 98d64ba..9f52c40 100644 --- a/app/converter/clients/audiocpp.py +++ b/app/converter/clients/audiocpp.py @@ -58,6 +58,18 @@ AUDIOCPP_VOICE_SPEAKER = "speaker" # built-in speaker name (Qwen CustomVoice) AUDIOCPP_VOICE_CLONE = "clone" # server-side preset / voice_dir (Base, others) AUDIOCPP_VOICE_DESIGN = "design" # voice described by --instructions (vdes) +# How a family's speech endpoint consumes --instructions: the OpenAI-style +# top-level "instructions" field (the default, read by most design/style +# implementations), or the "instruction" request option inside the +# "options" object (BreezeTTS 2: --request-option instruction=... and the +# endpoint example send it there). audio.cpp's adapter translates the +# top-level "instructions" field into that same "instruction" request +# option for every family, so both channels reach the model. +AUDIOCPP_INSTRUCTION_FIELD = "field" +AUDIOCPP_INSTRUCTION_OPTION = "option" + +AUDIOCPP_FAMILY_BREEZE_TTS = "breeze_tts" + # HTTP error body fragments identifying deterministic request-configuration # problems: the identical request will fail on every retry, so the chunk # loop must give up immediately instead of burning its attempt budget. @@ -68,6 +80,10 @@ AUDIOCPP_NON_RETRYABLE_ERRORS = ( # Cloning without the reference transcript (Qwen3-TTS Base ICL mode): # the server-side voice has reference audio but no transcript for it. "requires reference text", + # A CustomVoice entry given a voice preset that resolves to reference + # audio rather than a built-in speaker id: the model takes built-in + # speakers only. + "custom voice prefill requires speaker", # The server cannot resolve a model contract for the family (its own # hint text about model_specs/--model-spec-override follows the fragment). "model contract spec not found for family", @@ -160,6 +176,15 @@ AUDIOCPP_HINTED_ERRORS = ( "task and cannot generate audiobooks. Consider deleting the model " "from the server configuration (re-run Configure Backends → " "audio.cpp and unselect it)."), + # A CustomVoice entry whose selected server voice resolved to + # reference audio rather than a built-in speaker id (the flat voice + # list cannot distinguish them, so the connect-time check only + # warns): the model cannot clone reference audio. + ("custom voice prefill requires speaker", + "The CustomVoice model serves built-in speakers only: pick a " + "built-in speaker named on the entry (e.g. Vivian, Ryan, Uncle Fu) " + "with --voice, or select the Base model entry to clone a reference " + "voice."), # Graph allocation failures: mostly device memory. The server's log # carries the exact size the failed allocation attempted. ("failed to allocate", _ALLOCATION_HINT_TEXT()), @@ -315,6 +340,11 @@ def audiocpp_family_voice_policy(family: str) -> str: timbre reference despite the spec not declaring a clone task (Vevo2's zero-shot TTS route). Unknown families (no local specs) keep the conservative clone-only default the client has always applied. + + _AUDIOCPP_KNOWN_FAMILY_POLICIES carries verified policies for + *specific* families, consulted before the local spec lookup: a family + whose local spec is absent (older audio.cpp checkout against a newer + server) or incomplete must not be misread as reference-required. """ if family == AUDIOCPP_FAMILY_QWEN3_TTS \ or family in AUDIOCPP_CLONE_ONLY_FAMILIES \ @@ -322,6 +352,8 @@ def audiocpp_family_voice_policy(family: str) -> str: # Qwen3-TTS is entry-typed (speaker/clone/design capability per # model id), so the family policy stays out of its way. return AUDIOCPP_VOICE_REQUIRED + if family in _AUDIOCPP_KNOWN_FAMILY_POLICIES: + return _AUDIOCPP_KNOWN_FAMILY_POLICIES[family] tasks = audiocpp_family_spec_tasks(family) if not tasks: return AUDIOCPP_VOICE_REQUIRED @@ -531,13 +563,19 @@ class AudioCppFamilyProfile: """ def __init__(self, language_style: str = AUDIOCPP_LANG_OMIT, - script_prefix: Optional[str] = None): + script_prefix: Optional[str] = None, + instruction_channel: str = AUDIOCPP_INSTRUCTION_FIELD): self.language_style = language_style # SCRIPT_PREFIX, when set, formats every request's text as one # "<prefix>: text" script line (audiocpp_script_input): the # family's server implementation parses the prompt as a # speaker-script and silently drops unprefixed lines (VibeVoice). self.script_prefix = script_prefix + # INSTRUCTION_CHANNEL picks where --instructions go: the OpenAI + # "instructions" field (AUDIOCPP_INSTRUCTION_FIELD) or the + # "instruction" request option (AUDIOCPP_INSTRUCTION_OPTION, + # BreezeTTS 2 — see its profile below). + self.instruction_channel = instruction_channel # Generic profile for families not listed in AUDIOCPP_FAMILY_PROFILES: @@ -563,9 +601,75 @@ AUDIOCPP_FAMILY_PROFILES = { # client flattens each request into one Speaker-1 line (the server # renormalizes the lowest speaker id to zero — the cloned reference). "vibevoice": AudioCppFamilyProfile(script_prefix="Speaker 1"), + # BreezeTTS 2 (text design/clone/direction) reads the instruction as a + # request option, not the OpenAI "instructions" field: upstream's own + # endpoint example sends {"options": {"instruction": "..."}}. The + # model detects the language (zh/en) itself, so no language field. + "breeze_tts": AudioCppFamilyProfile( + instruction_channel=AUDIOCPP_INSTRUCTION_OPTION), +} + +# Voice-policy overrides for families whose evidence contradicts their +# local model spec (or whose spec predates audio.cpp's task declarations). +# Kept as data so the spec lookup can stay the single generic path. +_AUDIOCPP_KNOWN_FAMILY_POLICIES = { + # BreezeTTS 2 (upstream spec / implementation: tts, clone ("voice + # direction"), design). Older local checkouts carry no breeze_tts + # spec at all — without this fallback a remote Breeze entry reads as + # clone-only and an instructions-only run (voice direction without a + # reference) is refused at connect time. + AUDIOCPP_FAMILY_BREEZE_TTS: AUDIOCPP_VOICE_OPTIONAL, + # VibeVoice's spec declares only "tts", but its implementation + # explicitly accepts reference audio (the session uses a speaker + # reference when the request carries one): mixed tts+clone. The + # pick-falls-back rules then stop silently dropping the selected + # voice for it. + "vibevoice": AUDIOCPP_VOICE_OPTIONAL, } +def audiocpp_entry_supports_instructions(family: str, task: str, + model_id: str) -> Optional[bool]: + """Whether an entry's model consumes a style instruction, if known. + + Three-valued on purpose: True (the model's implementation reads the + instruction), False (proven not to), None (unknown — a spec the local + checkout does not describe or one without instruction markers). + Distinguishing unknown from unsupported keeps the TUI behind honest + wording instead of implying every family follows instructions. + + Verified cases: + - task "vdes" entries: the voice comes from the instruction. + - BreezeTTS 2: the instruction conditions synthesis with or without a + reference (voice design / voice direction). + - Qwen3-TTS: the CustomVoice and VoiceDesign implementations read + "instruction"; the Base (cloning) implementation does not — a + variant-specific split, which is why Qwen is resolved per entry. + - Other families: True only when their local model spec declares an + "instruction"/"instruct" request option; None otherwise. + """ + if task == AUDIOCPP_TASK_VDES: + return True + if family == AUDIOCPP_FAMILY_BREEZE_TTS: + return True + if family == AUDIOCPP_FAMILY_QWEN3_TTS: + lowered = (model_id or "").lower() + if "customvoice" in lowered: + return True + if "base" in lowered: + return False + return None + spec = _family_spec(family) + if spec is None: + return None + names = {str(option.get("name") or "") + for option in (spec.get("request_options") or []) + if isinstance(option, dict)} + if names & {"instruction", "instruct"}: + return True + return None + + def audiocpp_script_input(prefix: str, text: str) -> str: """TEXT formatted as one "<PREFIX>: text" script line. @@ -738,6 +842,15 @@ class AudioCppTTSClient(BaseTTSClient): # Free-form per-request options (--option KEY=VALUE) forwarded in the # request's "options" object; models ignore keys they don't know. self.request_options: Dict[str, str] = dict(request_options or {}) + # The instruction can also arrive as --option instruction=... The + # server folds both sources into one request option (the + # top-level field overwrites the option), so two *different* + # instructions would silently drop one — refuse instead, before a + # server is even contacted. + self._check_instruction_conflict() + # A guidance value recommended for instructed requests on specific + # families (resolved in _connect; None = no automatic guidance). + self._auto_guidance_scale: Optional[float] = None # Set during _connect: design_mode for "vdes" entries, instruction_voice # when a family without built-in speakers gets its voice from the # instruction alone (no voice field), and plain_mode for plain-TTS @@ -758,6 +871,51 @@ class AudioCppTTSClient(BaseTTSClient): # Connection # ------------------------------------------------------------------ + def _resolve_auto_guidance(self) -> None: + """Select the request-strengthening guidance for instructed runs. + + Breeze-TTS 2's model card recommends guidance scale 4 to + strengthen instruction-following (its own SDK examples carry + --cfg-scale 4 for voice direction and design); audio.cpp's + default is 1.0, which leaves the clone's reference delivery + dominant. When this run carries an instruction (the field or the + request option) and the user did not set their own guidance + value, the request gets the recommended one. + """ + if self.family == AUDIOCPP_FAMILY_BREEZE_TTS \ + and self.instructions \ + and "guidance_scale" not in (getattr( + self, "request_options", {}) or {}): + self._auto_guidance_scale = 4.0 + self._report("[INFO] Sending guidance_scale 4 with the " + "instruction (strengthens Breeze instruction " + "following; set --option guidance_scale=... to " + "override)") + + def _check_instruction_conflict(self) -> None: + """Refuse two different instructions given at once. + + The server normalizes both the top-level "instructions" field and + the "instruction" request option into the same request option, + with the field winning — so "Instructions: calm narration" plus + "--option instruction=screaming" would silently drop the field. + Identical values are allowed (same effective instruction). + """ + option_instruction = (self.request_options.get("instruction") + or "").strip() + if self.instructions and option_instruction \ + and self.instructions != option_instruction: + raise RuntimeError( + "Two conflicting instructions were given: the Instructions " + f"text ({self.instructions!r}) and the request option " + f"instruction={option_instruction!r}. Keep one source only: " + "an --option instruction=... overrides --instructions " + "silently, so use whichever you intend.") + if not self.instructions and option_instruction: + # Make the forwarded option visible where --instructions + # would be reported, for the run log's completeness. + self.instructions = option_instruction + def _connected(self, mode: str) -> None: """Report the resolved connection (MODE: speaker/voice/... label).""" self._report(f"[OK] Connected to audio.cpp server at {self.api_url} " @@ -813,6 +971,17 @@ class AudioCppTTSClient(BaseTTSClient): f"--voice cannot be used with the voice design model " f"'{self.model_id}': the voice is described by the " "--instructions text instead (see README).") + if audiocpp_entry_voice_capability( + self.family, self.task, self.model_id) \ + == AUDIOCPP_VOICE_SPEAKER \ + and not is_builtin_speaker(self.voice): + self._report( + "[WARNING] The selected CustomVoice entry serves " + f"built-in speakers, and '{self.voice}' is not one " + "of them: unless this server preset maps to a " + "built-in speaker id, synthesis will fail with " + "'custom voice prefill requires speaker'. To clone " + "reference audio, select the Base model entry.") self._check_voice() self._connected(f"voice '{self.voice}'") else: @@ -872,6 +1041,21 @@ class AudioCppTTSClient(BaseTTSClient): self._report(f"[INFO] Sending instruction with every request: {self.instructions}") self._report("[INFO] Its effect (style, emotion, delivery) depends on the " "model family; models without instruction support ignore it.") + # Breeze-TTS 2's model card recommends guidance scale 4 to + # strengthen instruction-following (its SDK default --cfg-scale + # 4 for voice direction); audio.cpp's default is 1.0, which + # leaves the clone's reference delivery dominant. When the run + # carries an instruction and the user did not set their own + # guidance value, send the recommended one. + self._resolve_auto_guidance() + logger.info( + "audio.cpp run settings: model=%s family=%s task=%s voice=%r " + "instruction=%r request_options=%s auto_guidance_scale=%s seed=%r", + self.model_id, self.family or "<unresolved>", self.task, + self.voice, self.instructions or None, + getattr(self, "request_options", {}) or {}, + getattr(self, "_auto_guidance_scale", None), + None if self._seed < 0 else self._seed) unload = (config.AUDIOCPP_UNLOAD_MODELS if self._unload_models_override is None else self._unload_models_override) @@ -1190,15 +1374,34 @@ class AudioCppTTSClient(BaseTTSClient): if self._seed >= 0: # audio.cpp has no negative "randomize" seed; a negative seed # means "let the server randomize", so the field is omitted. - payload["seed"] = self._seed - if self.instructions: - # Explicit voice-design or style instruction (required for task - # "vdes" entries; a Ctrl/style control on families that read it). - payload["instructions"] = self.instructions + # Above 2^53 a JSON number loses precision, so the full-range + # seeds audio.cpp documents (send uint64 as a decimal string) + # are sent as strings. + if self._seed > 2 ** 53: + payload["seed"] = str(self._seed) + else: + payload["seed"] = self._seed if self.request_options: # Generic per-model controls (--option KEY=VALUE): forwarded # verbatim; the model ignores keys it does not know. payload["options"] = dict(self.request_options) + if getattr(self, "_auto_guidance_scale", None) is not None: + # Instructed Breeze request without an explicit guidance + # value: the model card's recommended instruction strength. + payload["guidance_scale"] = self._auto_guidance_scale + if self.instructions: + # Explicit voice-design or style instruction (required for task + # "vdes" entries; a voice/style control on families that read + # it). Families on the option channel take it as a request + # option; an explicit --option instruction=... from the caller + # lands here too and is never overwritten. + if self.profile.instruction_channel == AUDIOCPP_INSTRUCTION_OPTION: + if not (self.request_options or {}).get("instruction", ""): + options = dict(self.request_options) + options["instruction"] = self.instructions + payload["options"] = options + else: + payload["instructions"] = self.instructions request = urllib.request.Request( url, data=json.dumps(payload).encode("utf-8"), headers={"Content-Type": "application/json"}, method="POST") diff --git a/app/converter/clients/qwen.py b/app/converter/clients/qwen.py index dd1c8ba..3f4025c 100644 --- a/app/converter/clients/qwen.py +++ b/app/converter/clients/qwen.py @@ -268,7 +268,16 @@ class QwenTTSClient(BaseTTSClient): # ------------------------------------------------------------------ def _generate_custom_voice(self, text: str) -> Tuple: - """Generate audio using CustomVoice mode with the run's speaker.""" + """Generate audio using CustomVoice mode with the run's speaker. + + When the endpoint exposes an instruction parameter and the run + carries one, it is sent as a delivery/style control (the demo's + run_instruct accepts an ``instruct`` argument alongside the + speaker; the model voices the text with that delivery instead of + the speaker's default). With no instruction the request is + unchanged. + """ + instructions = getattr(self, "instructions", "") or "" custom_api = self._resolve_api_name("/run_instruct", "/run_custom_voice", "/generate_custom_voice") if custom_api == "/run_instruct": payload = dict( @@ -276,6 +285,9 @@ class QwenTTSClient(BaseTTSClient): lang_disp=self.language, spk_disp=speaker_display_name_for(self.speaker), ) + if instructions \ + and self._endpoint_accepts_param(custom_api, "instruct"): + payload["instruct"] = instructions else: payload = dict( text=text, @@ -290,6 +302,10 @@ class QwenTTSClient(BaseTTSClient): if self._endpoint_accepts_param(custom_api, "seed"): payload["seed"] = self._seed + if instructions \ + and self._endpoint_accepts_param(custom_api, "instruct"): + payload["instruct"] = instructions + return self.client.predict(**payload, api_name=custom_api) def _generate_voice_design(self, text: str) -> Tuple: diff --git a/app/converter/clients/sglomni.py b/app/converter/clients/sglomni.py index 14c9990..13b98d7 100644 --- a/app/converter/clients/sglomni.py +++ b/app/converter/clients/sglomni.py @@ -20,7 +20,8 @@ catalog``): Clone-capable models without a reference synthesize their built-in default voice ("default") unless the catalog marks a reference as -mandatory (Qwen3-TTS Base, dots.tts, ZONOS2 — those refuse at connect). +mandatory (Qwen3-TTS Base, MOSS-TTS, dots.tts, ZONOS2 — those refuse at +connect). """ import base64 @@ -59,12 +60,6 @@ _MIME_BY_SUFFIX = { ".webm": "audio/webm", ".mp4": "audio/mp4", } -# Error-envelope types the server returns for deterministic request -# problems (bad voice, missing reference, unknown model): the identical -# request fails on every retry, so the chunk loop gives up immediately. -_NON_RETRYABLE_TYPES = ("BadRequestError", "InvalidRequestError", - "NotFoundError", "PermissionDeniedError") - # The scheduler's KV-window admission error ("Request requires more tokens # than the thinker KV cache can hold (input_tokens=684, max_new_tokens= # 12288, required_tokens=12972, kv_capacity=4095)..."): the server names @@ -130,8 +125,6 @@ class SgOmniTTSClient(BaseTTSClient): # The catalog entry this run targets (the backend package validates # the key; only its repo id and capability are client business). from backends.sglomni.catalog import entry_by_key - from backends.sglomni.constants import DEFAULT_PORT, SERVER_NAME - from backends.common import port_of self.entry = entry_by_key((model or "").strip()) if self.entry is None: raise RuntimeError( @@ -139,7 +132,6 @@ class SgOmniTTSClient(BaseTTSClient): "(see Configure Backends → SGLang-Omni or the backend docs).") self.api_url = ((api_url or config.SGLOMNI_API_URL).strip() .rstrip("/")) - self.port = port_of(self.api_url, DEFAULT_PORT) self.voice = (voice or "").strip() or None self.ref_audio = (ref_audio or "").strip() or None self.ref_text = (ref_text or "").strip() @@ -152,13 +144,18 @@ class SgOmniTTSClient(BaseTTSClient): # rejection (None = none learned): later requests keep their # max_new_tokens under it. See _kv_admission_fit. self._kv_fit = None + # The ref_audio request value, computed on first use (see + # _ref_audio_value); None = not computed yet. + self._ref_audio_cached = None # Seed sent with every request: config.SEED as-is, or (with # CONSTANT_SEED and SEED < 0) one random value drawn per run and # reused for every chunk so the voice stays consistent across # chunk boundaries. Only sent to models that accept a # request-scoped seed (Voxtral rejects it outright), and only # when a concrete seed is in play (a negative one means "re-sample - # every generation", so there is nothing to send). + # every generation", so there is nothing to send). NOTE(unverified + # upstream): whether the other pipelines accept a seed too — see + # the catalog's supports_seed note. seed = resolve_request_seed() if self.entry.supports_seed else None self._seed = seed if (seed is not None and seed >= 0) else None if language is None: @@ -178,6 +175,11 @@ class SgOmniTTSClient(BaseTTSClient): raise RuntimeError( f"{entry.label} designs the voice from an instruction: " 'pass --instructions "..." describing the voice.') + if self.instructions and not entry.supports_instructions: + raise RuntimeError( + f"{entry.label} does not consume style instructions: the " + "server would silently ignore them. Remove --instructions, " + "or pick a model that supports them (Qwen3-TTS, MOSS-TTS).") if entry.capability == "clone" and entry.requires_reference \ and not self.ref_audio: raise RuntimeError( @@ -188,15 +190,27 @@ class SgOmniTTSClient(BaseTTSClient): self._report(f"[WARNING] --clone is ignored with {entry.label}: " "it voices text with its built-in presets.") self.ref_audio = None + elif entry.capability == "design" and self.ref_audio: + self._report(f"[WARNING] --clone is ignored with {entry.label}: " + "it designs the voice from instructions.") + self.ref_audio = None elif self.ref_audio and not Path(self.ref_audio).is_file(): raise RuntimeError( f"Reference audio not found: {self.ref_audio}") - if entry.speakers and self.voice \ - and self.voice not in entry.speakers: + presets = self._preset_voices() + if presets and self.voice and self.voice not in presets: self._report( f"[WARNING] Voice {self.voice!r} is not one of " - f"{entry.label}'s presets ({', '.join(entry.speakers)}); " + f"{entry.label}'s presets ({', '.join(presets)}); " "the server will reject it if it does not know the name.") + + def _preset_voices(self) -> List[str]: + """The preset voice names ENTRY can speak with — the same list the + hub's voice menu offers (catalog-declared speakers, or the + checkpoint's own voice_embedding presets, e.g. Voxtral's).""" + from backends.sglomni.models import preset_voices + return preset_voices(self.entry) + def _connect(self) -> None: """Verify the server is up, healthy, and hosting the expected model. @@ -209,6 +223,24 @@ class SgOmniTTSClient(BaseTTSClient): entry, url = self.entry, self.api_url try: payload = self._fetch_json("/health", timeout=10) + except urllib.error.HTTPError as exc: + # A booting server answers 503 with an "unhealthy" body — + # urlopen turns that into an HTTPError before the healthy + # check below can see it. Tell the user to wait for the + # server that is already starting, not to start another. + detail = _http_error_detail(exc) + if exc.code == 503: + raise RuntimeError( + f"The SGLang-Omni server at {url} is not healthy yet " + f"(HTTP 503: {detail[:300] or 'no body'}). Wait for it " + "to finish booting and retry.") from exc + raise RuntimeError( + f"The SGLang-Omni server at {url} answered HTTP {exc.code} " + f"on /health ({detail[:300] or 'no body'}). Is this an " + "sgl-omni server? Start the sgl-omni server first (the " + "CLI and the hub start the managed instance automatically " + "when the backend is installed), or point --api-url at a " + "running server.") from exc except Exception as exc: raise RuntimeError( f"SGLang-Omni server not reachable at {url}: {exc}. Start " @@ -253,7 +285,10 @@ class SgOmniTTSClient(BaseTTSClient): local Whisper transcription.""" if self.entry.capability != "clone" or not self.ref_audio: return - if not self.ref_text and not self.skip_transcription: + if not self.ref_text and self.skip_transcription: + self._report("[INFO] Skipping reference audio transcription " + "(--no-transcription).") + elif not self.ref_text: self._report("[INFO] Transcribing reference audio for voice " "cloning...") from .transcribe import transcribe_reference_audio @@ -291,23 +326,40 @@ class SgOmniTTSClient(BaseTTSClient): def _ref_audio_value(self) -> str: """The ref_audio request value: a local path on a loopback server - (the server reads the file directly), else a base64 data URL.""" - path = Path(self.ref_audio) - if not path.is_file(): - raise RuntimeError( - f"Reference audio not found: {self.ref_audio}") - if _is_loopback(self.api_url): - return str(path.resolve()) - return _data_url(path) + (the server reads the file directly), else a base64 data URL. + + Computed once per run and cached: the clip is validated at connect + and cannot change mid-run, and re-encoding its bytes for every + sub-request would ship the same payload over and over.""" + cached = self._ref_audio_cached + if cached is None: + path = Path(self.ref_audio) + if not path.is_file(): + raise RuntimeError( + f"Reference audio not found: {self.ref_audio}") + if _is_loopback(self.api_url): + cached = str(path.resolve()) + else: + cached = _data_url(path) + self._ref_audio_cached = cached + return cached def _request_payload(self, text: str) -> dict: """The /v1/audio/speech JSON body for one sub-chunk.""" entry = self.entry payload = { "model": entry.repo, + # NOTE(unverified upstream): "voice" is sent even when nothing + # was picked (the "default" sentinel) and to design runs, + # which have no voice — audio.cpp omits the field there. + # Verify the server tolerates it for every pipeline. "voice": self.voice or DEFAULT_VOICE, "input": text, "response_format": RESPONSE_FORMAT, + # NOTE(unverified upstream): Qwen-style display names + # ("English", "Auto") go to every model; audio.cpp maps per + # family. Verify each pipeline accepts them (or wants ISO + # codes / the field omitted). "language": self.language, } if self._seed is not None: @@ -330,6 +382,16 @@ class SgOmniTTSClient(BaseTTSClient): payload["ref_audio"] = self._ref_audio_value() if self.ref_text: payload["ref_text"] = self.ref_text + if entry.supports_instructions and self.instructions: + # Reference + separate style instruction (Qwen3-TTS Base + # instruction conditioning, MOSS v1.5 user message). NOT + # the design task_type: the reference stays the voice. + payload["instructions"] = self.instructions + elif entry.supports_instructions and self.instructions: + # Speaker models (Qwen3-TTS CustomVoice) also shape the + # delivery with an instruction; unsupported models never get + # one (checked at connect). + payload["instructions"] = self.instructions return payload def _request_wav(self, text: str) -> bytes: @@ -352,9 +414,6 @@ class SgOmniTTSClient(BaseTTSClient): exc = retry_exc detail = _http_error_detail(exc) raise self._request_error(exc.code, detail) from exc - except urllib.error.URLError as exc: - raise RuntimeError( - f"SGLang-Omni request failed: {exc.reason}") from exc def _post_speech(self, payload: dict) -> bytes: """POST PAYLOAD to /v1/audio/speech; HTTPErrors propagate raw.""" @@ -373,8 +432,12 @@ class SgOmniTTSClient(BaseTTSClient): except urllib.error.URLError as exc: raise RuntimeError( f"SGLang-Omni request failed: {exc.reason}") from exc - if not wav: - raise RuntimeError("SGLang-Omni server returned empty audio") + if len(wav) < 12 or wav[:4] != b"RIFF" or wav[8:12] != b"WAVE": + # A JSON error body handed back with HTTP 200 would otherwise + # be written as chunk bytes and fail later, confusingly, in + # the concat step. + raise RuntimeError( + "SGLang-Omni server returned audio that is not a WAV file") return wav def _kv_admission_fit(self, detail: str) -> Optional[int]: @@ -417,17 +480,17 @@ class SgOmniTTSClient(BaseTTSClient): the server's message; anything else stays retryable. """ message = detail[:500] or f"HTTP {status}" - kind = None try: envelope = json.loads(detail) error = envelope.get("error") if isinstance(error, dict): message = str(error.get("message") or message) - kind = error.get("type") except ValueError: pass - if 400 <= status < 500 and (kind is None - or kind in _NON_RETRYABLE_TYPES): + # Every 4xx envelope is deterministic — the identical request + # fails identically on every attempt (this is a single-user local + # server: it queues work rather than answering 429-style limits). + if 400 <= status < 500: return NonRetryableTTSError( f"SGLang-Omni rejected the request (HTTP {status}): " f"{message}") @@ -455,31 +518,36 @@ class SgOmniTTSClient(BaseTTSClient): if not sub_chunks: raise RuntimeError("No text to synthesize") - with self._chunk_heartbeat(chunk_num): - wav_parts: List[bytes] = [ - self._request_wav(sub_text) for sub_text in sub_chunks] - output_path = self._chunk_path(chunk_num, ".wav") - if len(wav_parts) == 1: - output_path.write_bytes(wav_parts[0]) - else: - # Several sub-request WAVs: concatenate through the shared - # ffmpeg path (each part is a complete file with headers). - with tempfile.TemporaryDirectory( - prefix="sglomni_parts_") as parts_dir: - part_paths: List[Path] = [] - for index, wav in enumerate(wav_parts, 1): + with tempfile.TemporaryDirectory( + prefix="sglomni_parts_") as parts_dir: + # One part per sub-request, spooled to disk as it arrives + # (like the other clients) instead of buffering every + # response in memory until the chunk is complete. + part_paths: List[Path] = [] + with self._chunk_heartbeat(chunk_num): + for index, sub_text in enumerate(sub_chunks, 1): part = Path(parts_dir) / f"part_{index:02d}.wav" - part.write_bytes(wav) + part.write_bytes(self._request_wav(sub_text)) part_paths.append(part) + if len(part_paths) == 1: + output_path.write_bytes(part_paths[0].read_bytes()) + else: + # Several sub-request WAVs: concatenate through the + # shared ffmpeg path (each part is a complete file + # with headers). concat_audio_files(part_paths, output_path) logger.debug("Chunk %d generated (%d sub-request(s))", - chunk_num, len(wav_parts)) + chunk_num, len(part_paths)) return str(output_path) except ConversionCancelled: raise + except NonRetryableTTSError: + # Propagate past the generic handler so the retry loop skips + # its remaining attempts for deterministic server errors. + raise except Exception as exc: logger.error("SGLang-Omni chunk processing failed for chunk " "%d: %s", chunk_num, exc) |