diff options
Diffstat (limited to 'app/converter/clients/audiocpp.py')
| -rw-r--r-- | app/converter/clients/audiocpp.py | 560 |
1 files changed, 523 insertions, 37 deletions
diff --git a/app/converter/clients/audiocpp.py b/app/converter/clients/audiocpp.py index 224d24a..3637a50 100644 --- a/app/converter/clients/audiocpp.py +++ b/app/converter/clients/audiocpp.py @@ -1,14 +1,20 @@ """Client for the audio.cpp audiocpp_server (native ggml TTS families).""" +import array +import base64 import json import logging import shutil +import struct +import subprocess +import sys import tempfile import urllib.error import urllib.parse import urllib.request +import wave from pathlib import Path -from typing import Any, Dict, List, Optional, Set +from typing import Any, Dict, List, Optional, Set, Tuple from .. import config from ..audio import concat_audio_files @@ -113,6 +119,25 @@ AUDIOCPP_CLONE_ONLY_ERRORS = ( "only supports offline voice cloning", # Echo-TTS ) +# Managed-server log where ggml records its allocation failures with the +# exact attempted size and device (see allocation_log_note). +_AUDIOCPP_SERVER_LOG_NAME = "audiocpp" + + +def _ALLOCATION_HINT_TEXT() -> str: + return ( + "The server ran out of device memory while building a compute " + "graph: check what else is using the GPU, and read the server's " + "log (app/logs/audiocpp-server.log), which records the exact " + "allocation size it attempted. Cloning families that encode the " + "whole reference with attention over its length (MOSS-TTS-Local) " + "retry automatically with a shorter reference when the voice's " + "wav is readable locally. For DramaBox, adding \"session_options\": " + "{\"dramabox.mem_saver\": \"true\"} to its server.json model entry " + "trades speed for a much lower memory peak (restart the server " + "after editing).") + + # Deterministic failures whose one-line server message is not actionable # on its own: FRAGMENT -> guidance appended to the "not retryable" error. # Matched like AUDIOCPP_NON_RETRYABLE_ERRORS (case-insensitive, against the @@ -130,8 +155,11 @@ AUDIOCPP_HINTED_ERRORS = ( # accepts, so every request cloning it fails the same way. ("sample capacity exceeded", "The voice's reference audio is longer than this model's encoder " - "accepts: trim the voice's reference wav in the voices folder and " - "re-run Configure Backends → audio.cpp so the server picks it up."), + "accepts: trim the voice's reference wav in the voices folder, or " + "re-run Configure Backends → audio.cpp — the setup writes a larger " + "AudioVAE encoder-sample capacity for VoxCPM entries when a voice " + "wav is longer than the built-in ~15 s ceiling, so no trim is " + "needed (restart the server to pick the new config up)."), # An s2s-only family (e.g. PersonaPlex) hosted for generation: no # hosting of the entry makes it narrate text. ("supports only speech-to-speech", @@ -139,6 +167,37 @@ AUDIOCPP_HINTED_ERRORS = ( "task and cannot generate audiobooks. Consider deleting the model " "from the server configuration (re-run Configure Backends → " "audio.cpp and unselect it)."), + # Graph allocation failures: mostly device memory. The server's log + # carries the exact size the failed allocation attempted. + ("failed to allocate", _ALLOCATION_HINT_TEXT()), + ("allocation failed", _ALLOCATION_HINT_TEXT()), + # A companion package (e.g. MioTTS's MioCodec) is missing where the + # server looks for it. + ("model path does not exist", + "This model needs a companion package that is not installed where " + "the server looks for it (MioTTS loads MioCodec through its " + "miotts.codec_model_path session option). Re-run Configure Backends " + "→ audio.cpp: the setup downloads companion packages alongside the " + "model and writes the needed session options, then restart the " + "server."), + # The package on disk does not match its spec layout (the pre-repair + # GLM-TTS / OuteTTS installs nested their GGUF under a repo + # subdirectory; MiniMax-H3 hosts several GGUFs in one directory). + ("missing model package file", + "The model package on disk does not match its spec layout. Re-run " + "Configure Backends → audio.cpp: the model manager now repairs " + "broken package layouts when downloading, and models installed " + "with a stale layout are re-downloaded."), + ("missing model root", + "The model package on disk does not match its spec layout. Re-run " + "Configure Backends → audio.cpp: the model manager now repairs " + "broken package layouts when downloading, and models installed " + "with a stale layout are re-downloaded."), + ("model directory contains", + "The model directory holds several GGUFs where audio.cpp expects " + "one: hosting the entry from a specific GGUF file (as the current " + "setup does) makes the model loadable. Re-run Configure Backends → " + "audio.cpp to rewrite server.json with the fixed hosting."), ) # Families whose audio.cpp implementation only synthesizes by cloning a @@ -150,12 +209,37 @@ AUDIOCPP_HINTED_ERRORS = ( AUDIOCPP_CLONE_ONLY_FAMILIES = frozenset( {"chatterbox", "confucius4_tts", "echo_tts"}) +# Families whose plain-TTS route still requires a reference voice even +# though their model spec does not declare a "clone" task: Vevo2's +# zero-shot TTS route refuses every request without a timbre reference +# ("requires target_voice or voice speaker audio"), so the voice policy +# treats them like clone-only (the "All" flow then sends the picked +# voice, and a voice-less run is refused with the actionable message +# instead of failing every request server-side). +AUDIOCPP_VOICE_REQUIRED_FAMILIES = frozenset({"vevo2"}) + # How a family's voice is supplied — resolved per family from the local # audio.cpp checkout's model_specs (see audiocpp_family_voice_policy): AUDIOCPP_VOICE_REQUIRED = "required" # clone-only: a reference voice is mandatory AUDIOCPP_VOICE_OPTIONAL = "optional" # tts + clone: blank voice means plain TTS AUDIOCPP_VOICE_NONE = "none" # pure TTS: no cloning, no voice at all +# The allocation-failure fragments that trigger the server-log detail and +# the trimmed-reference retry (see AUDIOCPP_NON_RETRYABLE_ERRORS). +AUDIOCPP_ALLOCATION_FRAGMENTS = ("failed to allocate", "allocation failed") + +# A cloned reference long enough to blow up reference-attention encoders +# (MOSS-TTS-Local's codec encoder: memory grows with the reference's +# square) is retried as this many seconds of the same voice, read from +# the voice's local wav and sent as a base64 voice_ref (bounded by the +# server's 5 MiB inline-reference limit). +AUDIOCPP_REFERENCE_TRIM_SECONDS = 30.0 +_AUDIOCPP_VOICE_REF_MAX_BYTES = 5 * 1024 * 1024 + +# Warn about a nearly-full local GPU before the first request: with +# another process holding the memory, even small graph allocations fail. +_LOW_FREE_DEVICE_MIB = 4096 + # Spec cache (family -> parsed spec dict, or None for unknown). The form # consults the policy and capability tags on every menu render, so each # family's spec is read at most once per process. @@ -204,6 +288,27 @@ def audiocpp_family_spec_tasks(family: str) -> Optional[Set[str]]: return {str(task) for task in spec["tasks"]} +def spec_request_option_names(family: str) -> Set[str]: + """FAMILY's accepted request-option names from its local model spec. + + Used to decide whether an option may be attached to a request (e.g. + the reference_text carried alongside an inline voice_ref): families + whose runtime validates request options against the spec would reject + an unknown key outright. An unknown family (no local spec) yields an + empty set — the caller then omits the option rather than risking a + rejection. + """ + spec = _family_spec(family) + if not spec: + return set() + options = spec.get("options") + request = options.get("request") if isinstance(options, dict) else None + if not isinstance(request, list): + return set() + return {str(item["name"]) for item in request + if isinstance(item, dict) and item.get("name")} + + def audiocpp_entry_supports_design(family: str, task: str, model_id: str) -> bool: """Whether a server model entry can design a voice from a description. @@ -241,12 +346,15 @@ def audiocpp_family_voice_policy(family: str) -> str: voice at all; mixed families (tts + clone) may run without one (plain TTS) or clone a reference; clone-only families — the explicit AUDIOCPP_CLONE_ONLY_FAMILIES set, which also repairs specs that - wrongly claim "tts" — always need a reference voice. Unknown families - (no local specs) keep the conservative clone-only default the client - has always applied. + wrongly claim "tts" — always need a reference voice, as do the + AUDIOCPP_VOICE_REQUIRED_FAMILIES whose plain-TTS route demands a + timbre reference despite the spec not declaring a clone task (Vevo2's + zero-shot TTS route). Unknown families (no local specs) keep the + conservative clone-only default the client has always applied. """ if family == AUDIOCPP_FAMILY_QWEN3_TTS \ - or family in AUDIOCPP_CLONE_ONLY_FAMILIES: + or family in AUDIOCPP_CLONE_ONLY_FAMILIES \ + or family in AUDIOCPP_VOICE_REQUIRED_FAMILIES: # Qwen3-TTS is entry-typed (speaker/clone/design capability per # model id), so the family policy stays out of its way. return AUDIOCPP_VOICE_REQUIRED @@ -324,7 +432,8 @@ def _reference_text_error(voice: Optional[str], server_message: str) -> str: def audiocpp_request_error(status: int, detail: str, - voice: Optional[str] = None) -> Exception: + voice: Optional[str] = None, + log_note: Optional[str] = None) -> Exception: """The exception for a failed audio.cpp speech request. Deterministic request-configuration errors (a fragment in @@ -333,30 +442,206 @@ def audiocpp_request_error(status: int, detail: str, hosting errors (AUDIOCPP_CLONE_ONLY_ERRORS) also carry the re-host hint, hinted errors (AUDIOCPP_HINTED_ERRORS) their per-fragment guidance; everything else returns the plain RuntimeError the retry - loop has always retried. + loop has always retried. LOG_NOTE, when given for an allocation + failure, appends the server log's own record of the failed + allocation (the exact size it attempted, from the managed server's + log file) to the non-retryable message. """ message = _server_error_message(detail) lowered = message.lower() + allocation_failure = any( + fragment in lowered for fragment in AUDIOCPP_ALLOCATION_FRAGMENTS) + error: Exception if _REFERENCE_TEXT_FRAGMENT in lowered: - return NonRetryableTTSError( + error = NonRetryableTTSError( _reference_text_error(voice, message)) - if any(fragment in lowered for fragment in AUDIOCPP_CLONE_ONLY_ERRORS): - return NonRetryableTTSError( + elif any(fragment in lowered for fragment in AUDIOCPP_CLONE_ONLY_ERRORS): + error = NonRetryableTTSError( f"audio.cpp server returned HTTP {status} (not retryable): " f"{message}. This model family only synthesizes by cloning a " "reference voice, so its server entry must be hosted with task " '"clon" — re-run Configure Backends → audio.cpp (or edit ' "server.json) and restart the server.") - for fragment, hint in AUDIOCPP_HINTED_ERRORS: - if fragment in lowered: - return NonRetryableTTSError( + else: + hinted = next( + (hint for fragment, hint in AUDIOCPP_HINTED_ERRORS + if fragment in lowered), None) + if hinted is not None: + error = NonRetryableTTSError( f"audio.cpp server returned HTTP {status} (not retryable): " - f"{message}. {hint}") - if any(fragment in lowered for fragment in AUDIOCPP_NON_RETRYABLE_ERRORS): - return NonRetryableTTSError( - f"audio.cpp server returned HTTP {status} (not retryable): " - f"{message}") - return RuntimeError(f"audio.cpp server returned HTTP {status}: {detail}") + f"{message}. {hinted}") + elif any(fragment in lowered + for fragment in AUDIOCPP_NON_RETRYABLE_ERRORS): + error = NonRetryableTTSError( + f"audio.cpp server returned HTTP {status} (not retryable): " + f"{message}") + else: + error = RuntimeError( + f"audio.cpp server returned HTTP {status}: {detail}") + if allocation_failure and log_note \ + and isinstance(error, NonRetryableTTSError): + error = NonRetryableTTSError(f"{error}{log_note}") + return error + + +def allocation_log_note(message: str) -> str: + """The managed server's own record of a failed allocation, best-effort. + + ggml logs every failed backend-buffer allocation with the exact size + it attempted and the device ("allocating N MiB on device D: cudaMalloc + failed"), while the server's HTTP 500 only carries the model's one-line + message — so the log is where the number lives. Only meaningful for + the locally managed server (app/logs/audiocpp-server.log); remote + servers, or a moved/rotated log, yield "" and the message stays as-is. + """ + lowered = message.lower() + if not any(fragment in lowered + for fragment in AUDIOCPP_ALLOCATION_FRAGMENTS): + return "" + try: + # Imported lazily: backends.audiocpp imports this package, so a + # module-level import would cycle. + from backends import servers as _servers + path = _servers.server_log_path(_AUDIOCPP_SERVER_LOG_NAME) + except Exception: # noqa: BLE001 - diagnostics only, never fatal + return "" + try: + text = path.read_text(encoding="utf-8", errors="replace") + except OSError: + return "" + patterns = ("cudaMalloc failed", "not enough space in the buffer", + "failed to allocate") + matches = [line.strip() for line in text.splitlines() + if any(pattern in line for pattern in patterns)] + if not matches: + return "" + return (f" The server's log ({path}) records the failed allocation as: " + f"{matches[-1]}") + + +def nvidia_device_memory_report() -> Optional[str]: + """Local NVIDIA GPUs' total/free memory as one CSV block, or None. + + Runs ``nvidia-smi --query-gpu=index,memory.total,memory.free`` once; + None when the tool is missing (non-NVIDIA machines), times out, or + reports an error. Used by the client to warn about a nearly-full GPU + before the first request — with another process holding the memory, + even small graph allocations fail. + """ + try: + result = subprocess.run( + ["nvidia-smi", "--query-gpu=index,memory.total,memory.free", + "--format=csv,noheader,nounits"], + capture_output=True, text=True, timeout=15) + except (OSError, subprocess.SubprocessError): + return None + if result.returncode != 0 or not result.stdout.strip(): + return None + return result.stdout.strip() + + +def _pcm16_mono_samples(path: Path, max_seconds: float + ) -> Optional[Tuple[int, List[int]]]: + """The first MAX_SECONDS of a wav as (sample rate, mono PCM16 samples). + + Reads with the stdlib wave module (PCM u8/s16/s24/s32), mixes channels + by averaging, and stops at MAX_SECONDS so a long reference costs only + the frames actually sent. Returns None for files the wave module + cannot parse (float-format wavs, non-WAV files) or that carry no + samples — the trimmed-reference retry then does not fire. + """ + max_frames = 0 + if max_seconds > 0: + try: + with wave.open(str(path), "rb") as probe: + rate = probe.getframerate() + max_frames = int(max_seconds * rate) + 1 + except (OSError, EOFError, wave.Error): + return None + try: + with wave.open(str(path), "rb") as handle: + rate = handle.getframerate() + channels = handle.getnchannels() + width = handle.getsampwidth() + frames = handle.getnframes() + raw = handle.readframes(min(frames, max_frames) if max_frames + else frames) + except (OSError, EOFError, wave.Error): + return None + if rate <= 0 or channels <= 0 or width not in (1, 2, 3, 4) or not raw: + return None + frame_bytes = width * channels + frame_count = len(raw) // frame_bytes + if frame_count <= 0: + return None + if width == 2: + values = array.array("h") + values.frombytes(raw[:frame_count * frame_bytes]) + if sys.byteorder == "big": + values.byteswap() + elif width == 1: + # u8 -> s16, scaled to the full 16-bit range. + values = array.array("h", ((byte - 128) << 8 + for byte in raw[:frame_count])) + else: + # s24/s32 -> s16 by dropping the low bits (keeps every value inside + # int16 so the mixdown and the PCM16 container need no clipping). + values = array.array( + "i", (int.from_bytes(raw[i * width:i * width + width], + "little", signed=True) + for i in range(frame_count))) + shift = 8 if width == 3 else 16 + values = array.array("h", (value >> shift for value in values)) + if channels == 1: + return rate, values.tolist() + mixed: List[int] = [] + for index in range(frame_count): + start = index * channels + mixed.append(sum(values[start:start + channels]) // channels) + return rate, mixed + + +def pcm16_wav_bytes(sample_rate: int, samples: List[int]) -> bytes: + """Wrap mono PCM16 samples in a minimal RIFF/WAVE container.""" + data = array.array("h", samples) + if sys.byteorder == "big": + data.byteswap() + payload = data.tobytes() + return (b"RIFF" + + struct.pack("<I", 36 + len(payload)) + + b"WAVEfmt " + struct.pack("<IHHIIHH", 16, 1, 1, + sample_rate, sample_rate * 2, 2, 16) + + b"data" + struct.pack("<I", len(payload)) + + payload) + + +def build_trimmed_voice_reference( + path: Optional[Path], max_seconds: float = AUDIOCPP_REFERENCE_TRIM_SECONDS, + max_bytes: int = _AUDIOCPP_VOICE_REF_MAX_BYTES + ) -> Optional[Tuple[str, float, str]]: + """A base64 voice_ref of the first MAX_SECONDS of a reference wav. + + Returns (base64 wav, seconds used, file name), or None when PATH is + missing or unreadable. The payload keeps the file's native sample rate + and is mixed to mono; it is further capped so the decoded bytes stay + within the server's 5 MiB inline-reference limit (16-bit mono means + ~2.6 MB per 30 s at 44.1 kHz, comfortably inside). + """ + if path is None: + return None + decoded = _pcm16_mono_samples(path, max_seconds) + if decoded is None: + return None + rate, samples = decoded + if not samples: + return None + byte_cap_seconds = max_bytes / (2 * rate) + frames = int(min(max_seconds, byte_cap_seconds) * rate) + samples = samples[:frames] + if not samples: + return None + wav = pcm16_wav_bytes(rate, samples) + return base64.b64encode(wav).decode("ascii"), len(samples) / rate, path.name class AudioCppFamilyProfile: @@ -443,6 +728,12 @@ def audiocpp_entry_voice_capability(family: str, task: str, class AudioCppTTSClient(BaseTTSClient): + # Class-level defaults so a partially-constructed instance behaves like + # a fresh run (tests build clients via __new__; see BaseTTSClient). + _voice_ref_b64: Optional[str] = None + _voice_ref_reference_text: Optional[str] = None + _reference_trim_attempted = False + """Generates audio chunks through an audio.cpp audiocpp_server. Talks to the OpenAI-style HTTP API of audiocpp_server, which hosts TTS @@ -559,6 +850,15 @@ class AudioCppTTSClient(BaseTTSClient): self.design_mode = False self.instruction_voice = False self.plain_mode = False + # Trimmed-reference retry state (see _switch_to_trimmed_reference): + # an inline base64 voice_ref that replaces the voice name after an + # allocation failure, the transcript carried alongside it, and the + # one-attempt guard. The class-level defaults above keep partially + # constructed instances (tests via __new__) behaving like a fresh + # run; the assignments here shadow them for this instance. + self._voice_ref_b64 = None + self._voice_ref_reference_text = None + self._reference_trim_attempted = False # Family and task of the selected model entry and the family's request # profile; all are resolved from GET /v1/models during _connect. self.family = "" @@ -653,29 +953,33 @@ class AudioCppTTSClient(BaseTTSClient): f"'{self.family}') serves built-in speakers: pass " "--voice NAME with one of them (e.g. Vivian, Ryan, " "Uncle Fu) to synthesize with it (see README).") + elif audiocpp_family_voice_policy(self.family) \ + == AUDIOCPP_VOICE_REQUIRED: + # Checked before the instruction-voice branch: a family + # whose synthesis needs a reference voice (clone-only, + # Vevo2's zero-shot route) cannot take its voice from an + # instruction, so refuse with the fix instead of failing + # every request server-side. + raise RuntimeError( + f"The audio.cpp model '{self.model_id}' (family " + f"'{self.family}') has no built-in speakers, so its voice " + "must come from the server: rerun with --voice NAME " + "matching a voice_preset or voice_dir entry in the server " + "config, or select the CustomVoice entry for built-in " + "speakers (see README).") elif self.instructions: # Families without built-in speakers can still get their voice # from the instruction alone (e.g. OmniVoice voice design). self.instruction_voice = True self._connected("instruction voice") self._report(f"[INFO] Designing the voice from: {self.instructions}") - elif audiocpp_family_voice_policy(self.family) in ( - AUDIOCPP_VOICE_OPTIONAL, AUDIOCPP_VOICE_NONE): + else: # The family synthesizes without a reference voice — a # pure-TTS family (spec tasks without "clone") or a mixed # tts+clone family used without one. Plain TTS: no voice # field is sent at all. self.plain_mode = True self._connected("plain TTS") - else: - raise RuntimeError( - f"The audio.cpp model '{self.model_id}' (family " - f"'{self.family}') has no built-in speakers, so its voice " - "must come from the server: rerun with --voice NAME " - "matching a voice_preset or voice_dir entry in the server " - "config, or describe a voice with --instructions for " - "families that support it, or select the CustomVoice entry " - "for built-in speakers (see README).") if self.instructions and not self.design_mode and not self.instruction_voice: self._report(f"[INFO] Sending instruction with every request: {self.instructions}") self._report("[INFO] Its effect (style, emotion, delivery) depends on the " @@ -685,6 +989,7 @@ class AudioCppTTSClient(BaseTTSClient): else self._unload_models_override) if unload: self._unload_server_models() + self._warn_low_device_memory() def _require_synthesis_task(self, models: List[Dict[str, str]]) -> None: """Reject model entries that cannot synthesize narration from text. @@ -927,6 +1232,168 @@ class AudioCppTTSClient(BaseTTSClient): ) # ------------------------------------------------------------------ + # Reference trimming and device diagnostics + # ------------------------------------------------------------------ + + def _warn_low_device_memory(self) -> None: + """Warn once when a local GPU is nearly full before the first request. + + Graph-allocation failures read as "out of memory" even when the + card has gigabytes free for the model itself — what matters is the + free memory at request time, which other processes can hold. A + one-time nvidia-smi query (local hosts only; skipped silently + elsewhere or when the tool is missing) turns that case into an + explicit warning instead of a mysterious 500. + """ + host = (urllib.parse.urlparse(self.api_url).hostname or "").lower() + if host not in ("127.0.0.1", "localhost", "::1"): + return + report = nvidia_device_memory_report() + if not report: + return + for row in report.splitlines(): + parts = [part.strip() for part in row.split(",")] + if len(parts) < 3: + continue + try: + index = int(parts[0]) + total = int(parts[1]) + free = int(parts[2]) + except ValueError: + continue + if free < _LOW_FREE_DEVICE_MIB: + self._report( + f"[WARNING] GPU {index} has {free} MiB free of " + f"{total} MiB — allocation failures under this " + "condition usually mean another process is using " + "the GPU.") + + def _voice_wav_path(self) -> Optional[Path]: + """The selected voice's reference wav on this machine, when readable. + + Resolved from the local checkout's server.json exactly like the + server resolves the request's voice name: the entry's + ``voice_presets[name].voice_ref`` first, then ``voice_dir/<name>.wav``. + Only preset-mode runs with a locally readable file return a path — + remote-only servers (or voice names that only exist server-side) + yield None and the trimmed-reference retry does not fire. + """ + if not self.voice: + return None + name = self.voice + if not name or name in (".", "..") or "/" in name or "\\" in name: + return None + try: + # Imported lazily: backends.audiocpp imports this package, so a + # module-level import would cycle. + from backends.audiocpp.build import find_local_checkout + checkout = find_local_checkout() + except Exception: # noqa: BLE001 - best effort + return None + if checkout is None: + return None + server_json = checkout / "server.json" + try: + data = json.loads(server_json.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + if not isinstance(data, dict): + return None + for entry in data.get("models") or []: + if not isinstance(entry, dict) or entry.get("id") != self.model_id: + continue + presets = entry.get("voice_presets") + if isinstance(presets, dict): + preset = presets.get(name) + if isinstance(preset, dict): + ref = preset.get("voice_ref") + if isinstance(ref, str) and ref: + path = Path(ref) + if not path.is_absolute(): + path = server_json.parent / ref + if path.is_file(): + return path + voice_dir = data.get("voice_dir") + if isinstance(voice_dir, str) and voice_dir: + candidate = Path(voice_dir) / f"{name}.wav" + if candidate.is_file(): + return candidate + return None + + def _voice_transcript(self) -> Optional[str]: + """The voice library transcript for the selected voice, or None. + + Read from the voice directory's prompt_text mapping (the same file + the server consults when it resolves a voice NAME); only carried + alongside an inline voice_ref, where the server's own injection is + bypassed. + """ + if not self.voice: + return None + try: + # Imported lazily: backends.audiocpp imports this package, so a + # module-level import would cycle. + from backends.common import PROMPT_TEXT_FILENAME, read_prompt_text + from backends.audiocpp.build import find_local_checkout + checkout = find_local_checkout() + except Exception: # noqa: BLE001 - best effort + return None + if checkout is None: + return None + try: + data = json.loads((checkout / "server.json") + .read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + voice_dir = data.get("voice_dir") if isinstance(data, dict) else None + if not isinstance(voice_dir, str) or not voice_dir: + return None + try: + return (read_prompt_text(Path(voice_dir) / PROMPT_TEXT_FILENAME) + .get(self.voice) or None) + except OSError: + return None + + def _switch_to_trimmed_reference(self, server_message: str) -> bool: + """Switch the cloning reference to a trimmed local wav, once. + + Some families encode the whole reference with attention over its + length (MOSS-TTS-Local's codec encoder: the required memory grows + with the reference's square), so a long voice reference fails the + graph allocation regardless of how much VRAM the device has. When + the selected voice resolves to a locally readable wav, replace the + voice name with an inline base64 voice_ref cut to + AUDIOCPP_REFERENCE_TRIM_SECONDS and retry the request once; the + trimmed reference then applies to the rest of the run. Only fires + on allocation-failure messages for preset-mode runs; everything + else keeps the original behavior. + """ + lowered = server_message.lower() + if self._reference_trim_attempted \ + or not any(fragment in lowered + for fragment in AUDIOCPP_ALLOCATION_FRAGMENTS) \ + or self.design_mode or self.instruction_voice \ + or self.plain_mode or not self.voice: + return False + self._reference_trim_attempted = True + trimmed = build_trimmed_voice_reference(self._voice_wav_path()) + if trimmed is None: + return False + b64, seconds, source = trimmed + self._voice_ref_b64 = b64 + transcript = self._voice_transcript() + if transcript and "reference_text" in spec_request_option_names( + self.family): + self._voice_ref_reference_text = transcript + self._report( + f"[INFO] {source}'s family encodes the whole reference with " + f"attention over its length; retrying with the first " + f"{seconds:.0f}s of voice '{self.voice}' as the cloning " + "reference (the trimmed reference applies to the rest of this " + "run).") + return True + + # ------------------------------------------------------------------ # HTTP requests # ------------------------------------------------------------------ @@ -946,8 +1413,13 @@ class AudioCppTTSClient(BaseTTSClient): # Design models take no voice field (the voice comes from the # instruction); instruction-voice runs on families without built-in # speakers omit it too, since no speaker or preset was requested; - # plain-TTS runs (no reference voice needed) omit it likewise. - if not self.design_mode and not self.instruction_voice \ + # plain-TTS runs (no reference voice needed) omit it likewise. A + # trimmed-reference retry replaces the voice name with an inline + # base64 voice_ref (see _switch_to_trimmed_reference). + if self._voice_ref_b64 is not None: + payload["voice_ref"] = {"type": "base64", + "data": self._voice_ref_b64} + elif not self.design_mode and not self.instruction_voice \ and not self.plain_mode: payload["voice"] = self.voice if self.profile.language_style == AUDIOCPP_LANG_DISPLAY: @@ -968,10 +1440,17 @@ class AudioCppTTSClient(BaseTTSClient): # Explicit voice-design or style instruction (required for task # "vdes" entries; a Ctrl/style control on families that read it). payload["instructions"] = self.instructions - if self.request_options: + options = dict(self.request_options) + if self._voice_ref_reference_text \ + and "reference_text" not in options: + # The server only injects the voice library's transcript when it + # resolves the voice NAME; an inline voice_ref bypasses that, so + # carry the transcript explicitly for families that accept it. + options["reference_text"] = self._voice_ref_reference_text + if options: # Generic per-model controls (--option KEY=VALUE): forwarded # verbatim; the model ignores keys it does not know. - payload["options"] = dict(self.request_options) + payload["options"] = options request = urllib.request.Request( url, data=json.dumps(payload).encode("utf-8"), headers={"Content-Type": "application/json"}, method="POST") @@ -985,8 +1464,15 @@ class AudioCppTTSClient(BaseTTSClient): detail = exc.read().decode("utf-8", errors="replace")[:200] except Exception: pass + message = _server_error_message(detail) + if self._switch_to_trimmed_reference(message): + # One retry with the trimmed reference; if that fails too the + # error below carries the server log's allocation detail. + return self._request_wav(text) raise audiocpp_request_error(exc.code, detail, - voice=self.voice) from exc + voice=self.voice, + log_note=allocation_log_note( + message)) from exc except urllib.error.URLError as exc: raise RuntimeError(f"audio.cpp request failed: {exc.reason}") from exc if len(wav) < 12 or wav[:4] != b"RIFF" or wav[8:12] != b"WAVE": |