From dc6e7cd43029da62dabe2513fb5aa8a34df1bd6d Mon Sep 17 00:00:00 2001 From: historia Date: Tue, 1 Sep 2026 12:12:35 -0400 Subject: fix: spec santizer for glm, outetts, miotts, minimax. --- app/backends/audiocpp/__init__.py | 4 + app/backends/audiocpp/catalog.py | 314 +++++++++++++++++++- app/backends/audiocpp/models.py | 297 ++++++++++++++----- app/backends/audiocpp/wizard.py | 62 ++-- app/backends/servers.py | 12 + app/converter/clients/__init__.py | 7 + app/converter/clients/audiocpp.py | 560 +++++++++++++++++++++++++++++++++--- app/tests/test_backends_audiocpp.py | 455 ++++++++++++++++++++++++++++- app/tests/test_tts.py | 379 +++++++++++++++++++++++- 9 files changed, 1951 insertions(+), 139 deletions(-) diff --git a/app/backends/audiocpp/__init__.py b/app/backends/audiocpp/__init__.py index 64122c9..3344eb5 100644 --- a/app/backends/audiocpp/__init__.py +++ b/app/backends/audiocpp/__init__.py @@ -30,11 +30,13 @@ from .constants import ( ) from .catalog import ( detect_backend, + entry_model_path, is_clone_only_family, is_design_package, hosting_task, load_model_catalog, package_dir_options, + sanitize_model_spec, build_model_entry, build_server_config, load_server_config, @@ -42,6 +44,7 @@ from .catalog import ( rehost_clone_only_entries, request_options_families, supports_request_options, + apply_entry_session_options, ) from .models import ( delete_model_files, @@ -100,6 +103,7 @@ __all__ = [ "build_model_entry", "build_server_config", "load_server_config", "server_config_selections", "rehost_clone_only_entries", "request_options_families", "supports_request_options", + "entry_model_path", "sanitize_model_spec", "apply_entry_session_options", # models "missing_model_entries", "installed_model_entries", "unused_installed_entries", "delete_model_files", "install_models", diff --git a/app/backends/audiocpp/catalog.py b/app/backends/audiocpp/catalog.py index 1743d15..1048185 100644 --- a/app/backends/audiocpp/catalog.py +++ b/app/backends/audiocpp/catalog.py @@ -3,9 +3,11 @@ import json import re import sys +import wave from pathlib import Path from typing import Dict, List, Optional, Set, Tuple +from backends.common import find_wav_files from converter.clients import AUDIOCPP_CLONE_ONLY_FAMILIES, audiocpp_family_spec_tasks from .constants import TASK_CLON, TASK_TTS, TASK_VDES @@ -167,14 +169,117 @@ def _default_package(packages: List[dict]) -> Optional[dict]: return packages[0] +def spec_gguf_rooted(spec: dict) -> bool: + """True when SPEC's gguf source resolves its weights from ``$gguf``. + + The ``$gguf`` root is audio.cpp's single-GGUF convention: the model + directory must hold exactly one top-level GGUF (or ``model.gguf``), + which then carries the weights. Packages that feed such a source must + therefore install their GGUF at the top of the target directory, not + nested under a repository subdirectory. + """ + for source in spec.get("sources") or []: + if not isinstance(source, dict) or source.get("format") != "gguf": + continue + roots = source.get("roots") + if isinstance(roots, dict) and any( + value == "$gguf" for value in roots.values() + if isinstance(value, str)): + return True + return False + + +def package_common_prefix(files: list) -> Optional[str]: + """The one directory prefix every file path in FILES shares, or None. + + Only exact single-component matches count: every file must contain a + ``/`` and start with the same first component. A flat package (no ``/`` + at all) or a mixed one (some files under ``config/``, some at the root, + like minimax_music3's intentional multi-file layout) yields None. + """ + prefixes: List[str] = [] + for item in files: + if not isinstance(item, str) or "/" not in item: + return None + prefix = item.split("/", 1)[0] + if not prefix: + return None + prefixes.append(prefix) + if not prefixes: + return None + first = prefixes[0] + if first in (".", ".."): + return None + return first if all(prefix == first for prefix in prefixes) else None + + +def sanitize_model_spec(spec: dict) -> bool: + """Repair broken package ``strip_prefix`` entries in SPEC, in place. + + Two upstream spec bug classes have shipped in the audio.cpp checkout, + and both make the model manager install files where the server cannot + find them: + + - A dot ``strip_prefix`` ("." or "./") is meant for files written + ``./``; when the package instead lists repo-root files bare + (``model.gguf``), the manager rejects the whole package ("file path + does not start with strip_prefix '.': ...") and nothing can be + downloaded. Root-level files need no prefix at all (upstream specs + like minimax_music3.json store ""), so dropping the dot prefix is + the safe repair. Prefixes naming a real directory are left alone — + the correct remote paths cannot be guessed. + + - A package with NO strip_prefix whose files all nest under one + repository directory (``Text to audio (TTS)/GLM-TTS_Q8.gguf`` — + glm_tts and outetts shipped like this) installs the GGUF under that + subdirectory. The server then finds no top-level GGUF and falls back + to the safetensors source, failing on a companion file the GGUF + package never ships ("missing model package file 'tokenizer_merges'"). + When the spec's gguf source resolves weights from ``$gguf`` (the + single-GGUF convention) and every file shares one directory prefix, + that prefix is what the strip_prefix should have been, so set it. + Packages whose gguf source names tensors by explicit file paths + (minimax_music3's nested multi-GGUF layout) are left untouched. + + Returns True when SPEC changed. + """ + changed = False + for package in spec.get("packages") or []: + if not isinstance(package, dict): + continue + prefix = str(package.get("strip_prefix") or "").rstrip("/") + if prefix in (".", ".."): + files = package.get("files") + if isinstance(files, list) and files \ + and all(isinstance(item, str) + and item.startswith(prefix + "/") + for item in files): + continue + package["strip_prefix"] = "" + changed = True + continue + if prefix or package.get("format") != "gguf" \ + or not spec_gguf_rooted(spec): + continue + common = package_common_prefix(package.get("files") or []) + if common is None: + continue + package["strip_prefix"] = common + changed = True + return changed + + def load_model_catalog(audiocpp_dir: Path) -> List[dict]: """Read model_specs/*.json and return the TTS-capable families. Each returned entry has: family, display_name, description, languages, - clone_capable, packages (the full list from the spec), install_id - (recommended package id), and default_path (``models/``). - All families are treated equally and listed in alphabetical order by - display name. + clone_capable, packages (the full list from the spec, with broken + ``strip_prefix`` entries repaired in memory via sanitize_model_spec — + the checkout's files are never written back; the download path + materializes the same repair through its sanitized specs copy), + install_id (recommended package id), and default_path + (``models/``). All families are treated equally and + listed in alphabetical order by display name. """ specs_dir = audiocpp_dir / "model_specs" if not specs_dir.is_dir(): @@ -187,6 +292,7 @@ def load_model_catalog(audiocpp_dir: Path) -> List[dict]: spec = json.loads(spec_path.read_text(encoding="utf-8")) except (OSError, ValueError): continue + sanitize_model_spec(spec) tasks = spec.get("tasks") or [] if tasks: # A task list that names no text-synthesis capability means the @@ -342,21 +448,72 @@ def package_dir_options(entry: dict) -> List[dict]: return options +def entry_model_path(entry: dict, target_directory: Optional[str] = None) -> str: + """The server.json model path that hosts ENTRY's package for TARGET_DIRECTORY. + + ``models/`` normally (the recommended package's + directory when TARGET_DIRECTORY is None). audio.cpp only loads a model + directory that holds exactly one top-level GGUF, so a package that + ships several GGUFs into one directory (MiniMax-H3's text-encoder / + DiT / audio-VAE / video-VAE bundle) is hosted from its first GGUF + file instead: the explicit file path makes the server pick the gguf + source, and the directory stays the model root that the spec's named + tensors resolve from. + """ + directory = target_directory + if directory is None: + default_path = str(entry.get("default_path") or "") + directory = default_path[len("models/"):] \ + if default_path.startswith("models/") else default_path + if not directory: + directory = str(entry.get("family") or "") + package = next( + (item for item in entry.get("packages") or [] + if isinstance(item, dict) + and str(item.get("target_directory") or entry.get("family")) == directory + and item.get("format") == "gguf"), + None) + if package is None: + return f"models/{directory}" + prefix = str(package.get("strip_prefix") or "").rstrip("/") + ggufs: List[str] = [] + for item in package.get("files") or []: + if not isinstance(item, str) or not item.lower().endswith(".gguf"): + continue + local = item + if prefix and local.startswith(prefix + "/"): + local = local[len(prefix) + 1:] + elif prefix: + continue + ggufs.append(local) + if len(ggufs) > 1: + return f"models/{directory}/{ggufs[0]}" + return f"models/{directory}" + + def build_model_entry(family: str, model_id: str, model_path: str, - task: str = TASK_TTS) -> dict: + task: str = TASK_TTS, + session_options: Optional[Dict[str, str]] = None) -> dict: """Assemble one server.json model entry. ``task`` defaults to "tts"; voice design packages are hosted with "vdes" so the server runs its design session for speech requests (audiobook.py then requires --instructions with that entry). + SESSION_OPTIONS carries per-entry session-level options the server + applies at session creation (e.g. MioTTS's codec model path, or a + VoxCPM AudioVAE encoder capacity sized for long voice references); + an empty mapping is omitted so the entry keeps its minimal shape. """ - return { + entry = { "id": model_id, "family": family, "path": model_path, "task": task, "mode": "offline", } + if session_options: + entry["session_options"] = dict(session_options) + return entry def build_server_config(host: str, port: int, backend: str, lazy_load: bool, @@ -423,7 +580,14 @@ def server_config_selections(server_config: dict, path = entry.get("path") if not isinstance(path, str): continue - target = path[len("models/"):] if path.startswith("models/") else path + rel = path[len("models/"):] if path.startswith("models/") else path + # Entries hosted from a specific model file (e.g. MiniMax-H3's + # multi-GGUF directory, "models//") belong to the + # directory's catalog option, not to a made-up nested one. Unprefixed + # and absolute paths keep their raw target as before (the wizard's + # valid-directory check filters those it cannot offer again). + target = rel.split("/", 1)[0] \ + if path.startswith("models/") and "/" in rel else rel if family not in selected_dirs: selected_dirs[family] = [] if target not in selected_dirs[family]: @@ -432,3 +596,139 @@ def server_config_selections(server_config: dict, return selected_dirs, tasks + + +# --------------------------------------------------------------------------- +# Session options the setup bakes into server.json entries +# --------------------------------------------------------------------------- + +# MioTTS loads its MioCodec companion through the ``miotts.codec_model_path`` +# session option: the server's built-in default looks for the codec as a +# sibling of the GGUF's materialized sidecar root (/tmp/audiocpp-gguf/...), +# which is never where the model manager installs it. Pointing the option at +# the installed codec package directory is the supported path, so the wizard +# writes it (and downloads the codec alongside the model — it is an +# audio_tools family, not a TTS one, so the model catalog never offers it). +MIOTTS_CODEC_INSTALL_ID = "miocodec_q8_0" +MIOTTS_CODEC_DIRECTORY = "MioCodec-25Hz-44.1kHz-v2-GGUF" +MIOTTS_CODEC_MODEL_PATH = f"models/{MIOTTS_CODEC_DIRECTORY}" +MIOTTS_CODEC_DISPLAY_NAME = "MioCodec 25Hz 44.1kHz v2 (required by MioTTS)" + +# VoxCPM-style AudioVAE encoders cap the reference audio they encode at a +# fixed sample count (240000 samples ≈ 15 s at the VAE's 16 kHz rate, per +# the audio.cpp runtime). A voice reference longer than that fails every +# cloning request ("sample capacity exceeded"). The session option below +# raises the cap; the wizard sizes it to the longest wav in the voice +# directory so the configured voices all work without hand-trimming. +VOXCPM_ENCODER_CAPACITY_DEFAULT_SAMPLES = 240_000 +VOXCPM_ENCODER_CAPACITY_SAMPLE_RATE = 16_000 +VOXCPM_ENCODER_CAPACITY_MAX_SAMPLES = 240_000 * 20 +# Fallback key when a family's spec does not name the option (older +# checkouts may lack voxcpm1.json; the spec is preferred when present). +VOXCPM_ENCODER_CAPACITY_FALLBACK_KEYS = { + "voxcpm2": "voxcpm2.audiovae_encoder_sample_capacity", + "voxcpm1": "voxcpm1.audiovae_encoder_sample_capacity", +} + + +def wav_seconds(path: Path) -> Optional[float]: + """A PCM WAV file's duration in seconds, or None when unreadable. + + The stdlib wave module handles the PCM variants voice references use; + float-format or non-WAV files (mp3 renamed, exotic headers) yield None + so callers skip them instead of guessing a duration. + """ + try: + with wave.open(str(path), "rb") as handle: + frames = handle.getnframes() + rate = handle.getframerate() + except (OSError, EOFError, wave.Error): + return None + if rate <= 0 or frames <= 0: + return None + return frames / rate + + +def voxcpm_encoder_capacity_option(family: str, audiocpp_dir: Path) -> Optional[str]: + """FAMILY's AudioVAE encoder-capacity session option key, or None. + + Discovered from the family's model spec (the option whose name carries + ``encoder_sample_capacity``), falling back to the known VoxCPM keys so + a spec-less family still gets the right option name rather than a + broken server.json entry. + """ + try: + spec = json.loads( + (audiocpp_dir / "model_specs" / f"{family}.json") + .read_text(encoding="utf-8")) + except (OSError, ValueError): + spec = None + if isinstance(spec, dict): + options = spec.get("options") + session = options.get("session") if isinstance(options, dict) else None + if isinstance(session, list): + for item in session: + name = item.get("name") if isinstance(item, dict) else None + if isinstance(name, str) and "encoder_sample_capacity" in name: + return name + return VOXCPM_ENCODER_CAPACITY_FALLBACK_KEYS.get(family) + + +def voxcpm_encoder_capacity_samples(max_seconds: Optional[float]) -> Optional[int]: + """The encoder-sample capacity that fits MAX_SECONDS, or None. + + Rounds up to a multiple of the model's default 240000-sample capacity + and clamps to a generous ceiling (8 minutes) so an absurdly long + reference cannot push VRAM through the roof; anything at or below the + default needs no override at all. + """ + if max_seconds is None or max_seconds <= 0: + return None + default = VOXCPM_ENCODER_CAPACITY_DEFAULT_SAMPLES + needed = max_seconds * VOXCPM_ENCODER_CAPACITY_SAMPLE_RATE + samples = ((int(needed) + default - 1) // default) * default + if samples <= default: + return None + return min(samples, VOXCPM_ENCODER_CAPACITY_MAX_SAMPLES) + + +def apply_entry_session_options(model_entries: List[dict], + wav_dir: Optional[Path], + audiocpp_dir: Path) -> List[str]: + """Add the per-entry session options heavy families need, in place. + + MioTTS entries get ``miotts.codec_model_path`` (see MIOTTS_CODEC_*: + the server's default codec path cannot be satisfied by a normal + install). VoxCPM-family entries get an AudioVAE encoder-sample + capacity sized to the longest wav in WAV_DIR, so voice references + longer than the model's built-in 15 s ceiling still clone (returned + as None when the voice directory is unknown, unreadable, or only + holds short wavs — the default then applies as before). Existing + session options are preserved; a hand-set codec path is never + overridden. Returns the entry ids that gained options (for the + wizard's summary line), in order. + """ + max_seconds: Optional[float] = None + if wav_dir is not None: + for path in find_wav_files(Path(wav_dir)): + seconds = wav_seconds(path) + if seconds is not None and (max_seconds is None + or seconds > max_seconds): + max_seconds = seconds + applied: List[str] = [] + for entry in model_entries: + family = str(entry.get("family") or "") + current = dict(entry.get("session_options") or {}) + options = dict(current) + if family == "miotts" \ + and "miotts.codec_model_path" not in options: + options["miotts.codec_model_path"] = MIOTTS_CODEC_MODEL_PATH + if family.startswith("voxcpm") and max_seconds is not None: + key = voxcpm_encoder_capacity_option(family, audiocpp_dir) + samples = voxcpm_encoder_capacity_samples(max_seconds) + if key and samples: + options[key] = str(samples) + if options != current: + entry["session_options"] = options + applied.append(str(entry.get("id") or family)) + return applied diff --git a/app/backends/audiocpp/models.py b/app/backends/audiocpp/models.py index d81dcfc..16c61e9 100644 --- a/app/backends/audiocpp/models.py +++ b/app/backends/audiocpp/models.py @@ -16,6 +16,11 @@ from . import catalog as _catalog # runner kills it and reports exit 124 (see run_console_subprocess). DOWNLOAD_STALL_TIMEOUT = 300 +# Spec sanitizing lives with the catalog (the wizard's catalog view applies +# the same in-memory repair; the download path materializes it through its +# sanitized specs copy). The alias keeps this module's historical name. +_sanitize_model_spec = _catalog.sanitize_model_spec + def _installed_display_names(audiocpp_dir: Path, model_entries: Optional[List[dict]], install_guidance: List[Tuple[str, str]] @@ -25,19 +30,19 @@ def _installed_display_names(audiocpp_dir: Path, MODEL_ENTRIES and INSTALL_GUIDANCE are built in lockstep by ``_build_entries`` (one guidance pair per entry), so the pairs resolve positionally: each entry's ``path`` is checked against the checkout - exactly like ``_all_models_present`` resolves it. Returns an empty set + file-precisely (see ``_entry_present``). Returns an empty set when ENTRIES is None or does not line up with the guidance (no filtering — every model counts as not installed). """ if model_entries is None or len(model_entries) != len(install_guidance): return set() + packages_by_dir = _catalog_packages_by_dir(audiocpp_dir) installed: Set[str] = set() for entry, (name, _install_id) in zip(model_entries, install_guidance): rel = entry.get("path") if not isinstance(rel, str) or not rel: continue - path = Path(rel) if Path(rel).is_absolute() else audiocpp_dir / rel - if _model_path_present(path): + if _entry_present(audiocpp_dir, rel, packages_by_dir): installed.add(name) return installed @@ -67,10 +72,40 @@ def _split_pending_and_installed( return pending, noted +def _merge_companions(audiocpp_dir: Path, + pending: List[Tuple[str, str]], + companions: Optional[List[Tuple[str, str]]] + ) -> List[Tuple[str, str]]: + """Fold COMPANIONS into PENDING, skipping ones already on disk. + + Each companion is a (display name, install id) pair for a package a + hosted model requires but the TTS catalog never offers (MioCodec for + MioTTS). Companions already pending (same install id) or already + installed (their package's files present, checked file-precisely) are + dropped; the installed ones are reported so the user knows the + requirement is satisfied. Companion ids with no matching spec package + are kept (the install command will report what is wrong). + """ + merged = list(pending) + seen = {install_id for _name, install_id in pending} + for name, install_id in companions or []: + if install_id in seen: + continue + package = _companion_package(audiocpp_dir, install_id) + if package is not None \ + and _package_files_present(audiocpp_dir, package): + print(f"[OK] {name} is already installed.") + continue + seen.add(install_id) + merged.append((name, install_id)) + return merged + + def _install_models(audiocpp_dir: Path, install_guidance: List[Tuple[str, str]], download: bool, emit=None, cancel=None, - model_entries: Optional[List[dict]] = None) -> int: + model_entries: Optional[List[dict]] = None, + companions: Optional[List[Tuple[str, str]]] = None) -> int: """Report and optionally run the model install commands. When MODEL_ENTRIES (built in lockstep with INSTALL_GUIDANCE by @@ -78,11 +113,16 @@ def _install_models(audiocpp_dir: Path, installed and never re-downloaded or printed as commands; when every selected model is present nothing runs at all. The remaining models get one ``python install `` command each (de-duped by - install id). When DOWNLOAD is True each command is run in the audio.cpp - checkout via ``subprocess`` so the models are downloaded automatically; - a failing install is reported as a warning and does not abort the - remaining downloads. When DOWNLOAD is False (or the model manager is - missing) the commands are only printed after a note that setup downloads + install id). COMPANIONS carries (display name, install id) pairs for + packages a hosted model needs but the TTS catalog does not offer + (MioCodec for MioTTS): they join the pending list, de-duped against + it and skipped when their package's files are already on disk — so a + configure run that only adds the companion still downloads it. When + DOWNLOAD is True each command is run in the audio.cpp checkout via + ``subprocess`` so the models are downloaded automatically; a failing + install is reported as a warning and does not abort the remaining + downloads. When DOWNLOAD is False (or the model manager is missing) + the commands are only printed after a note that setup downloads them automatically — copy-pasteable for a manual install. With EMIT given (the in-TUI task view) each download streams its output @@ -91,9 +131,10 @@ def _install_models(audiocpp_dir: Path, progress bar and cancel gracefully. CANCEL aborts a running download. When the checkout's model specs carry a broken ``strip_prefix`` (a dot - prefix over repo-root files — the manager rejects every file in such a - package) and the manager supports ``--specs-dir``, the installs run - against a sanitized temporary copy of the specs (see + prefix over repo-root files, or a single-GGUF package nested under a + repository directory with no prefix — the manager installs files the + server cannot load) and the manager supports ``--specs-dir``, the + installs run against a sanitized temporary copy of the specs (see ``_prepare_specs_dir``); the checkout itself is left untouched. Specs that cannot be repaired confidently are left as-is: those installs fail, are reported as warnings, and the remaining downloads continue. @@ -101,13 +142,14 @@ def _install_models(audiocpp_dir: Path, Returns 0 when every command succeeded (or nothing needed running), 130 when cancelled, 1 when any download failed. """ - if not install_guidance: + if not install_guidance and not companions: return 0 manager = audiocpp_dir / "tools" / "model_manager_v2.py" installed_names = _installed_display_names( audiocpp_dir, model_entries, install_guidance) pending, installed_noted = _split_pending_and_installed( install_guidance, installed_names) + pending = _merge_companions(audiocpp_dir, pending, companions) supports_progress = emit is not None and _manager_supports_progress(manager) @@ -295,18 +337,29 @@ def _prepare_specs_dir(audiocpp_dir: Path) -> Optional[Path]: return staging -def download_applicable(audiocpp_dir: Path, model_entries: List[dict]) -> bool: +def download_applicable(audiocpp_dir: Path, model_entries: List[dict], + companions: Optional[List[Tuple[str, str]]] = None + ) -> bool: """True when the wizard's "download models automatically?" row applies. The audio.cpp model manager must be present (otherwise the install commands can only be printed), and at least one selected model must be - missing from disk (see ``_all_models_present``), so an already-configured - checkout is not asked to re-download models it already has. + missing from disk (see ``_all_models_present``), or one COMPANION + package a hosted model needs (MioCodec for MioTTS) must be — so an + already-configured checkout is not asked to re-download models it + already has. """ manager = audiocpp_dir / "tools" / "model_manager_v2.py" if not manager.is_file(): return False - return not _all_models_present(audiocpp_dir, model_entries) + if not _all_models_present(audiocpp_dir, model_entries): + return True + for _name, install_id in companions or []: + package = _companion_package(audiocpp_dir, install_id) + if package is None \ + or not _package_files_present(audiocpp_dir, package): + return True + return False def _build_tree_families(catalog: List[dict]) -> List[dict]: @@ -357,33 +410,146 @@ def _model_path_present(path: Path) -> bool: return False +def _package_files_present(audiocpp_dir: Path, package: dict) -> bool: + """True when PACKAGE's files are on disk at their strip_prefix-stripped paths. + + Mirrors model_manager_v2.py's own ``package_is_installed`` check, which + is what decides a re-install. Used instead of the plain "directory is + non-empty" check so a stale install from a broken spec layout (e.g. the + GLM/OuteTTS GGUFs nested under ``Text to audio (TTS)/`` before the spec + sanitizer existed) counts as missing and gets re-downloaded correctly. + """ + files = package.get("files") or [] + if not files: + return False + prefix = str(package.get("strip_prefix") or "").rstrip("/") + base = audiocpp_dir / "models" / str(package.get("target_directory") or "") + for item in files: + if not isinstance(item, str): + return False + local = item + if prefix: + if not local.startswith(prefix + "/"): + return False + local = local[len(prefix) + 1:] + try: + if not (base / local).is_file(): + return False + except OSError: + return False + return True + + +def _entry_directory_key(rel: str) -> str: + """The ```` a server.json model path belongs to. + + ``models/`` entries map to ````; entries hosted from a file + inside a package directory (``models//``, the multi-GGUF + hosting convention) map to ```` as well, so old directory-style + and new file-style entries of the same package compare equal. + """ + stripped = rel[len("models/"):] if rel.startswith("models/") else rel + return stripped.split("/", 1)[0] + + +def _catalog_packages_by_dir(audiocpp_dir: Path) -> Dict[str, List[dict]]: + """Map each catalog package target directory to its packages. + + The catalog's packages carry the sanitized strip prefixes (see + ``load_model_catalog``), so presence checks see the same layout the + model manager will install. Families whose specs cannot be read yield + no entries and callers fall back to the plain path check. + """ + try: + entries = _catalog.load_model_catalog(audiocpp_dir) + except (NotADirectoryError, OSError): + return {} + by_dir: Dict[str, List[dict]] = {} + for entry in entries: + for package in entry.get("packages") or []: + if not isinstance(package, dict): + continue + directory = str(package.get("target_directory") or entry["family"]) + by_dir.setdefault(directory, []).append(package) + return by_dir + + +def _companion_package(audiocpp_dir: Path, install_id: str) -> Optional[dict]: + """The spec package with INSTALL_ID across every model spec. + + Companion packages live in specs the TTS catalog filters out + (miocodec is an audio_tools family with no text-synthesis task), so + the lookup scans all model_specs/*.json with the sanitizer applied, + matching the sanitized layout the download will use. + """ + try: + spec_paths = sorted((audiocpp_dir / "model_specs").glob("*.json")) + except OSError: + return None + for spec_path in spec_paths: + try: + spec = json.loads(spec_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + continue + if not isinstance(spec, dict): + continue + _catalog.sanitize_model_spec(spec) + for package in spec.get("packages") or []: + if isinstance(package, dict) \ + and str(package.get("id") or "") == install_id: + return package + return None + + +def _entry_present(audiocpp_dir: Path, rel: str, + packages_by_dir: Dict[str, List[dict]]) -> bool: + """Whether the model entry path REL holds its package's files. + + When REL's directory matches a catalog package, the check is + file-precise (every package file at its stripped path). Otherwise the + plain path check applies: the entry may host a package the local + specs do not describe (an older checkout, a custom path), and "the + configured path exists" is then the best available signal. + """ + directory = _entry_directory_key(rel) + candidates = packages_by_dir.get(directory) + if candidates is not None: + return any(_package_files_present(audiocpp_dir, package) + for package in candidates) + path = Path(rel) if Path(rel).is_absolute() else audiocpp_dir / rel + return _model_path_present(path) + + def _all_models_present(audiocpp_dir: Path, model_entries: List[dict]) -> bool: - """True when every selected model entry's path already holds files on disk. + """True when every selected model entry's files are on disk. - Paths resolve against the checkout (where model_manager_v2.py installs - them), honoring absolute paths. Used by the wizard to skip the - "Automatically download the selected models" prompt when nothing is - actually missing. An empty selection is treated as not-present. + Presence is file-precise against the catalog packages (see + ``_entry_present``): a directory that merely exists — e.g. holding a + stale install from a since-repaired spec layout — counts as missing so + the next download replaces it with the correct layout. An empty + selection is treated as not-present. """ if not model_entries: return False + packages_by_dir = _catalog_packages_by_dir(audiocpp_dir) for entry in model_entries: rel = entry.get("path") if not isinstance(rel, str) or not rel: return False - path = Path(rel) if Path(rel).is_absolute() else audiocpp_dir / rel - if not _model_path_present(path): + if not _entry_present(audiocpp_dir, rel, packages_by_dir): return False return True -def missing_model_entries(server_json: Path) -> List[dict]: - """Return the server.json model entries whose files are not on disk. +def _server_entries_by_presence(server_json: Path, present: bool) -> List[dict]: + """The server.json model entries whose on-disk presence matches PRESENT. - Paths resolve exactly like audiocpp_server resolves them (relative paths - against the server.json's directory). Each returned entry carries the - entry ``id`` and ``rel`` (the configured path string); used by ``detect`` - to warn that a conversion would fail until the models are installed. + Presence is file-precise against the catalog packages when the + server.json's directory hosts the model_specs (the usual checkout + layout); otherwise the plain path check applies. Each returned entry + carries the entry ``id`` and ``rel`` (the configured path string), + resolved exactly like the server resolves them (relative against the + server.json's directory; absolute paths honored). """ try: data = json.loads(server_json.read_text(encoding="utf-8")) @@ -392,18 +558,37 @@ def missing_model_entries(server_json: Path) -> List[dict]: if not isinstance(data, dict): return [] base = server_json.parent - missing: List[dict] = [] + packages_by_dir = _catalog_packages_by_dir(base) + matching: List[dict] = [] for entry in data.get("models") or []: if not isinstance(entry, dict): continue rel = entry.get("path") if not isinstance(rel, str) or not rel: continue - path = Path(rel) if Path(rel).is_absolute() else base / rel - if _model_path_present(path): + if _entry_present(base, rel, packages_by_dir) != present: continue - missing.append({"id": str(entry.get("id") or rel), "rel": rel}) - return missing + matching.append({"id": str(entry.get("id") or rel), "rel": rel}) + return matching + + +def missing_model_entries(server_json: Path) -> List[dict]: + """Return the server.json model entries whose files are not on disk. + + Used by ``detect`` to warn that a conversion would fail until the + models are installed (see ``_server_entries_by_presence``). + """ + return _server_entries_by_presence(server_json, present=False) + + +def installed_model_entries(server_json: Path) -> List[dict]: + """Return the server.json model entries whose files ARE on disk. + + The complement of ``missing_model_entries`` (see + ``_server_entries_by_presence``). Used by the wizard's "Delete unused + models?" step to find already-downloaded models that were unselected. + """ + return _server_entries_by_presence(server_json, present=True) def _install_id_by_path(audiocpp_dir: Path) -> Dict[str, str]: @@ -422,35 +607,6 @@ def _install_id_by_path(audiocpp_dir: Path) -> Dict[str, str]: return by_path -def installed_model_entries(server_json: Path) -> List[dict]: - """Return the server.json model entries whose files ARE on disk. - - The complement of ``missing_model_entries``: each returned entry carries - the entry ``id`` and ``rel`` (the configured path string), resolved - exactly like ``missing_model_entries`` (relative against the server.json's - directory). Used by the wizard's "Delete unused models?" step to find - already-downloaded models that were unselected. - """ - try: - data = json.loads(server_json.read_text(encoding="utf-8")) - except (OSError, ValueError): - return [] - if not isinstance(data, dict): - return [] - base = server_json.parent - installed: List[dict] = [] - for entry in data.get("models") or []: - if not isinstance(entry, dict): - continue - rel = entry.get("path") - if not isinstance(rel, str) or not rel: - continue - path = Path(rel) if Path(rel).is_absolute() else base / rel - if _model_path_present(path): - installed.append({"id": str(entry.get("id") or rel), "rel": rel}) - return installed - - def missing_model_install_guidance(audiocpp_dir: Path, missing: List[dict]) -> List[Tuple[str, str]]: """Map MISSING model entries to (display name, install id) pairs. @@ -530,15 +686,20 @@ def hand_install_guidance(audiocpp_dir: Path, def unused_installed_entries(server_json: Path, new_paths: Set[str]) -> List[dict]: - """Return installed server.json entries whose path is not in NEW_PATHS. + """Return installed server.json entries whose package is not in NEW_PATHS. The already-downloaded models (see ``installed_model_entries``) that the new selection does not host any more — the candidates for the wizard's - "Delete unused models?" prompt. Entries whose files are not on disk are - never listed (there is nothing to delete). + "Delete unused models?" prompt. Entries are compared by their package + directory key (``_entry_directory_key``), so an entry re-hosted from + ``models/`` to ``models//`` (the multi-GGUF convention) + is not offered for deletion when the new config still hosts that + directory. Entries whose files are not on disk are never listed (there + is nothing to delete). """ + new_keys = {_entry_directory_key(path) for path in new_paths} return [entry for entry in installed_model_entries(server_json) - if entry["rel"] not in new_paths] + if _entry_directory_key(entry["rel"]) not in new_keys] def delete_model_files(server_json: Path, entries: List[dict]) -> int: diff --git a/app/backends/audiocpp/wizard.py b/app/backends/audiocpp/wizard.py index 97f50a5..a1a2a6a 100644 --- a/app/backends/audiocpp/wizard.py +++ b/app/backends/audiocpp/wizard.py @@ -26,10 +26,12 @@ from . import configsync as _configsync from . import models as _models from . import prebuilt as _prebuilt from . import voices as _voices -from .catalog import (_backend_options, build_model_entry, - build_server_config, detect_backend, hosting_task, +from .catalog import (_backend_options, build_model_entry, build_server_config, + detect_backend, entry_model_path, hosting_task, load_model_catalog, load_server_config, - package_dir_options, server_config_selections) + MIOTTS_CODEC_DISPLAY_NAME, MIOTTS_CODEC_INSTALL_ID, + package_dir_options, server_config_selections, + apply_entry_session_options) from .constants import (AUDIOCPP_DIR_NAME, AUDIOCPP_GIT_URL, BACKENDS, DEFAULT_HOST, TASK_TTS, TASK_VDES) @@ -80,7 +82,7 @@ def _build_entries(family_keys: List[str], chosen: Dict[str, List[dict]], task_picker: Callable[[str], str], known_tasks: Optional[Dict[Tuple[str, str], str]] = None ) -> Tuple[List[dict], List[str], List[Tuple[str, str]], - List[str], bool]: + List[Tuple[str, str]], List[str], bool]: """Build server.json model entries from the selected families/packages. TASK_PICKER is called for each design package to choose vdes/tts. @@ -89,12 +91,18 @@ def _build_entries(family_keys: List[str], chosen: Dict[str, List[dict]], was hosted instead of re-asking. Each entry's server id is its package ``target_directory`` (flattened to a token), so packages from the same family never collide; an id that does collide (across families) is - auto-suffixed without prompting. Returns (model_entries, entry_ids, - install_guidance, design_entry_ids, include_clone). + auto-suffixed without prompting. Entry paths come from the catalog + (``entry_model_path``): normally ``models/``, or the + package's first GGUF file when the package ships several GGUFs into one + directory (audio.cpp refuses multi-GGUF directories). COMPANION_GUIDANCE + carries the companion packages hosted models require but the TTS catalog + never offers (MioCodec for MioTTS). Returns (model_entries, entry_ids, + install_guidance, companion_guidance, design_entry_ids, include_clone). """ model_entries: List[dict] = [] entry_ids: List[str] = [] install_guidance: List[Tuple[str, str]] = [] + companion_guidance: List[Tuple[str, str]] = [] design_entry_ids: List[str] = [] include_clone = False for family in family_keys: @@ -120,12 +128,16 @@ def _build_entries(family_keys: List[str], chosen: Dict[str, List[dict]], model_id = f"{base_id}-{n}" entry_ids.append(model_id) model_entries.append(build_model_entry( - family, model_id, f"models/{opt['target_directory']}", + family, model_id, + entry_model_path(entry, opt["target_directory"]), task=task)) install_guidance.append((entry["display_name"], opt["install_id"])) if task == TASK_VDES: design_entry_ids.append(model_id) - return (model_entries, entry_ids, install_guidance, + if any(str(e.get("family")) == "miotts" for e in model_entries): + companion_guidance.append( + (MIOTTS_CODEC_DISPLAY_NAME, MIOTTS_CODEC_INSTALL_ID)) + return (model_entries, entry_ids, install_guidance, companion_guidance, design_entry_ids, include_clone) @@ -139,7 +151,11 @@ def _write_and_advise(audiocpp_dir: Path, wav_dir: Optional[Path], After a successful run the console output is the path of the written server.json. The model install commands (and optional automatic download) are handled separately by _install_models, called by both - UI modes once the user has decided whether to download. + UI modes once the user has decided whether to download. Before the + document is written, apply_entry_session_options bakes in the + per-entry session options heavy families need (MioTTS's codec path; + a VoxCPM AudioVAE encoder capacity sized to the voice directory's + longest reference), reported as one summary line. """ voice_dir: Optional[str] = None if transcripts: @@ -149,6 +165,11 @@ def _write_and_advise(audiocpp_dir: Path, wav_dir: Optional[Path], print(f"[OK] Wrote {prompt_path}") voice_dir = str(wav_dir.resolve()) + configured = apply_entry_session_options(model_entries, wav_dir, + audiocpp_dir) + if configured: + print(f"[OK] Added family session options to: {', '.join(configured)}") + server_config = build_server_config( host=host, port=port, backend=backend, lazy_load=lazy_load, model_entries=model_entries, voice_dir=voice_dir) @@ -269,7 +290,7 @@ def _wizard(stdscr, args: argparse.Namespace, parser: argparse.ArgumentParser raise _GoBack() return result - model_entries, entry_ids, install_guidance, \ + model_entries, entry_ids, install_guidance, companion_guidance, \ design_entry_ids, include_clone = _build_entries( s["family_keys"], s["chosen"], s["catalog_by_family"], task_picker, known_tasks=s["existing_tasks"]) @@ -277,6 +298,7 @@ def _wizard(stdscr, args: argparse.Namespace, parser: argparse.ArgumentParser "model_entries": model_entries, "entry_ids": entry_ids, "install_guidance": install_guidance, + "companion_guidance": companion_guidance, "design_entry_ids": design_entry_ids, "include_clone": include_clone, }) @@ -299,6 +321,7 @@ def _wizard(stdscr, args: argparse.Namespace, parser: argparse.ArgumentParser "model_entries": s["model_entries"], "entry_ids": s["entry_ids"], "install_guidance": s["install_guidance"], + "companion_guidance": s["companion_guidance"], "design_entry_ids": s["design_entry_ids"], "include_clone": s["include_clone"], "host": host, @@ -419,8 +442,9 @@ def _wizard(stdscr, args: argparse.Namespace, parser: argparse.ArgumentParser s["plan"] = _plan_from_mode(mode, wav_files, existing) s["download"] = bool(result.get("download")) and ( - _models.download_applicable(s["audiocpp_dir"], - s["model_entries"])) + _models.download_applicable( + s["audiocpp_dir"], s["model_entries"], + companions=s.get("companion_guidance"))) s["delete_unused"] = bool(result.get("delete_unused")) \ and bool(s["unused_entries"]) return _finalize() @@ -540,7 +564,9 @@ def _wizard(stdscr, args: argparse.Namespace, parser: argparse.ArgumentParser "visible": lambda fs: bool(s["include_clone"]), }) - if _models.download_applicable(s["audiocpp_dir"], s["model_entries"]): + if _models.download_applicable( + s["audiocpp_dir"], s["model_entries"], + companions=s.get("companion_guidance")): fields.append({ "key": "download", "label": "Download the selected models automatically?", @@ -718,7 +744,8 @@ def _execute_lanes(settings: dict, def install(emit, cancel): _models._install_models(audiocpp_dir, settings["install_guidance"], settings["download"], emit=emit, cancel=cancel, - model_entries=settings["model_entries"]) + model_entries=settings["model_entries"], + companions=settings.get("companion_guidance")) _build._print_launch_hint(audiocpp_dir, settings["output_path"]) return 0 # Everything already on disk: the install step just reports it, so the @@ -992,9 +1019,9 @@ def _collect_from_flags(args: argparse.Namespace, def task_picker(install_id: str) -> str: return TASK_VDES - model_entries, entry_ids, install_guidance, design_entry_ids, include_clone = \ - _build_entries(family_keys, chosen, catalog_by_family, - task_picker) + model_entries, entry_ids, install_guidance, companion_guidance, \ + design_entry_ids, include_clone = _build_entries( + family_keys, chosen, catalog_by_family, task_picker) # Server settings. Host is always 127.0.0.1 and the port comes from # AUDIOCPP_API_URL in app/converter/config.py (the Settings screen) — @@ -1056,6 +1083,7 @@ def _collect_from_flags(args: argparse.Namespace, "model_entries": model_entries, "entry_ids": entry_ids, "install_guidance": install_guidance, + "companion_guidance": companion_guidance, "design_entry_ids": design_entry_ids, "include_clone": include_clone, "host": host, diff --git a/app/backends/servers.py b/app/backends/servers.py index c258398..c0f8f4d 100644 --- a/app/backends/servers.py +++ b/app/backends/servers.py @@ -85,6 +85,18 @@ def _console_progress(event: dict) -> None: print(f"[ERROR] {event['message']}") +def server_log_path(name: str) -> Path: + """The full path of the managed NAME server's log file. + + ``app/logs/-server.log`` (the same file ``start`` streams stdout + and stderr into). Clients use it to surface a server's own runtime + detail — e.g. the exact allocation size a failed ggml graph build + attempted — in their error messages. The audio.cpp server's spec name + is ``"audiocpp"``. + """ + return _log_path(name) + + def _log_path(name: str) -> Path: return LOG_DIR / f"{name}-server.log" diff --git a/app/converter/clients/__init__.py b/app/converter/clients/__init__.py index d67237d..a107d29 100644 --- a/app/converter/clients/__init__.py +++ b/app/converter/clients/__init__.py @@ -51,6 +51,9 @@ from .audiocpp import ( audiocpp_family_voice_policy, audiocpp_request_error, audiocpp_script_input, + allocation_log_note, + build_trimmed_voice_reference, + nvidia_device_memory_report, ) __all__ = [ @@ -84,4 +87,8 @@ __all__ = [ "audiocpp_family_narrates", "audiocpp_family_spec_tasks", "audiocpp_family_voice_policy", "audiocpp_request_error", "audiocpp_script_input", + "allocation_log_note", "build_trimmed_voice_reference", + "nvidia_device_memory_report", "spec_request_option_names", + "AUDIOCPP_VOICE_REQUIRED_FAMILIES", "AUDIOCPP_ALLOCATION_FRAGMENTS", + "AUDIOCPP_REFERENCE_TRIM_SECONDS", ] diff --git a/app/converter/clients/audiocpp.py b/app/converter/clients/audiocpp.py index 224d24a..3637a50 100644 --- a/app/converter/clients/audiocpp.py +++ b/app/converter/clients/audiocpp.py @@ -1,14 +1,20 @@ """Client for the audio.cpp audiocpp_server (native ggml TTS families).""" +import array +import base64 import json import logging import shutil +import struct +import subprocess +import sys import tempfile import urllib.error import urllib.parse import urllib.request +import wave from pathlib import Path -from typing import Any, Dict, List, Optional, Set +from typing import Any, Dict, List, Optional, Set, Tuple from .. import config from ..audio import concat_audio_files @@ -113,6 +119,25 @@ AUDIOCPP_CLONE_ONLY_ERRORS = ( "only supports offline voice cloning", # Echo-TTS ) +# Managed-server log where ggml records its allocation failures with the +# exact attempted size and device (see allocation_log_note). +_AUDIOCPP_SERVER_LOG_NAME = "audiocpp" + + +def _ALLOCATION_HINT_TEXT() -> str: + return ( + "The server ran out of device memory while building a compute " + "graph: check what else is using the GPU, and read the server's " + "log (app/logs/audiocpp-server.log), which records the exact " + "allocation size it attempted. Cloning families that encode the " + "whole reference with attention over its length (MOSS-TTS-Local) " + "retry automatically with a shorter reference when the voice's " + "wav is readable locally. For DramaBox, adding \"session_options\": " + "{\"dramabox.mem_saver\": \"true\"} to its server.json model entry " + "trades speed for a much lower memory peak (restart the server " + "after editing).") + + # Deterministic failures whose one-line server message is not actionable # on its own: FRAGMENT -> guidance appended to the "not retryable" error. # Matched like AUDIOCPP_NON_RETRYABLE_ERRORS (case-insensitive, against the @@ -130,8 +155,11 @@ AUDIOCPP_HINTED_ERRORS = ( # accepts, so every request cloning it fails the same way. ("sample capacity exceeded", "The voice's reference audio is longer than this model's encoder " - "accepts: trim the voice's reference wav in the voices folder and " - "re-run Configure Backends → audio.cpp so the server picks it up."), + "accepts: trim the voice's reference wav in the voices folder, or " + "re-run Configure Backends → audio.cpp — the setup writes a larger " + "AudioVAE encoder-sample capacity for VoxCPM entries when a voice " + "wav is longer than the built-in ~15 s ceiling, so no trim is " + "needed (restart the server to pick the new config up)."), # An s2s-only family (e.g. PersonaPlex) hosted for generation: no # hosting of the entry makes it narrate text. ("supports only speech-to-speech", @@ -139,6 +167,37 @@ AUDIOCPP_HINTED_ERRORS = ( "task and cannot generate audiobooks. Consider deleting the model " "from the server configuration (re-run Configure Backends → " "audio.cpp and unselect it)."), + # Graph allocation failures: mostly device memory. The server's log + # carries the exact size the failed allocation attempted. + ("failed to allocate", _ALLOCATION_HINT_TEXT()), + ("allocation failed", _ALLOCATION_HINT_TEXT()), + # A companion package (e.g. MioTTS's MioCodec) is missing where the + # server looks for it. + ("model path does not exist", + "This model needs a companion package that is not installed where " + "the server looks for it (MioTTS loads MioCodec through its " + "miotts.codec_model_path session option). Re-run Configure Backends " + "→ audio.cpp: the setup downloads companion packages alongside the " + "model and writes the needed session options, then restart the " + "server."), + # The package on disk does not match its spec layout (the pre-repair + # GLM-TTS / OuteTTS installs nested their GGUF under a repo + # subdirectory; MiniMax-H3 hosts several GGUFs in one directory). + ("missing model package file", + "The model package on disk does not match its spec layout. Re-run " + "Configure Backends → audio.cpp: the model manager now repairs " + "broken package layouts when downloading, and models installed " + "with a stale layout are re-downloaded."), + ("missing model root", + "The model package on disk does not match its spec layout. Re-run " + "Configure Backends → audio.cpp: the model manager now repairs " + "broken package layouts when downloading, and models installed " + "with a stale layout are re-downloaded."), + ("model directory contains", + "The model directory holds several GGUFs where audio.cpp expects " + "one: hosting the entry from a specific GGUF file (as the current " + "setup does) makes the model loadable. Re-run Configure Backends → " + "audio.cpp to rewrite server.json with the fixed hosting."), ) # Families whose audio.cpp implementation only synthesizes by cloning a @@ -150,12 +209,37 @@ AUDIOCPP_HINTED_ERRORS = ( AUDIOCPP_CLONE_ONLY_FAMILIES = frozenset( {"chatterbox", "confucius4_tts", "echo_tts"}) +# Families whose plain-TTS route still requires a reference voice even +# though their model spec does not declare a "clone" task: Vevo2's +# zero-shot TTS route refuses every request without a timbre reference +# ("requires target_voice or voice speaker audio"), so the voice policy +# treats them like clone-only (the "All" flow then sends the picked +# voice, and a voice-less run is refused with the actionable message +# instead of failing every request server-side). +AUDIOCPP_VOICE_REQUIRED_FAMILIES = frozenset({"vevo2"}) + # How a family's voice is supplied — resolved per family from the local # audio.cpp checkout's model_specs (see audiocpp_family_voice_policy): AUDIOCPP_VOICE_REQUIRED = "required" # clone-only: a reference voice is mandatory AUDIOCPP_VOICE_OPTIONAL = "optional" # tts + clone: blank voice means plain TTS AUDIOCPP_VOICE_NONE = "none" # pure TTS: no cloning, no voice at all +# The allocation-failure fragments that trigger the server-log detail and +# the trimmed-reference retry (see AUDIOCPP_NON_RETRYABLE_ERRORS). +AUDIOCPP_ALLOCATION_FRAGMENTS = ("failed to allocate", "allocation failed") + +# A cloned reference long enough to blow up reference-attention encoders +# (MOSS-TTS-Local's codec encoder: memory grows with the reference's +# square) is retried as this many seconds of the same voice, read from +# the voice's local wav and sent as a base64 voice_ref (bounded by the +# server's 5 MiB inline-reference limit). +AUDIOCPP_REFERENCE_TRIM_SECONDS = 30.0 +_AUDIOCPP_VOICE_REF_MAX_BYTES = 5 * 1024 * 1024 + +# Warn about a nearly-full local GPU before the first request: with +# another process holding the memory, even small graph allocations fail. +_LOW_FREE_DEVICE_MIB = 4096 + # Spec cache (family -> parsed spec dict, or None for unknown). The form # consults the policy and capability tags on every menu render, so each # family's spec is read at most once per process. @@ -204,6 +288,27 @@ def audiocpp_family_spec_tasks(family: str) -> Optional[Set[str]]: return {str(task) for task in spec["tasks"]} +def spec_request_option_names(family: str) -> Set[str]: + """FAMILY's accepted request-option names from its local model spec. + + Used to decide whether an option may be attached to a request (e.g. + the reference_text carried alongside an inline voice_ref): families + whose runtime validates request options against the spec would reject + an unknown key outright. An unknown family (no local spec) yields an + empty set — the caller then omits the option rather than risking a + rejection. + """ + spec = _family_spec(family) + if not spec: + return set() + options = spec.get("options") + request = options.get("request") if isinstance(options, dict) else None + if not isinstance(request, list): + return set() + return {str(item["name"]) for item in request + if isinstance(item, dict) and item.get("name")} + + def audiocpp_entry_supports_design(family: str, task: str, model_id: str) -> bool: """Whether a server model entry can design a voice from a description. @@ -241,12 +346,15 @@ def audiocpp_family_voice_policy(family: str) -> str: voice at all; mixed families (tts + clone) may run without one (plain TTS) or clone a reference; clone-only families — the explicit AUDIOCPP_CLONE_ONLY_FAMILIES set, which also repairs specs that - wrongly claim "tts" — always need a reference voice. Unknown families - (no local specs) keep the conservative clone-only default the client - has always applied. + wrongly claim "tts" — always need a reference voice, as do the + AUDIOCPP_VOICE_REQUIRED_FAMILIES whose plain-TTS route demands a + timbre reference despite the spec not declaring a clone task (Vevo2's + zero-shot TTS route). Unknown families (no local specs) keep the + conservative clone-only default the client has always applied. """ if family == AUDIOCPP_FAMILY_QWEN3_TTS \ - or family in AUDIOCPP_CLONE_ONLY_FAMILIES: + or family in AUDIOCPP_CLONE_ONLY_FAMILIES \ + or family in AUDIOCPP_VOICE_REQUIRED_FAMILIES: # Qwen3-TTS is entry-typed (speaker/clone/design capability per # model id), so the family policy stays out of its way. return AUDIOCPP_VOICE_REQUIRED @@ -324,7 +432,8 @@ def _reference_text_error(voice: Optional[str], server_message: str) -> str: def audiocpp_request_error(status: int, detail: str, - voice: Optional[str] = None) -> Exception: + voice: Optional[str] = None, + log_note: Optional[str] = None) -> Exception: """The exception for a failed audio.cpp speech request. Deterministic request-configuration errors (a fragment in @@ -333,30 +442,206 @@ def audiocpp_request_error(status: int, detail: str, hosting errors (AUDIOCPP_CLONE_ONLY_ERRORS) also carry the re-host hint, hinted errors (AUDIOCPP_HINTED_ERRORS) their per-fragment guidance; everything else returns the plain RuntimeError the retry - loop has always retried. + loop has always retried. LOG_NOTE, when given for an allocation + failure, appends the server log's own record of the failed + allocation (the exact size it attempted, from the managed server's + log file) to the non-retryable message. """ message = _server_error_message(detail) lowered = message.lower() + allocation_failure = any( + fragment in lowered for fragment in AUDIOCPP_ALLOCATION_FRAGMENTS) + error: Exception if _REFERENCE_TEXT_FRAGMENT in lowered: - return NonRetryableTTSError( + error = NonRetryableTTSError( _reference_text_error(voice, message)) - if any(fragment in lowered for fragment in AUDIOCPP_CLONE_ONLY_ERRORS): - return NonRetryableTTSError( + elif any(fragment in lowered for fragment in AUDIOCPP_CLONE_ONLY_ERRORS): + error = NonRetryableTTSError( f"audio.cpp server returned HTTP {status} (not retryable): " f"{message}. This model family only synthesizes by cloning a " "reference voice, so its server entry must be hosted with task " '"clon" — re-run Configure Backends → audio.cpp (or edit ' "server.json) and restart the server.") - for fragment, hint in AUDIOCPP_HINTED_ERRORS: - if fragment in lowered: - return NonRetryableTTSError( + else: + hinted = next( + (hint for fragment, hint in AUDIOCPP_HINTED_ERRORS + if fragment in lowered), None) + if hinted is not None: + error = NonRetryableTTSError( f"audio.cpp server returned HTTP {status} (not retryable): " - f"{message}. {hint}") - if any(fragment in lowered for fragment in AUDIOCPP_NON_RETRYABLE_ERRORS): - return NonRetryableTTSError( - f"audio.cpp server returned HTTP {status} (not retryable): " - f"{message}") - return RuntimeError(f"audio.cpp server returned HTTP {status}: {detail}") + f"{message}. {hinted}") + elif any(fragment in lowered + for fragment in AUDIOCPP_NON_RETRYABLE_ERRORS): + error = NonRetryableTTSError( + f"audio.cpp server returned HTTP {status} (not retryable): " + f"{message}") + else: + error = RuntimeError( + f"audio.cpp server returned HTTP {status}: {detail}") + if allocation_failure and log_note \ + and isinstance(error, NonRetryableTTSError): + error = NonRetryableTTSError(f"{error}{log_note}") + return error + + +def allocation_log_note(message: str) -> str: + """The managed server's own record of a failed allocation, best-effort. + + ggml logs every failed backend-buffer allocation with the exact size + it attempted and the device ("allocating N MiB on device D: cudaMalloc + failed"), while the server's HTTP 500 only carries the model's one-line + message — so the log is where the number lives. Only meaningful for + the locally managed server (app/logs/audiocpp-server.log); remote + servers, or a moved/rotated log, yield "" and the message stays as-is. + """ + lowered = message.lower() + if not any(fragment in lowered + for fragment in AUDIOCPP_ALLOCATION_FRAGMENTS): + return "" + try: + # Imported lazily: backends.audiocpp imports this package, so a + # module-level import would cycle. + from backends import servers as _servers + path = _servers.server_log_path(_AUDIOCPP_SERVER_LOG_NAME) + except Exception: # noqa: BLE001 - diagnostics only, never fatal + return "" + try: + text = path.read_text(encoding="utf-8", errors="replace") + except OSError: + return "" + patterns = ("cudaMalloc failed", "not enough space in the buffer", + "failed to allocate") + matches = [line.strip() for line in text.splitlines() + if any(pattern in line for pattern in patterns)] + if not matches: + return "" + return (f" The server's log ({path}) records the failed allocation as: " + f"{matches[-1]}") + + +def nvidia_device_memory_report() -> Optional[str]: + """Local NVIDIA GPUs' total/free memory as one CSV block, or None. + + Runs ``nvidia-smi --query-gpu=index,memory.total,memory.free`` once; + None when the tool is missing (non-NVIDIA machines), times out, or + reports an error. Used by the client to warn about a nearly-full GPU + before the first request — with another process holding the memory, + even small graph allocations fail. + """ + try: + result = subprocess.run( + ["nvidia-smi", "--query-gpu=index,memory.total,memory.free", + "--format=csv,noheader,nounits"], + capture_output=True, text=True, timeout=15) + except (OSError, subprocess.SubprocessError): + return None + if result.returncode != 0 or not result.stdout.strip(): + return None + return result.stdout.strip() + + +def _pcm16_mono_samples(path: Path, max_seconds: float + ) -> Optional[Tuple[int, List[int]]]: + """The first MAX_SECONDS of a wav as (sample rate, mono PCM16 samples). + + Reads with the stdlib wave module (PCM u8/s16/s24/s32), mixes channels + by averaging, and stops at MAX_SECONDS so a long reference costs only + the frames actually sent. Returns None for files the wave module + cannot parse (float-format wavs, non-WAV files) or that carry no + samples — the trimmed-reference retry then does not fire. + """ + max_frames = 0 + if max_seconds > 0: + try: + with wave.open(str(path), "rb") as probe: + rate = probe.getframerate() + max_frames = int(max_seconds * rate) + 1 + except (OSError, EOFError, wave.Error): + return None + try: + with wave.open(str(path), "rb") as handle: + rate = handle.getframerate() + channels = handle.getnchannels() + width = handle.getsampwidth() + frames = handle.getnframes() + raw = handle.readframes(min(frames, max_frames) if max_frames + else frames) + except (OSError, EOFError, wave.Error): + return None + if rate <= 0 or channels <= 0 or width not in (1, 2, 3, 4) or not raw: + return None + frame_bytes = width * channels + frame_count = len(raw) // frame_bytes + if frame_count <= 0: + return None + if width == 2: + values = array.array("h") + values.frombytes(raw[:frame_count * frame_bytes]) + if sys.byteorder == "big": + values.byteswap() + elif width == 1: + # u8 -> s16, scaled to the full 16-bit range. + values = array.array("h", ((byte - 128) << 8 + for byte in raw[:frame_count])) + else: + # s24/s32 -> s16 by dropping the low bits (keeps every value inside + # int16 so the mixdown and the PCM16 container need no clipping). + values = array.array( + "i", (int.from_bytes(raw[i * width:i * width + width], + "little", signed=True) + for i in range(frame_count))) + shift = 8 if width == 3 else 16 + values = array.array("h", (value >> shift for value in values)) + if channels == 1: + return rate, values.tolist() + mixed: List[int] = [] + for index in range(frame_count): + start = index * channels + mixed.append(sum(values[start:start + channels]) // channels) + return rate, mixed + + +def pcm16_wav_bytes(sample_rate: int, samples: List[int]) -> bytes: + """Wrap mono PCM16 samples in a minimal RIFF/WAVE container.""" + data = array.array("h", samples) + if sys.byteorder == "big": + data.byteswap() + payload = data.tobytes() + return (b"RIFF" + + struct.pack(" Optional[Tuple[str, float, str]]: + """A base64 voice_ref of the first MAX_SECONDS of a reference wav. + + Returns (base64 wav, seconds used, file name), or None when PATH is + missing or unreadable. The payload keeps the file's native sample rate + and is mixed to mono; it is further capped so the decoded bytes stay + within the server's 5 MiB inline-reference limit (16-bit mono means + ~2.6 MB per 30 s at 44.1 kHz, comfortably inside). + """ + if path is None: + return None + decoded = _pcm16_mono_samples(path, max_seconds) + if decoded is None: + return None + rate, samples = decoded + if not samples: + return None + byte_cap_seconds = max_bytes / (2 * rate) + frames = int(min(max_seconds, byte_cap_seconds) * rate) + samples = samples[:frames] + if not samples: + return None + wav = pcm16_wav_bytes(rate, samples) + return base64.b64encode(wav).decode("ascii"), len(samples) / rate, path.name class AudioCppFamilyProfile: @@ -443,6 +728,12 @@ def audiocpp_entry_voice_capability(family: str, task: str, class AudioCppTTSClient(BaseTTSClient): + # Class-level defaults so a partially-constructed instance behaves like + # a fresh run (tests build clients via __new__; see BaseTTSClient). + _voice_ref_b64: Optional[str] = None + _voice_ref_reference_text: Optional[str] = None + _reference_trim_attempted = False + """Generates audio chunks through an audio.cpp audiocpp_server. Talks to the OpenAI-style HTTP API of audiocpp_server, which hosts TTS @@ -559,6 +850,15 @@ class AudioCppTTSClient(BaseTTSClient): self.design_mode = False self.instruction_voice = False self.plain_mode = False + # Trimmed-reference retry state (see _switch_to_trimmed_reference): + # an inline base64 voice_ref that replaces the voice name after an + # allocation failure, the transcript carried alongside it, and the + # one-attempt guard. The class-level defaults above keep partially + # constructed instances (tests via __new__) behaving like a fresh + # run; the assignments here shadow them for this instance. + self._voice_ref_b64 = None + self._voice_ref_reference_text = None + self._reference_trim_attempted = False # Family and task of the selected model entry and the family's request # profile; all are resolved from GET /v1/models during _connect. self.family = "" @@ -653,29 +953,33 @@ class AudioCppTTSClient(BaseTTSClient): f"'{self.family}') serves built-in speakers: pass " "--voice NAME with one of them (e.g. Vivian, Ryan, " "Uncle Fu) to synthesize with it (see README).") + elif audiocpp_family_voice_policy(self.family) \ + == AUDIOCPP_VOICE_REQUIRED: + # Checked before the instruction-voice branch: a family + # whose synthesis needs a reference voice (clone-only, + # Vevo2's zero-shot route) cannot take its voice from an + # instruction, so refuse with the fix instead of failing + # every request server-side. + raise RuntimeError( + f"The audio.cpp model '{self.model_id}' (family " + f"'{self.family}') has no built-in speakers, so its voice " + "must come from the server: rerun with --voice NAME " + "matching a voice_preset or voice_dir entry in the server " + "config, or select the CustomVoice entry for built-in " + "speakers (see README).") elif self.instructions: # Families without built-in speakers can still get their voice # from the instruction alone (e.g. OmniVoice voice design). self.instruction_voice = True self._connected("instruction voice") self._report(f"[INFO] Designing the voice from: {self.instructions}") - elif audiocpp_family_voice_policy(self.family) in ( - AUDIOCPP_VOICE_OPTIONAL, AUDIOCPP_VOICE_NONE): + else: # The family synthesizes without a reference voice — a # pure-TTS family (spec tasks without "clone") or a mixed # tts+clone family used without one. Plain TTS: no voice # field is sent at all. self.plain_mode = True self._connected("plain TTS") - else: - raise RuntimeError( - f"The audio.cpp model '{self.model_id}' (family " - f"'{self.family}') has no built-in speakers, so its voice " - "must come from the server: rerun with --voice NAME " - "matching a voice_preset or voice_dir entry in the server " - "config, or describe a voice with --instructions for " - "families that support it, or select the CustomVoice entry " - "for built-in speakers (see README).") if self.instructions and not self.design_mode and not self.instruction_voice: self._report(f"[INFO] Sending instruction with every request: {self.instructions}") self._report("[INFO] Its effect (style, emotion, delivery) depends on the " @@ -685,6 +989,7 @@ class AudioCppTTSClient(BaseTTSClient): else self._unload_models_override) if unload: self._unload_server_models() + self._warn_low_device_memory() def _require_synthesis_task(self, models: List[Dict[str, str]]) -> None: """Reject model entries that cannot synthesize narration from text. @@ -926,6 +1231,168 @@ class AudioCppTTSClient(BaseTTSClient): "with --voice (see README)." ) + # ------------------------------------------------------------------ + # Reference trimming and device diagnostics + # ------------------------------------------------------------------ + + def _warn_low_device_memory(self) -> None: + """Warn once when a local GPU is nearly full before the first request. + + Graph-allocation failures read as "out of memory" even when the + card has gigabytes free for the model itself — what matters is the + free memory at request time, which other processes can hold. A + one-time nvidia-smi query (local hosts only; skipped silently + elsewhere or when the tool is missing) turns that case into an + explicit warning instead of a mysterious 500. + """ + host = (urllib.parse.urlparse(self.api_url).hostname or "").lower() + if host not in ("127.0.0.1", "localhost", "::1"): + return + report = nvidia_device_memory_report() + if not report: + return + for row in report.splitlines(): + parts = [part.strip() for part in row.split(",")] + if len(parts) < 3: + continue + try: + index = int(parts[0]) + total = int(parts[1]) + free = int(parts[2]) + except ValueError: + continue + if free < _LOW_FREE_DEVICE_MIB: + self._report( + f"[WARNING] GPU {index} has {free} MiB free of " + f"{total} MiB — allocation failures under this " + "condition usually mean another process is using " + "the GPU.") + + def _voice_wav_path(self) -> Optional[Path]: + """The selected voice's reference wav on this machine, when readable. + + Resolved from the local checkout's server.json exactly like the + server resolves the request's voice name: the entry's + ``voice_presets[name].voice_ref`` first, then ``voice_dir/.wav``. + Only preset-mode runs with a locally readable file return a path — + remote-only servers (or voice names that only exist server-side) + yield None and the trimmed-reference retry does not fire. + """ + if not self.voice: + return None + name = self.voice + if not name or name in (".", "..") or "/" in name or "\\" in name: + return None + try: + # Imported lazily: backends.audiocpp imports this package, so a + # module-level import would cycle. + from backends.audiocpp.build import find_local_checkout + checkout = find_local_checkout() + except Exception: # noqa: BLE001 - best effort + return None + if checkout is None: + return None + server_json = checkout / "server.json" + try: + data = json.loads(server_json.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + if not isinstance(data, dict): + return None + for entry in data.get("models") or []: + if not isinstance(entry, dict) or entry.get("id") != self.model_id: + continue + presets = entry.get("voice_presets") + if isinstance(presets, dict): + preset = presets.get(name) + if isinstance(preset, dict): + ref = preset.get("voice_ref") + if isinstance(ref, str) and ref: + path = Path(ref) + if not path.is_absolute(): + path = server_json.parent / ref + if path.is_file(): + return path + voice_dir = data.get("voice_dir") + if isinstance(voice_dir, str) and voice_dir: + candidate = Path(voice_dir) / f"{name}.wav" + if candidate.is_file(): + return candidate + return None + + def _voice_transcript(self) -> Optional[str]: + """The voice library transcript for the selected voice, or None. + + Read from the voice directory's prompt_text mapping (the same file + the server consults when it resolves a voice NAME); only carried + alongside an inline voice_ref, where the server's own injection is + bypassed. + """ + if not self.voice: + return None + try: + # Imported lazily: backends.audiocpp imports this package, so a + # module-level import would cycle. + from backends.common import PROMPT_TEXT_FILENAME, read_prompt_text + from backends.audiocpp.build import find_local_checkout + checkout = find_local_checkout() + except Exception: # noqa: BLE001 - best effort + return None + if checkout is None: + return None + try: + data = json.loads((checkout / "server.json") + .read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + voice_dir = data.get("voice_dir") if isinstance(data, dict) else None + if not isinstance(voice_dir, str) or not voice_dir: + return None + try: + return (read_prompt_text(Path(voice_dir) / PROMPT_TEXT_FILENAME) + .get(self.voice) or None) + except OSError: + return None + + def _switch_to_trimmed_reference(self, server_message: str) -> bool: + """Switch the cloning reference to a trimmed local wav, once. + + Some families encode the whole reference with attention over its + length (MOSS-TTS-Local's codec encoder: the required memory grows + with the reference's square), so a long voice reference fails the + graph allocation regardless of how much VRAM the device has. When + the selected voice resolves to a locally readable wav, replace the + voice name with an inline base64 voice_ref cut to + AUDIOCPP_REFERENCE_TRIM_SECONDS and retry the request once; the + trimmed reference then applies to the rest of the run. Only fires + on allocation-failure messages for preset-mode runs; everything + else keeps the original behavior. + """ + lowered = server_message.lower() + if self._reference_trim_attempted \ + or not any(fragment in lowered + for fragment in AUDIOCPP_ALLOCATION_FRAGMENTS) \ + or self.design_mode or self.instruction_voice \ + or self.plain_mode or not self.voice: + return False + self._reference_trim_attempted = True + trimmed = build_trimmed_voice_reference(self._voice_wav_path()) + if trimmed is None: + return False + b64, seconds, source = trimmed + self._voice_ref_b64 = b64 + transcript = self._voice_transcript() + if transcript and "reference_text" in spec_request_option_names( + self.family): + self._voice_ref_reference_text = transcript + self._report( + f"[INFO] {source}'s family encodes the whole reference with " + f"attention over its length; retrying with the first " + f"{seconds:.0f}s of voice '{self.voice}' as the cloning " + "reference (the trimmed reference applies to the rest of this " + "run).") + return True + # ------------------------------------------------------------------ # HTTP requests # ------------------------------------------------------------------ @@ -946,8 +1413,13 @@ class AudioCppTTSClient(BaseTTSClient): # Design models take no voice field (the voice comes from the # instruction); instruction-voice runs on families without built-in # speakers omit it too, since no speaker or preset was requested; - # plain-TTS runs (no reference voice needed) omit it likewise. - if not self.design_mode and not self.instruction_voice \ + # plain-TTS runs (no reference voice needed) omit it likewise. A + # trimmed-reference retry replaces the voice name with an inline + # base64 voice_ref (see _switch_to_trimmed_reference). + if self._voice_ref_b64 is not None: + payload["voice_ref"] = {"type": "base64", + "data": self._voice_ref_b64} + elif not self.design_mode and not self.instruction_voice \ and not self.plain_mode: payload["voice"] = self.voice if self.profile.language_style == AUDIOCPP_LANG_DISPLAY: @@ -968,10 +1440,17 @@ class AudioCppTTSClient(BaseTTSClient): # Explicit voice-design or style instruction (required for task # "vdes" entries; a Ctrl/style control on families that read it). payload["instructions"] = self.instructions - if self.request_options: + options = dict(self.request_options) + if self._voice_ref_reference_text \ + and "reference_text" not in options: + # The server only injects the voice library's transcript when it + # resolves the voice NAME; an inline voice_ref bypasses that, so + # carry the transcript explicitly for families that accept it. + options["reference_text"] = self._voice_ref_reference_text + if options: # Generic per-model controls (--option KEY=VALUE): forwarded # verbatim; the model ignores keys it does not know. - payload["options"] = dict(self.request_options) + payload["options"] = options request = urllib.request.Request( url, data=json.dumps(payload).encode("utf-8"), headers={"Content-Type": "application/json"}, method="POST") @@ -985,8 +1464,15 @@ class AudioCppTTSClient(BaseTTSClient): detail = exc.read().decode("utf-8", errors="replace")[:200] except Exception: pass + message = _server_error_message(detail) + if self._switch_to_trimmed_reference(message): + # One retry with the trimmed reference; if that fails too the + # error below carries the server log's allocation detail. + return self._request_wav(text) raise audiocpp_request_error(exc.code, detail, - voice=self.voice) from exc + voice=self.voice, + log_note=allocation_log_note( + message)) from exc except urllib.error.URLError as exc: raise RuntimeError(f"audio.cpp request failed: {exc.reason}") from exc if len(wav) < 12 or wav[:4] != b"RIFF" or wav[8:12] != b"WAVE": diff --git a/app/tests/test_backends_audiocpp.py b/app/tests/test_backends_audiocpp.py index 7edb3c9..18c38c4 100644 --- a/app/tests/test_backends_audiocpp.py +++ b/app/tests/test_backends_audiocpp.py @@ -710,7 +710,7 @@ class BuildEntriesHostingTests(unittest.TestCase): "design": False, "recommended": True} def _entries(self, catalog_entry): - entries, _, _, _, _ = make_server.wizard._build_entries( + entries, _, _, _, _, _ = make_server.wizard._build_entries( [catalog_entry["family"]], {catalog_entry["family"]: [self._option(catalog_entry["family"])]}, {catalog_entry["family"]: catalog_entry}, @@ -4242,3 +4242,456 @@ class PrebuiltFallbackTests(unittest.TestCase): self.assertEqual(rc, 0) mk_i.assert_not_called() mk_build.assert_called_once() + + +class MissingStripPrefixSanitizeTests(unittest.TestCase): + """The missing-strip_prefix repair for nested single-GGUF packages.""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.checkout_holder = Path(self._tmp.name) / "audio.cpp" + self.checkout_holder.mkdir() + + def tearDown(self): + self._tmp.cleanup() + + _GGUF_SPEC = { + "family": "glm_tts", + "sources": [{"format": "gguf", + "roots": {"model": ".", "weights": "$gguf"}}], + } + + def _spec(self, files, roots=(("model", "."), ("weights", "$gguf")), + fmt="gguf"): + spec = json.loads(json.dumps(self._GGUF_SPEC)) + spec["sources"][0]["roots"] = dict(roots) + spec["sources"][0]["format"] = fmt + spec["packages"] = [{"id": "pkg", "format": fmt, "files": files}] + return spec + + def test_nested_single_gguf_gets_its_prefix(self): + spec = self._spec(["Text to audio (TTS)/GLM-TTS_Q8.gguf"]) + self.assertTrue(make_server.catalog.sanitize_model_spec(spec)) + self.assertEqual(spec["packages"][0]["strip_prefix"], + "Text to audio (TTS)") + + def test_explicit_gguf_paths_without_gguf_root_are_untouched(self): + # minimax_music3-style: the gguf source names tensors by explicit + # file paths (no $gguf root), so the nested layout is intentional. + spec = self._spec(["config/a.json", "tokenizer/t.json", + "language_model_q4_0.gguf"], + roots=(("model", "."),)) + self.assertFalse(make_server.catalog.sanitize_model_spec(spec)) + + def test_mixed_prefixes_are_untouched(self): + spec = self._spec(["config/a.json", "model.safetensors"]) + self.assertFalse(make_server.catalog.sanitize_model_spec(spec)) + + def test_flat_package_is_untouched(self): + spec = self._spec(["model.gguf"]) + self.assertFalse(make_server.catalog.sanitize_model_spec(spec)) + + def test_safetensors_packages_are_untouched(self): + spec = self._spec(["Some-Dir/model.safetensors"], fmt="safetensors") + self.assertFalse(make_server.catalog.sanitize_model_spec(spec)) + + def test_catalog_carries_the_sanitized_prefix(self): + _write_spec(self.checkout_holder, "glm_like", + packages=[{ + "id": "glm_like_q8_0", "default": True, + "format": "gguf", + "target_directory": "GLM-Like-Q8", + "files": ["Text to audio (TTS)/GLM-Like_Q8.gguf"], + }]) + specs_dir = self.checkout_holder / "model_specs" + spec = json.loads( + (specs_dir / "glm_like.json").read_text(encoding="utf-8")) + spec["sources"] = [{"format": "gguf", + "roots": {"model": ".", "weights": "$gguf"}}] + (specs_dir / "glm_like.json").write_text( + json.dumps(spec), encoding="utf-8") + catalog = make_server.catalog.load_model_catalog(self.checkout_holder) + glm_like = next(e for e in catalog if e["family"] == "glm_like") + self.assertEqual(glm_like["packages"][0]["strip_prefix"], + "Text to audio (TTS)") + self.assertEqual( + make_server.catalog.entry_model_path(glm_like), + "models/GLM-Like-Q8") + + +class EntryModelPathTests(unittest.TestCase): + """entry_model_path: directory hosting vs. the multi-GGUF file rule.""" + + def _entry(self, packages, default_directory=None): + if default_directory is None and packages: + default_directory = str( + packages[0].get("target_directory") or "Bundle-GGUF") + return {"family": "minimax_h3", "packages": packages, + "default_path": f"models/{default_directory or ''}"} + + def test_multi_gguf_package_is_hosted_from_its_first_gguf(self): + entry = self._entry([{ + "id": "minimax_h3_q4_k", "default": True, "format": "gguf", + "target_directory": "MiniMax-H3-Q4-GGUF", + "strip_prefix": "MiniMax-H3-Q4-GGUF", + "files": [ + "MiniMax-H3-Q4-GGUF/configuration.json", + "MiniMax-H3-Q4-GGUF/text_encoder_q4_k.gguf", + "MiniMax-H3-Q4-GGUF/dit.gguf", + "MiniMax-H3-Q4-GGUF/audio_vae_folded_f16.gguf", + "MiniMax-H3-Q4-GGUF/video_vae.gguf", + ], + }]) + self.assertEqual( + make_server.catalog.entry_model_path(entry), + "models/MiniMax-H3-Q4-GGUF/text_encoder_q4_k.gguf") + + def test_single_gguf_package_hosts_the_directory(self): + entry = self._entry([{ + "id": "voxcpm2_q8_0", "default": True, "format": "gguf", + "target_directory": "VoxCPM2-GGUF", + "strip_prefix": "VoxCPM2-GGUF", + "files": ["VoxCPM2-GGUF/voxcpm2-q8_0.gguf"], + }], default_directory="VoxCPM2-GGUF") + self.assertEqual(make_server.catalog.entry_model_path(entry), + "models/VoxCPM2-GGUF") + + def test_alternate_directory_uses_that_packages_files(self): + entry = self._entry([ + {"id": "a_q8", "default": True, "format": "gguf", + "target_directory": "A-GGUF", + "files": ["A-GGUF/a.gguf", "A-GGUF/b.gguf"]}, + {"id": "b_q8", "format": "gguf", "target_directory": "B-GGUF", + "files": ["B-GGUF/b.gguf"]}, + ]) + self.assertEqual(make_server.catalog.entry_model_path(entry, "B-GGUF"), + "models/B-GGUF") + + def test_safetensors_directory_hosts_the_directory(self): + entry = self._entry([{ + "id": "voxcpm2_safetensors", "format": "safetensors", + "target_directory": "VoxCPM2", + "files": ["config.json", "model.safetensors"], + }], default_directory="VoxCPM2") + self.assertEqual(make_server.catalog.entry_model_path(entry), + "models/VoxCPM2") + + def test_entry_without_packages_falls_back_to_the_directory(self): + entry = self._entry([], default_directory="Fallback-GGUF") + self.assertEqual(make_server.catalog.entry_model_path(entry), + "models/Fallback-GGUF") + + +class BuildModelEntrySessionOptionsTests(unittest.TestCase): + """build_model_entry carries per-entry session options when given.""" + + def test_session_options_added_when_given(self): + entry = make_server.catalog.build_model_entry( + "miotts", "MioTTS-1.7B-GGUF", "models/MioTTS-1.7B-GGUF", + session_options={"miotts.codec_model_path": "models/MioCodec"}) + self.assertEqual(entry["session_options"], + {"miotts.codec_model_path": "models/MioCodec"}) + + def test_session_options_omitted_when_empty(self): + entry = make_server.catalog.build_model_entry( + "miotts", "MioTTS-1.7B-GGUF", "models/MioTTS-1.7B-GGUF") + self.assertNotIn("session_options", entry) + + +class FilePrecisePresenceTests(unittest.TestCase): + """Installed checks are file-precise against the catalog packages.""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.checkout = Path(self._tmp.name) / "audio.cpp" + self.checkout.mkdir() + specs = self.checkout / "model_specs" + specs.mkdir() + spec = { + "family": "glm_like", "category": "tts", "tasks": ["tts", "clone"], + "packages": [{ + "id": "glm_like_q8_0", "default": True, "format": "gguf", + "target_directory": "GLM-Like-Q8", + "files": ["Text to audio (TTS)/GLM-Like_Q8.gguf"], + }], + "sources": [{"format": "gguf", + "roots": {"model": ".", "weights": "$gguf"}}], + } + (specs / "glm_like.json").write_text(json.dumps(spec), + encoding="utf-8") + + def tearDown(self): + self._tmp.cleanup() + + def _entry(self, rel="models/GLM-Like-Q8"): + return {"id": "GLM-Like-Q8", "family": "glm_like", "path": rel} + + def test_stale_nested_layout_counts_as_missing(self): + stale = self.checkout / "models" / "GLM-Like-Q8" \ + / "Text to audio (TTS)" + stale.mkdir(parents=True) + (stale / "GLM-Like_Q8.gguf").write_bytes(b"x") + self.assertFalse( + make_server.models._all_models_present(self.checkout, + [self._entry()])) + server_json = self.checkout / "server.json" + server_json.write_text(json.dumps({"models": [self._entry()]}), + encoding="utf-8") + missing = make_server.models.missing_model_entries(server_json) + self.assertEqual([m["id"] for m in missing], ["GLM-Like-Q8"]) + + def test_flat_layout_after_the_repair_counts_as_installed(self): + target = self.checkout / "models" / "GLM-Like-Q8" + target.mkdir(parents=True) + (target / "GLM-Like_Q8.gguf").write_bytes(b"x") + self.assertTrue( + make_server.models._all_models_present(self.checkout, + [self._entry()])) + server_json = self.checkout / "server.json" + server_json.write_text(json.dumps({"models": [self._entry()]}), + encoding="utf-8") + self.assertEqual( + make_server.models.missing_model_entries(server_json), []) + + def test_file_style_entry_of_a_multi_gguf_package(self): + specs = self.checkout / "model_specs" + spec = { + "family": "multi", "category": "tts", "tasks": ["tts"], + "packages": [{ + "id": "multi_q4", "default": True, "format": "gguf", + "target_directory": "Multi-Q4-GGUF", + "strip_prefix": "Multi-Q4-GGUF", + "files": ["Multi-Q4-GGUF/dit.gguf", + "Multi-Q4-GGUF/vae.gguf"], + }], + } + (specs / "multi.json").write_text(json.dumps(spec), encoding="utf-8") + target = self.checkout / "models" / "Multi-Q4-GGUF" + target.mkdir(parents=True) + (target / "dit.gguf").write_bytes(b"x") + (target / "vae.gguf").write_bytes(b"x") + self.assertTrue(make_server.models._all_models_present( + self.checkout, + [{"id": "Multi", "path": "models/Multi-Q4-GGUF/dit.gguf"}])) + + def test_unmatched_entry_keeps_the_plain_path_check(self): + target = self.checkout / "elsewhere" + target.mkdir() + (target / "m.gguf").write_bytes(b"x") + self.assertTrue(make_server.models._all_models_present( + self.checkout, [{"id": "x", "path": str(target)}])) + + +class CompanionInstallTests(unittest.TestCase): + """Companion packages (MioCodec for MioTTS) join the install list.""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.checkout = Path(self._tmp.name) / "audio.cpp" + self.checkout.mkdir() + specs = self.checkout / "model_specs" + specs.mkdir() + manager = self.checkout / "tools" / "model_manager_v2.py" + manager.parent.mkdir() + manager.write_text("#!/usr/bin/env python3\n", encoding="utf-8") + (specs / "miocodec.json").write_text(json.dumps({ + "family": "miocodec", "category": "audio_tools", + "tasks": ["codec"], + "packages": [{ + "id": "miocodec_q8_0", "default": True, "format": "gguf", + "target_directory": "MioCodec-25Hz-44.1kHz-v2-GGUF", + "strip_prefix": "MioCodec-25Hz-44.1kHz-v2-GGUF", + "files": ["MioCodec-25Hz-44.1kHz-v2-GGUF/codec.gguf"], + }], + }), encoding="utf-8") + + def tearDown(self): + self._tmp.cleanup() + + def test_missing_companion_is_merged_into_pending(self): + merged = make_server.models._merge_companions( + self.checkout, [], [("MioCodec", "miocodec_q8_0")]) + self.assertEqual(merged, [("MioCodec", "miocodec_q8_0")]) + + def test_installed_companion_is_reported_and_skipped(self): + target = self.checkout / "models" / "MioCodec-25Hz-44.1kHz-v2-GGUF" + target.mkdir(parents=True) + (target / "codec.gguf").write_bytes(b"x") + buf = io.StringIO() + with redirect_stdout(buf): + merged = make_server.models._merge_companions( + self.checkout, [], [("MioCodec", "miocodec_q8_0")]) + self.assertEqual(merged, []) + self.assertIn("MioCodec is already installed", buf.getvalue()) + + def test_install_models_prints_the_companion_command(self): + buf = io.StringIO() + with redirect_stdout(buf): + rc = make_server.models._install_models( + self.checkout, [], download=False, + companions=[("MioCodec", "miocodec_q8_0")]) + self.assertEqual(rc, 0) + self.assertIn("install miocodec_q8_0", buf.getvalue()) + + def test_install_models_runs_the_companion_download(self): + buf = io.StringIO() + with redirect_stdout(buf), \ + patch.object(common, "run_console_subprocess", + return_value=0) as run: + rc = make_server.models._install_models( + self.checkout, [], download=True, + companions=[("MioCodec", "miocodec_q8_0")]) + self.assertEqual(rc, 0) + argv = run.call_args[0][0] + self.assertEqual(argv[-2:], ["install", "miocodec_q8_0"]) + + +class ApplyEntrySessionOptionsTests(unittest.TestCase): + """The wizard bakes companion/session options into server.json entries.""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.checkout = Path(self._tmp.name) / "audio.cpp" + self.checkout.mkdir() + self.wav_dir = Path(self._tmp.name) / "voices" + self.wav_dir.mkdir() + + def tearDown(self): + self._tmp.cleanup() + + def _write_wav(self, name, seconds, rate=16000): + import wave as wave_mod + with wave_mod.open(str(self.wav_dir / name), "wb") as handle: + handle.setnchannels(1) + handle.setsampwidth(2) + handle.setframerate(rate) + handle.writeframes(b"\x00\x00" * int(rate * seconds)) + + def test_miotts_entry_gets_the_codec_model_path(self): + entries = [{"id": "MioTTS-1.7B-GGUF", "family": "miotts"}] + applied = make_server.catalog.apply_entry_session_options( + entries, None, self.checkout) + self.assertEqual(applied, ["MioTTS-1.7B-GGUF"]) + self.assertEqual( + entries[0]["session_options"]["miotts.codec_model_path"], + "models/MioCodec-25Hz-44.1kHz-v2-GGUF") + + def test_hand_set_codec_path_is_not_overridden(self): + entries = [{"id": "MioTTS-1.7B-GGUF", "family": "miotts", + "session_options": + {"miotts.codec_model_path": "/custom/codec"}}] + applied = make_server.catalog.apply_entry_session_options( + entries, None, self.checkout) + self.assertEqual(applied, []) + self.assertEqual( + entries[0]["session_options"]["miotts.codec_model_path"], + "/custom/codec") + + def test_voxcpm_capacity_sized_to_the_longest_voice(self): + self._write_wav("short.wav", 5) + self._write_wav("long.wav", 40) + entries = [{"id": "VoxCPM2-GGUF", "family": "voxcpm2"}] + applied = make_server.catalog.apply_entry_session_options( + entries, self.wav_dir, self.checkout) + self.assertEqual(applied, ["VoxCPM2-GGUF"]) + self.assertEqual( + entries[0]["session_options"] + ["voxcpm2.audiovae_encoder_sample_capacity"], "720000") + + def test_short_voices_need_no_capacity(self): + self._write_wav("short.wav", 5) + entries = [{"id": "VoxCPM2-GGUF", "family": "voxcpm2"}] + applied = make_server.catalog.apply_entry_session_options( + entries, self.wav_dir, self.checkout) + self.assertEqual(applied, []) + self.assertNotIn("session_options", entries[0]) + + def test_no_voice_directory_means_no_capacity(self): + entries = [{"id": "VoxCPM2-GGUF", "family": "voxcpm2"}] + make_server.catalog.apply_entry_session_options(entries, None, + self.checkout) + self.assertNotIn("session_options", entries[0]) + + def test_existing_session_options_are_preserved(self): + entries = [{"id": "VoxCPM2-GGUF", "family": "voxcpm2", + "session_options": {"voxcpm2.mem_saver": "true"}}] + self._write_wav("long.wav", 40) + make_server.catalog.apply_entry_session_options( + entries, self.wav_dir, self.checkout) + options = entries[0]["session_options"] + self.assertEqual(options["voxcpm2.mem_saver"], "true") + self.assertIn("voxcpm2.audiovae_encoder_sample_capacity", options) + + +class FilePathSelectionTests(unittest.TestCase): + """Selections and unused-model matching for file-hosted entries.""" + + def test_file_style_entry_maps_to_its_directory(self): + config = {"models": [ + {"id": "MiniMax-H3-Q4-GGUF", "family": "minimax_h3", + "path": "models/MiniMax-H3-Q4-GGUF/text_encoder_q4_k.gguf", + "task": "tts"}, + ]} + catalog = [{"family": "minimax_h3", "packages": [], + "default_path": "models/MiniMax-H3-Q4-GGUF"}] + selected, tasks = make_server.catalog.server_config_selections( + config, catalog) + self.assertEqual(selected["minimax_h3"], ["MiniMax-H3-Q4-GGUF"]) + self.assertEqual(tasks[("minimax_h3", "MiniMax-H3-Q4-GGUF")], "tts") + + def test_directory_style_entry_matches_a_file_style_selection(self): + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + server_json = Path(self._tmp.name) / "server.json" + server_json.write_text(json.dumps({"models": [{ + "id": "MiniMax-H3-Q4-GGUF", "family": "minimax_h3", + "path": "models/MiniMax-H3-Q4-GGUF", + }]}), encoding="utf-8") + unused = make_server.models.unused_installed_entries( + server_json, {"models/MiniMax-H3-Q4-GGUF/dit.gguf"}) + self.assertEqual(unused, []) + + +class WizardCompanionGuidanceTests(unittest.TestCase): + """_build_entries returns MioCodec guidance for MioTTS selections.""" + + _CATALOG_ENTRY = { + "family": "miotts", "display_name": "MioTTS", "description": "", + "languages": ["en"], "tasks": ["tts", "clone"], "clone_capable": True, + "packages": [{ + "id": "miotts_1_7b_q8_0", "default": True, "format": "gguf", + "target_directory": "MioTTS-1.7B-GGUF", + "strip_prefix": "MioTTS-1.7B-GGUF", + "files": ["MioTTS-1.7B-GGUF/miotts-1.7b-q8_0.gguf"], + }], + "install_id": "miotts_1_7b_q8_0", + "default_path": "models/MioTTS-1.7B-GGUF", + } + + def test_miotts_selection_carries_the_mio_codec_companion(self): + entries, _ids, guidance, companions, _design, _clone = \ + make_server.wizard._build_entries( + ["miotts"], + {"miotts": [{"target_directory": "MioTTS-1.7B-GGUF", + "install_id": "miotts_1_7b_q8_0", + "design": False, "recommended": True}]}, + {"miotts": self._CATALOG_ENTRY}, + lambda install_id: "tts") + self.assertEqual(companions, + [("MioCodec 25Hz 44.1kHz v2 (required by MioTTS)", + "miocodec_q8_0")]) + self.assertEqual(entries[0]["path"], "models/MioTTS-1.7B-GGUF") + self.assertEqual(len(guidance), 1) + + def test_other_families_carry_no_companions(self): + catalog_entry = dict(self._CATALOG_ENTRY, family="voxcpm2", + display_name="VoxCPM2") + _entries, _ids, _guidance, companions, _design, _clone = \ + make_server.wizard._build_entries( + ["voxcpm2"], + {"voxcpm2": [{"target_directory": "VoxCPM2-GGUF", + "install_id": "voxcpm2_q8_0", + "design": False, "recommended": True}]}, + {"voxcpm2": catalog_entry}, + lambda install_id: "tts") + self.assertEqual(companions, []) diff --git a/app/tests/test_tts.py b/app/tests/test_tts.py index 6c245e5..45e3812 100644 --- a/app/tests/test_tts.py +++ b/app/tests/test_tts.py @@ -1,7 +1,9 @@ """Tests for the TTS client wrappers (language handling and payloads).""" +import base64 import io import json +import struct import tempfile import time import urllib.error @@ -45,6 +47,9 @@ from converter.clients import ( audiocpp_family_voice_policy, audiocpp_request_error, audiocpp_script_input, + allocation_log_note, + build_trimmed_voice_reference, + nvidia_device_memory_report, normalize_language, transcribe_reference_audio_detailed, whisper_backend_problem, @@ -1192,6 +1197,15 @@ class AudioCppFamilyVoicePolicyTests(unittest.TestCase): self.assertEqual(audiocpp_family_voice_policy("qwen3_tts"), AUDIOCPP_VOICE_REQUIRED) + def test_vevo2_is_required_despite_its_spec(self): + # Vevo2's spec lists tts/vc/svc but no "clone" task, yet its + # zero-shot TTS route refuses every request without a timbre + # reference: the explicit required set mirrors the server, so an + # "All" run sends the picked voice instead of failing every + # request with no voice at all. + self.assertEqual(audiocpp_family_voice_policy("vevo2"), + AUDIOCPP_VOICE_REQUIRED) + def test_unknown_family_keeps_the_conservative_default(self): self.assertEqual(audiocpp_family_voice_policy("brand_new_family"), AUDIOCPP_VOICE_REQUIRED) @@ -1240,7 +1254,8 @@ class AudioCppPlainTtsModeTests(unittest.TestCase): # Minimal WAV: _request_wav only validates the RIFF/WAVE header. _WAV = b"RIFF\x04\x00\x00\x00WAVE" - def _client(self, family, task="tts", voice=None, captured=None): + def _client(self, family, task="tts", voice=None, captured=None, + instructions=None): def _dispatch(request, **_kwargs): url = request if isinstance(request, str) else request.full_url if url.endswith("/health"): @@ -1269,7 +1284,8 @@ class AudioCppPlainTtsModeTests(unittest.TestCase): patcher.start() self.addCleanup(patcher.stop) return AudioCppTTSClient(_DUMMY_CHUNKS, voice=voice, - model_id="model") + model_id="model", + instructions=instructions) def test_pure_tts_family_connects_in_plain_mode(self): client = self._client("supertonic") @@ -1317,6 +1333,30 @@ class AudioCppPlainTtsModeTests(unittest.TestCase): self.assertIn("--voice", message) self.assertIn("voice_preset", message) + def test_clone_only_instructions_alone_do_not_define_the_voice(self): + # The REQUIRED-policy refusal precedes the instruction-voice + # branch: clone-only (and Vevo2-style) families cannot take their + # voice from an instruction, so a voice-less run fails fast with + # the --voice fix instead of 500ing every request server-side. + with self.assertRaises(RuntimeError) as ctx: + self._client("chatterbox", task="clon", + instructions="Calm and steady.") + message = str(ctx.exception) + self.assertIn("--voice", message) + self.assertNotIn("instruction", message) + + def test_vevo2_without_voice_refuses_at_connect(self): + with self.assertRaises(RuntimeError) as ctx: + self._client("vevo2") + message = str(ctx.exception) + self.assertIn("--voice", message) + self.assertIn("vevo2", message) + + def test_vevo2_with_voice_connects_in_preset_mode(self): + client = self._client("vevo2", voice="narrator") + self.assertTrue(client.preset_mode) + self.assertFalse(client.plain_mode) + class AudioCppCloneOnlyErrorTests(unittest.TestCase): """The non-retryable classification of clone-only hosting 500s.""" @@ -1404,15 +1444,54 @@ class AudioCppDeterministicErrorTests(unittest.TestCase): self.assertIsInstance(exc, NonRetryableTTSError) self.assertIn("trim", str(exc)) - def test_allocation_failures_are_not_retryable(self): + def test_allocation_failures_are_not_retryable_with_a_hint(self): # VRAM does not change between attempts of a sequential run (the # "All" loop unloads models between books, not between retries). - self.assertIsInstance( - self._error("DramaBox vocoder backend buffer allocation failed"), - NonRetryableTTSError) - self.assertIsInstance( - self._error("failed to allocate MOSS codec encoder forward graph"), - NonRetryableTTSError) + # The hint names the server log (which records the exact attempted + # allocation size) and the DramaBox mem_saver session option. + for message in ("DramaBox vocoder backend buffer allocation failed", + "failed to allocate MOSS codec encoder forward graph"): + exc = self._error(message) + self.assertIsInstance(exc, NonRetryableTTSError) + self.assertIn("audiocpp-server.log", str(exc)) + self.assertIn("dramabox.mem_saver", str(exc)) + + def test_missing_companion_hint_names_the_configure_fix(self): + exc = self._error( + "model path does not exist: /tmp/audiocpp-gguf/MioCodec-25Hz" + "-44.1kHz-v2") + self.assertIn("companion package", str(exc)) + self.assertIn("Configure Backends", str(exc)) + + def test_stale_package_layout_hint_names_the_re_download(self): + exc = self._error("missing model package file 'tokenizer_merges'") + self.assertIn("Configure Backends", str(exc)) + self.assertIn("re-downloaded", str(exc)) + + def test_multi_gguf_directory_hint_names_the_hosting_fix(self): + exc = self._error("model directory contains 4 GGUF files: /m") + self.assertIn("several GGUFs", str(exc)) + self.assertIn("Configure Backends", str(exc)) + + def test_sample_capacity_hint_names_the_capacity_override(self): + exc = self._error("VoxCPM2 AudioVAE encoder sample capacity exceeded") + self.assertIn("encoder-sample capacity", str(exc)) + self.assertIn("Configure Backends", str(exc)) + + def test_allocation_log_note_is_appended_to_the_error(self): + exc = audiocpp_request_error( + 500, json.dumps({"error": {"message": + "DramaBox audio VAE backend buffer allocation failed"}}), + log_note=" The server's log (/x) records the failed allocation " + "as: allocating 12.5 MiB on device 0") + self.assertIn("allocating 12.5 MiB on device 0", str(exc)) + + def test_log_note_is_not_appended_to_unrelated_errors(self): + exc = audiocpp_request_error( + 500, json.dumps({"error": {"message": "model busy"}}), + log_note=" The server's log (/x) records the failed allocation " + "as: allocating 12.5 MiB on device 0") + self.assertNotIn("allocating 12.5 MiB", str(exc)) def test_max_tokens_before_eoc_stays_retryable(self): # Proven transient: a request that hit it has succeeded on retry. @@ -2467,3 +2546,285 @@ class BackendWiringTests(unittest.TestCase): if __name__ == "__main__": unittest.main() + + +class TrimmedVoiceReferenceTests(unittest.TestCase): + """build_trimmed_voice_reference: a bounded inline cloning reference.""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.dir = Path(self._tmp.name) + + def tearDown(self): + self._tmp.cleanup() + + @staticmethod + def _wav_bytes(rate, channels, sampwidth, frames, fill): + if sampwidth == 1: + payload = bytes(fill & 0xFF for _ in range(frames * channels)) + else: + payload = fill.to_bytes(sampwidth, "little", signed=True) \ + * frames * channels + return (b"RIFF" + struct.pack("