diff options
Diffstat (limited to 'app/backends/audiocpp/models.py')
| -rw-r--r-- | app/backends/audiocpp/models.py | 297 |
1 files changed, 229 insertions, 68 deletions
diff --git a/app/backends/audiocpp/models.py b/app/backends/audiocpp/models.py index d81dcfc..16c61e9 100644 --- a/app/backends/audiocpp/models.py +++ b/app/backends/audiocpp/models.py @@ -16,6 +16,11 @@ from . import catalog as _catalog # runner kills it and reports exit 124 (see run_console_subprocess). DOWNLOAD_STALL_TIMEOUT = 300 +# Spec sanitizing lives with the catalog (the wizard's catalog view applies +# the same in-memory repair; the download path materializes it through its +# sanitized specs copy). The alias keeps this module's historical name. +_sanitize_model_spec = _catalog.sanitize_model_spec + def _installed_display_names(audiocpp_dir: Path, model_entries: Optional[List[dict]], install_guidance: List[Tuple[str, str]] @@ -25,19 +30,19 @@ def _installed_display_names(audiocpp_dir: Path, MODEL_ENTRIES and INSTALL_GUIDANCE are built in lockstep by ``_build_entries`` (one guidance pair per entry), so the pairs resolve positionally: each entry's ``path`` is checked against the checkout - exactly like ``_all_models_present`` resolves it. Returns an empty set + file-precisely (see ``_entry_present``). Returns an empty set when ENTRIES is None or does not line up with the guidance (no filtering — every model counts as not installed). """ if model_entries is None or len(model_entries) != len(install_guidance): return set() + packages_by_dir = _catalog_packages_by_dir(audiocpp_dir) installed: Set[str] = set() for entry, (name, _install_id) in zip(model_entries, install_guidance): rel = entry.get("path") if not isinstance(rel, str) or not rel: continue - path = Path(rel) if Path(rel).is_absolute() else audiocpp_dir / rel - if _model_path_present(path): + if _entry_present(audiocpp_dir, rel, packages_by_dir): installed.add(name) return installed @@ -67,10 +72,40 @@ def _split_pending_and_installed( return pending, noted +def _merge_companions(audiocpp_dir: Path, + pending: List[Tuple[str, str]], + companions: Optional[List[Tuple[str, str]]] + ) -> List[Tuple[str, str]]: + """Fold COMPANIONS into PENDING, skipping ones already on disk. + + Each companion is a (display name, install id) pair for a package a + hosted model requires but the TTS catalog never offers (MioCodec for + MioTTS). Companions already pending (same install id) or already + installed (their package's files present, checked file-precisely) are + dropped; the installed ones are reported so the user knows the + requirement is satisfied. Companion ids with no matching spec package + are kept (the install command will report what is wrong). + """ + merged = list(pending) + seen = {install_id for _name, install_id in pending} + for name, install_id in companions or []: + if install_id in seen: + continue + package = _companion_package(audiocpp_dir, install_id) + if package is not None \ + and _package_files_present(audiocpp_dir, package): + print(f"[OK] {name} is already installed.") + continue + seen.add(install_id) + merged.append((name, install_id)) + return merged + + def _install_models(audiocpp_dir: Path, install_guidance: List[Tuple[str, str]], download: bool, emit=None, cancel=None, - model_entries: Optional[List[dict]] = None) -> int: + model_entries: Optional[List[dict]] = None, + companions: Optional[List[Tuple[str, str]]] = None) -> int: """Report and optionally run the model install commands. When MODEL_ENTRIES (built in lockstep with INSTALL_GUIDANCE by @@ -78,11 +113,16 @@ def _install_models(audiocpp_dir: Path, installed and never re-downloaded or printed as commands; when every selected model is present nothing runs at all. The remaining models get one ``python <manager> install <id>`` command each (de-duped by - install id). When DOWNLOAD is True each command is run in the audio.cpp - checkout via ``subprocess`` so the models are downloaded automatically; - a failing install is reported as a warning and does not abort the - remaining downloads. When DOWNLOAD is False (or the model manager is - missing) the commands are only printed after a note that setup downloads + install id). COMPANIONS carries (display name, install id) pairs for + packages a hosted model needs but the TTS catalog does not offer + (MioCodec for MioTTS): they join the pending list, de-duped against + it and skipped when their package's files are already on disk — so a + configure run that only adds the companion still downloads it. When + DOWNLOAD is True each command is run in the audio.cpp checkout via + ``subprocess`` so the models are downloaded automatically; a failing + install is reported as a warning and does not abort the remaining + downloads. When DOWNLOAD is False (or the model manager is missing) + the commands are only printed after a note that setup downloads them automatically — copy-pasteable for a manual install. With EMIT given (the in-TUI task view) each download streams its output @@ -91,9 +131,10 @@ def _install_models(audiocpp_dir: Path, progress bar and cancel gracefully. CANCEL aborts a running download. When the checkout's model specs carry a broken ``strip_prefix`` (a dot - prefix over repo-root files — the manager rejects every file in such a - package) and the manager supports ``--specs-dir``, the installs run - against a sanitized temporary copy of the specs (see + prefix over repo-root files, or a single-GGUF package nested under a + repository directory with no prefix — the manager installs files the + server cannot load) and the manager supports ``--specs-dir``, the + installs run against a sanitized temporary copy of the specs (see ``_prepare_specs_dir``); the checkout itself is left untouched. Specs that cannot be repaired confidently are left as-is: those installs fail, are reported as warnings, and the remaining downloads continue. @@ -101,13 +142,14 @@ def _install_models(audiocpp_dir: Path, Returns 0 when every command succeeded (or nothing needed running), 130 when cancelled, 1 when any download failed. """ - if not install_guidance: + if not install_guidance and not companions: return 0 manager = audiocpp_dir / "tools" / "model_manager_v2.py" installed_names = _installed_display_names( audiocpp_dir, model_entries, install_guidance) pending, installed_noted = _split_pending_and_installed( install_guidance, installed_names) + pending = _merge_companions(audiocpp_dir, pending, companions) supports_progress = emit is not None and _manager_supports_progress(manager) @@ -295,18 +337,29 @@ def _prepare_specs_dir(audiocpp_dir: Path) -> Optional[Path]: return staging -def download_applicable(audiocpp_dir: Path, model_entries: List[dict]) -> bool: +def download_applicable(audiocpp_dir: Path, model_entries: List[dict], + companions: Optional[List[Tuple[str, str]]] = None + ) -> bool: """True when the wizard's "download models automatically?" row applies. The audio.cpp model manager must be present (otherwise the install commands can only be printed), and at least one selected model must be - missing from disk (see ``_all_models_present``), so an already-configured - checkout is not asked to re-download models it already has. + missing from disk (see ``_all_models_present``), or one COMPANION + package a hosted model needs (MioCodec for MioTTS) must be — so an + already-configured checkout is not asked to re-download models it + already has. """ manager = audiocpp_dir / "tools" / "model_manager_v2.py" if not manager.is_file(): return False - return not _all_models_present(audiocpp_dir, model_entries) + if not _all_models_present(audiocpp_dir, model_entries): + return True + for _name, install_id in companions or []: + package = _companion_package(audiocpp_dir, install_id) + if package is None \ + or not _package_files_present(audiocpp_dir, package): + return True + return False def _build_tree_families(catalog: List[dict]) -> List[dict]: @@ -357,33 +410,146 @@ def _model_path_present(path: Path) -> bool: return False +def _package_files_present(audiocpp_dir: Path, package: dict) -> bool: + """True when PACKAGE's files are on disk at their strip_prefix-stripped paths. + + Mirrors model_manager_v2.py's own ``package_is_installed`` check, which + is what decides a re-install. Used instead of the plain "directory is + non-empty" check so a stale install from a broken spec layout (e.g. the + GLM/OuteTTS GGUFs nested under ``Text to audio (TTS)/`` before the spec + sanitizer existed) counts as missing and gets re-downloaded correctly. + """ + files = package.get("files") or [] + if not files: + return False + prefix = str(package.get("strip_prefix") or "").rstrip("/") + base = audiocpp_dir / "models" / str(package.get("target_directory") or "") + for item in files: + if not isinstance(item, str): + return False + local = item + if prefix: + if not local.startswith(prefix + "/"): + return False + local = local[len(prefix) + 1:] + try: + if not (base / local).is_file(): + return False + except OSError: + return False + return True + + +def _entry_directory_key(rel: str) -> str: + """The ``<target_directory>`` a server.json model path belongs to. + + ``models/<dir>`` entries map to ``<dir>``; entries hosted from a file + inside a package directory (``models/<dir>/<gguf>``, the multi-GGUF + hosting convention) map to ``<dir>`` as well, so old directory-style + and new file-style entries of the same package compare equal. + """ + stripped = rel[len("models/"):] if rel.startswith("models/") else rel + return stripped.split("/", 1)[0] + + +def _catalog_packages_by_dir(audiocpp_dir: Path) -> Dict[str, List[dict]]: + """Map each catalog package target directory to its packages. + + The catalog's packages carry the sanitized strip prefixes (see + ``load_model_catalog``), so presence checks see the same layout the + model manager will install. Families whose specs cannot be read yield + no entries and callers fall back to the plain path check. + """ + try: + entries = _catalog.load_model_catalog(audiocpp_dir) + except (NotADirectoryError, OSError): + return {} + by_dir: Dict[str, List[dict]] = {} + for entry in entries: + for package in entry.get("packages") or []: + if not isinstance(package, dict): + continue + directory = str(package.get("target_directory") or entry["family"]) + by_dir.setdefault(directory, []).append(package) + return by_dir + + +def _companion_package(audiocpp_dir: Path, install_id: str) -> Optional[dict]: + """The spec package with INSTALL_ID across every model spec. + + Companion packages live in specs the TTS catalog filters out + (miocodec is an audio_tools family with no text-synthesis task), so + the lookup scans all model_specs/*.json with the sanitizer applied, + matching the sanitized layout the download will use. + """ + try: + spec_paths = sorted((audiocpp_dir / "model_specs").glob("*.json")) + except OSError: + return None + for spec_path in spec_paths: + try: + spec = json.loads(spec_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + continue + if not isinstance(spec, dict): + continue + _catalog.sanitize_model_spec(spec) + for package in spec.get("packages") or []: + if isinstance(package, dict) \ + and str(package.get("id") or "") == install_id: + return package + return None + + +def _entry_present(audiocpp_dir: Path, rel: str, + packages_by_dir: Dict[str, List[dict]]) -> bool: + """Whether the model entry path REL holds its package's files. + + When REL's directory matches a catalog package, the check is + file-precise (every package file at its stripped path). Otherwise the + plain path check applies: the entry may host a package the local + specs do not describe (an older checkout, a custom path), and "the + configured path exists" is then the best available signal. + """ + directory = _entry_directory_key(rel) + candidates = packages_by_dir.get(directory) + if candidates is not None: + return any(_package_files_present(audiocpp_dir, package) + for package in candidates) + path = Path(rel) if Path(rel).is_absolute() else audiocpp_dir / rel + return _model_path_present(path) + + def _all_models_present(audiocpp_dir: Path, model_entries: List[dict]) -> bool: - """True when every selected model entry's path already holds files on disk. + """True when every selected model entry's files are on disk. - Paths resolve against the checkout (where model_manager_v2.py installs - them), honoring absolute paths. Used by the wizard to skip the - "Automatically download the selected models" prompt when nothing is - actually missing. An empty selection is treated as not-present. + Presence is file-precise against the catalog packages (see + ``_entry_present``): a directory that merely exists — e.g. holding a + stale install from a since-repaired spec layout — counts as missing so + the next download replaces it with the correct layout. An empty + selection is treated as not-present. """ if not model_entries: return False + packages_by_dir = _catalog_packages_by_dir(audiocpp_dir) for entry in model_entries: rel = entry.get("path") if not isinstance(rel, str) or not rel: return False - path = Path(rel) if Path(rel).is_absolute() else audiocpp_dir / rel - if not _model_path_present(path): + if not _entry_present(audiocpp_dir, rel, packages_by_dir): return False return True -def missing_model_entries(server_json: Path) -> List[dict]: - """Return the server.json model entries whose files are not on disk. +def _server_entries_by_presence(server_json: Path, present: bool) -> List[dict]: + """The server.json model entries whose on-disk presence matches PRESENT. - Paths resolve exactly like audiocpp_server resolves them (relative paths - against the server.json's directory). Each returned entry carries the - entry ``id`` and ``rel`` (the configured path string); used by ``detect`` - to warn that a conversion would fail until the models are installed. + Presence is file-precise against the catalog packages when the + server.json's directory hosts the model_specs (the usual checkout + layout); otherwise the plain path check applies. Each returned entry + carries the entry ``id`` and ``rel`` (the configured path string), + resolved exactly like the server resolves them (relative against the + server.json's directory; absolute paths honored). """ try: data = json.loads(server_json.read_text(encoding="utf-8")) @@ -392,18 +558,37 @@ def missing_model_entries(server_json: Path) -> List[dict]: if not isinstance(data, dict): return [] base = server_json.parent - missing: List[dict] = [] + packages_by_dir = _catalog_packages_by_dir(base) + matching: List[dict] = [] for entry in data.get("models") or []: if not isinstance(entry, dict): continue rel = entry.get("path") if not isinstance(rel, str) or not rel: continue - path = Path(rel) if Path(rel).is_absolute() else base / rel - if _model_path_present(path): + if _entry_present(base, rel, packages_by_dir) != present: continue - missing.append({"id": str(entry.get("id") or rel), "rel": rel}) - return missing + matching.append({"id": str(entry.get("id") or rel), "rel": rel}) + return matching + + +def missing_model_entries(server_json: Path) -> List[dict]: + """Return the server.json model entries whose files are not on disk. + + Used by ``detect`` to warn that a conversion would fail until the + models are installed (see ``_server_entries_by_presence``). + """ + return _server_entries_by_presence(server_json, present=False) + + +def installed_model_entries(server_json: Path) -> List[dict]: + """Return the server.json model entries whose files ARE on disk. + + The complement of ``missing_model_entries`` (see + ``_server_entries_by_presence``). Used by the wizard's "Delete unused + models?" step to find already-downloaded models that were unselected. + """ + return _server_entries_by_presence(server_json, present=True) def _install_id_by_path(audiocpp_dir: Path) -> Dict[str, str]: @@ -422,35 +607,6 @@ def _install_id_by_path(audiocpp_dir: Path) -> Dict[str, str]: return by_path -def installed_model_entries(server_json: Path) -> List[dict]: - """Return the server.json model entries whose files ARE on disk. - - The complement of ``missing_model_entries``: each returned entry carries - the entry ``id`` and ``rel`` (the configured path string), resolved - exactly like ``missing_model_entries`` (relative against the server.json's - directory). Used by the wizard's "Delete unused models?" step to find - already-downloaded models that were unselected. - """ - try: - data = json.loads(server_json.read_text(encoding="utf-8")) - except (OSError, ValueError): - return [] - if not isinstance(data, dict): - return [] - base = server_json.parent - installed: List[dict] = [] - for entry in data.get("models") or []: - if not isinstance(entry, dict): - continue - rel = entry.get("path") - if not isinstance(rel, str) or not rel: - continue - path = Path(rel) if Path(rel).is_absolute() else base / rel - if _model_path_present(path): - installed.append({"id": str(entry.get("id") or rel), "rel": rel}) - return installed - - def missing_model_install_guidance(audiocpp_dir: Path, missing: List[dict]) -> List[Tuple[str, str]]: """Map MISSING model entries to (display name, install id) pairs. @@ -530,15 +686,20 @@ def hand_install_guidance(audiocpp_dir: Path, def unused_installed_entries(server_json: Path, new_paths: Set[str]) -> List[dict]: - """Return installed server.json entries whose path is not in NEW_PATHS. + """Return installed server.json entries whose package is not in NEW_PATHS. The already-downloaded models (see ``installed_model_entries``) that the new selection does not host any more — the candidates for the wizard's - "Delete unused models?" prompt. Entries whose files are not on disk are - never listed (there is nothing to delete). + "Delete unused models?" prompt. Entries are compared by their package + directory key (``_entry_directory_key``), so an entry re-hosted from + ``models/<dir>`` to ``models/<dir>/<gguf>`` (the multi-GGUF convention) + is not offered for deletion when the new config still hosts that + directory. Entries whose files are not on disk are never listed (there + is nothing to delete). """ + new_keys = {_entry_directory_key(path) for path in new_paths} return [entry for entry in installed_model_entries(server_json) - if entry["rel"] not in new_paths] + if _entry_directory_key(entry["rel"]) not in new_keys] def delete_model_files(server_json: Path, entries: List[dict]) -> int: |