diff options
Diffstat (limited to 'app/backends/audiocpp/models.py')
| -rw-r--r-- | app/backends/audiocpp/models.py | 194 |
1 files changed, 157 insertions, 37 deletions
diff --git a/app/backends/audiocpp/models.py b/app/backends/audiocpp/models.py index 772ae8d..f2c4e5e 100644 --- a/app/backends/audiocpp/models.py +++ b/app/backends/audiocpp/models.py @@ -90,6 +90,14 @@ def _install_models(audiocpp_dir: Path, runs with ``--progress --cancel-file`` so the view can show a real byte progress bar and cancel gracefully. CANCEL aborts a running download. + When the checkout's model specs carry a broken ``strip_prefix`` (a dot + prefix over repo-root files — the manager rejects every file in such a + package) and the manager supports ``--specs-dir``, the installs run + against a sanitized temporary copy of the specs (see + ``_prepare_specs_dir``); the checkout itself is left untouched. Specs + that cannot be repaired confidently are left as-is: those installs + fail, are reported as warnings, and the remaining downloads continue. + Returns 0 when every command succeeded (or nothing needed running), 130 when cancelled, 1 when any download failed. """ @@ -122,45 +130,78 @@ def _install_models(audiocpp_dir: Path, return 0 failed = False - for _, install_id in pending: - print(f"[INFO] Downloading {install_id}...") - argv = [sys.executable, str(manager), "install", install_id] - cancel_file: Optional[Path] = None - on_cancel = None - if supports_progress: - fd, cancel_path = tempfile.mkstemp( - prefix="audiocpp_cancel_", suffix=".cancel") - os.close(fd) - cancel_file = Path(cancel_path) - cancel_file.unlink() # absent = not cancelled - argv += ["--progress", "--cancel-file", str(cancel_file)] - on_cancel = cancel_file.touch + specs_dir: Optional[Path] = None + if _manager_supports_flag(manager, "--specs-dir"): try: - rc = common.run_console_subprocess( - argv, cwd=str(audiocpp_dir), emit=emit, cancel=cancel, - on_cancel=on_cancel, - stall_timeout=(DOWNLOAD_STALL_TIMEOUT - if supports_progress else None)) + specs_dir = _prepare_specs_dir(audiocpp_dir) except OSError as exc: - print(f"[WARNING] Could not run python {manager} install " - f"{install_id}: {exc}") - rc = 1 - finally: - if cancel_file is not None: - try: - cancel_file.unlink() - except OSError: - pass - if rc == 130 or (cancel is not None and cancel.is_set()): - return 130 - if rc != 0: - failed = True - print(f"[WARNING] install {install_id} exited with code " - f"{rc}; the model may need to be downloaded " - "by hand") + print(f"[WARNING] Could not prepare sanitized model specs: {exc}") + specs_dir = None + if specs_dir is not None: + print(f"[INFO] The checkout's model specs carry a broken " + "strip_prefix; installing from a sanitized copy " + f"({specs_dir})") + specs_args = (["--specs-dir", str(specs_dir)] + if specs_dir is not None else []) + try: + for _, install_id in pending: + print(f"[INFO] Downloading {install_id}...") + argv = [sys.executable, str(manager)] + specs_args + [ + "install", install_id] + cancel_file: Optional[Path] = None + on_cancel = None + if supports_progress: + fd, cancel_path = tempfile.mkstemp( + prefix="audiocpp_cancel_", suffix=".cancel") + os.close(fd) + cancel_file = Path(cancel_path) + cancel_file.unlink() # absent = not cancelled + argv += ["--progress", "--cancel-file", str(cancel_file)] + on_cancel = cancel_file.touch + try: + rc = common.run_console_subprocess( + argv, cwd=str(audiocpp_dir), emit=emit, cancel=cancel, + on_cancel=on_cancel, + stall_timeout=(DOWNLOAD_STALL_TIMEOUT + if supports_progress else None)) + except OSError as exc: + print(f"[WARNING] Could not run python {manager} install " + f"{install_id}: {exc}") + rc = 1 + finally: + if cancel_file is not None: + try: + cancel_file.unlink() + except OSError: + pass + if rc == 130 or (cancel is not None and cancel.is_set()): + return 130 + if rc != 0: + failed = True + print(f"[WARNING] install {install_id} exited with code " + f"{rc}; the model may need to be downloaded " + "by hand") + finally: + if specs_dir is not None: + shutil.rmtree(specs_dir, ignore_errors=True) return 1 if failed else 0 +def _manager_supports_flag(manager: Path, flag: str) -> bool: + """True when MANAGER's (model_manager_v2.py's) source mentions FLAG. + + The checkout is downloaded, so an older copy may lack a relatively + recent flag; probing the script source once is cheaper than failing an + install with an unknown option. A false positive (the string appears + outside argparse) is caught when the subprocess reports the error. + """ + try: + text = manager.read_text(encoding="utf-8", errors="ignore") + except OSError: + return False + return flag in text + + def _manager_supports_progress(manager: Path) -> bool: """True when MANAGER (model_manager_v2.py) supports --progress output. @@ -168,11 +209,90 @@ def _manager_supports_progress(manager: Path) -> bool: older audio.cpp checkout may not have them, so probe the script source once instead of failing the download with an unknown flag. """ + return (_manager_supports_flag(manager, "AUDIOCPP_PROGRESS") + and _manager_supports_flag(manager, "--cancel-file")) + + +def _sanitize_model_spec(spec: dict) -> bool: + """Repair dot ``strip_prefix`` packages in SPEC, in place. + + A package's ``strip_prefix`` is stripped from the front of every file + path to get the local layout, so it only works when every file is + listed under that prefix (``<prefix>/<file>``). A dot prefix ("." or + "./") is meant for files written ``./<file>``; when the package instead + lists repo-root files bare (``model.gguf``), the manager rejects the + whole package ("file path does not start with strip_prefix '.': ...") + and nothing can be downloaded. Root-level files need no prefix at all + (upstream specs like minimax_music3.json store ""), so dropping the dot + prefix is the safe repair. Prefixes naming a real directory are left + alone — the correct remote paths cannot be guessed. Returns True when + SPEC changed. + """ + changed = False + for package in spec.get("packages") or []: + if not isinstance(package, dict): + continue + prefix = str(package.get("strip_prefix") or "").rstrip("/") + if prefix not in (".", ".."): + continue + files = package.get("files") + if not isinstance(files, list) or not files: + continue + if all(isinstance(item, str) and item.startswith(prefix + "/") + for item in files): + continue + package["strip_prefix"] = "" + changed = True + return changed + + +def _prepare_specs_dir(audiocpp_dir: Path) -> Optional[Path]: + """Return a temp specs dir with dot ``strip_prefix`` entries repaired. + + audio.cpp's model manager accepts ``--specs-dir``, so a checkout whose + specs carry a broken ``strip_prefix`` can be installed from a sanitized + copy without modifying the checkout. Every ``model_specs/*.json`` is + copied; the ones needing a repair are rewritten via + ``_sanitize_model_spec`` (specs that fail to parse are copied verbatim + so the manager reports them exactly as it would upstream). Returns None + when no spec needed a repair (or the specs directory is missing or + unreadable) — the caller then runs against the checkout's own specs. + The caller owns the returned directory and removes it when the installs + are done. + """ + specs_dir = audiocpp_dir / "model_specs" try: - text = manager.read_text(encoding="utf-8", errors="ignore") + spec_paths = sorted(specs_dir.glob("*.json")) except OSError: - return False - return "AUDIOCPP_PROGRESS" in text and "--cancel-file" in text + return None + if not spec_paths: + return None + payloads: List[Tuple[str, str]] = [] + sanitized = False + for spec_path in spec_paths: + try: + text = spec_path.read_text(encoding="utf-8") + except OSError: + return None + try: + spec = json.loads(text) + except ValueError: + payloads.append((spec_path.name, text)) + continue + if isinstance(spec, dict) and _sanitize_model_spec(spec): + sanitized = True + text = json.dumps(spec, indent=2, ensure_ascii=False) + "\n" + payloads.append((spec_path.name, text)) + if not sanitized: + return None + staging = Path(tempfile.mkdtemp(prefix="audiocpp_specs_")) + try: + for name, payload in payloads: + (staging / name).write_text(payload, encoding="utf-8") + except OSError: + shutil.rmtree(staging, ignore_errors=True) + raise + return staging def download_applicable(audiocpp_dir: Path, model_entries: List[dict]) -> bool: |