diff options
Diffstat (limited to 'app/backends')
| -rw-r--r-- | app/backends/audiocpp/build.py | 130 | ||||
| -rw-r--r-- | app/backends/audiocpp/models.py | 9 | ||||
| -rw-r--r-- | app/backends/audiocpp/wizard.py | 6 | ||||
| -rw-r--r-- | app/backends/common.py | 46 |
4 files changed, 181 insertions, 10 deletions
diff --git a/app/backends/audiocpp/build.py b/app/backends/audiocpp/build.py index 22c72a1..9ffe429 100644 --- a/app/backends/audiocpp/build.py +++ b/app/backends/audiocpp/build.py @@ -2,6 +2,7 @@ import contextlib import io +import os import re import shlex import shutil @@ -260,6 +261,119 @@ GGML_PATCHES = [ }, ] +# No-output watchdog for the build: ninja prints a line per completed +# compile, so 15 minutes of silence means a compiler job wedged (ptxas +# hangs are the known failure mode of a buggy CUDA toolkit). The runner +# kills the build and reports exit 124 (see run_console_subprocess). +BUILD_STALL_TIMEOUT = 900 + +# Env override for the CUDA architectures passed to build_linux.sh +# (--cuda-arch): a ';'-or-comma separated list of compute capabilities, +# e.g. "86" or "86;89". Set it when GPU detection cannot run. +CUDA_ARCH_ENV = "AUDIOCPP_CUDA_ARCH" + +# Compute capability -> common GPUs, shown when detection is impossible +# and in ptxas failure guidance. Terse on purpose; one line. +CUDA_ARCH_GUIDE = ("61 GTX 10xx/P40; 75 RTX 20xx; 80 A100; 86 RTX 30xx " + "(3090)/A6000; 89 RTX 40xx (4090)/L40S; 90 H100; " + "120/121 RTX 50xx (5090)/B200") + + +def detect_cuda_arch() -> Optional[str]: + """The CUDA architecture token to build for, or None when unknown. + + ``AUDIOCPP_CUDA_ARCH`` wins verbatim (validated as a ';'-or-comma + separated list of compute capabilities like ``86`` or ``86;89``), so a + user can pin the arch on machines where detection cannot run. Otherwise + ``nvidia-smi`` reports each GPU's compute capability (works on Linux + and Windows; it does not exist on macOS, where the CUDA backend is not + a choice anyway): ``8.6`` becomes ``86``, several distinct GPUs join + as ``86;89``. audio.cpp's CMake upgrades bare new architectures + (``120``) to their suffixed forms (``120a``) itself. + """ + override = os.environ.get(CUDA_ARCH_ENV, "").strip() + if override: + parts = [part.strip() for part in + override.replace(",", ";").split(";") if part.strip()] + if parts and all(re.fullmatch(r"\d+(-real|-virtual)?", part) + for part in parts): + return ";".join(parts) + print(f"[WARNING] {CUDA_ARCH_ENV}={override!r} is not an arch list " + "(e.g. \"86\" or \"86;89\"); ignoring it") + proc = common.run_console_subprocess_quiet( + ["nvidia-smi", "--query-gpu=compute_cap", "--format=csv,noheader"], + timeout=10) + if proc is None or proc.returncode != 0: + return None + arches: List[str] = [] + for line in proc.stdout.decode("utf-8", errors="replace").splitlines(): + cap = line.strip() + if not re.fullmatch(r"\d+\.\d+", cap): + continue + arch = cap.replace(".", "") + if arch not in arches: + arches.append(arch) + return ";".join(arches) if arches else None + + +def _cuda_arch_argv(backend: str, emit=None) -> List[str]: + """The ``--cuda-arch`` flags for a CUDA build, plus a status line. + + EMIT is the in-TUI line sink when building from the task view (the + line lands in the view, not the real terminal behind curses); without + it the line prints to the console. Detection failure is not an error: + the build then uses audio.cpp's portable default arch list, which is + slower to compile but runs on any GPU. + """ + if backend != "cuda": + return [] + arch = detect_cuda_arch() + say = emit if emit is not None else print + if arch is None: + say(f"[INFO] CUDA architecture: portable default list (could not " + f"detect a GPU; set {CUDA_ARCH_ENV}=<arch> to build only for " + "this machine's GPU — much faster)") + return [] + say(f"[INFO] CUDA architecture: {arch} (detected via nvidia-smi; " + f"override with {CUDA_ARCH_ENV})") + return ["--cuda-arch", arch] + + +def _ptxas_failure_hint(log_path: Path) -> str: + """Guidance appended when the build log shows a ptxas failure. + + ``ptxas fatal`` / ``nvcc error`` lines mean the CUDA toolkit's + assembler (or nvcc itself) failed — an internal compiler error is a + toolkit bug, not a broken checkout, and newer ggml template code on + newer toolkit releases trips it. Building only for the local GPU's + architecture skips most of the codegen paths ptxas chokes on, so the + hint points at ``AUDIOCPP_CUDA_ARCH`` (with the detected arch, or the + GPU table when detection cannot run); a different toolkit version is + the remaining fix when narrowing the arch is not enough. + """ + try: + text = log_path.read_text(encoding="utf-8", errors="ignore") + except OSError: + return "" + if "ptxas fatal" not in text and "nvcc error" not in text: + return "" + arch = detect_cuda_arch() + lines = [ + " ptxas (the CUDA toolkit's GPU assembler) failed — with an " + "internal compiler error this is a CUDA toolkit bug, not your " + "sources.", + f" Rebuild for this machine's GPU only: set {CUDA_ARCH_ENV}=<arch> " + "(semicolon-separated for several GPUs) and re-run the build.", + ] + if arch: + lines.append(f" Detected arch for this machine: {arch}") + else: + lines.append(f" Arch per GPU: {CUDA_ARCH_GUIDE}") + lines.append(" If narrowing the arch still fails, a different CUDA " + "toolkit version usually does (ptxas bugs are fixed in " + "toolkit updates).") + return "\n".join(lines) + def apply_ggml_patches(audiocpp_dir: Path, *, emit=None, cancel=None) -> int: """Apply the shipped ggml build patches to an audio.cpp checkout. @@ -348,6 +462,7 @@ def build_audiocpp(audiocpp_dir: Path, backend: str, *, return 1 argv = ["sh", str(script), "--backend", backend, "--target", "audiocpp_server", "--deployment-build"] + argv += _cuda_arch_argv(backend, emit=emit) command = f"cd {audiocpp_dir} && {shlex.join(argv)}" if emit is None: print(f"[INFO] Building audiocpp_server for {backend} ({command})...") @@ -409,12 +524,23 @@ def _build_audiocpp_tui(emit, cancel, argv: List[str], command: str, return patch_rc tee(f"[INFO] Building audiocpp_server ({command})...") rc = common.run_console_subprocess( - argv, cwd=audiocpp_dir, emit=tee, cancel=cancel) + argv, cwd=audiocpp_dir, emit=tee, cancel=cancel, + stall_timeout=BUILD_STALL_TIMEOUT) if rc != 0 and (cancel is None or not cancel.is_set()): - notice = (f"[ERROR] audio.cpp build failed (exit code {rc}).\n" + if rc == 124: + head = (f"[ERROR] audio.cpp build stalled — no output for " + f"{BUILD_STALL_TIMEOUT // 60} minutes, so it was " + "stopped (a wedged compiler job; often a ptxas " + "hang from a buggy CUDA toolkit).") + else: + head = f"[ERROR] audio.cpp build failed (exit code {rc})." + notice = (f"{head}\n" f" Build log: {log_path}\n" f" Troubleshoot by re-running this command:\n" f" {command}") + ptxas_hint = _ptxas_failure_hint(log_path) + if ptxas_hint: + notice += "\n" + ptxas_hint for line in notice.splitlines(): tee(line) common.record_post_tui_notice(notice) diff --git a/app/backends/audiocpp/models.py b/app/backends/audiocpp/models.py index 75bc06a..772ae8d 100644 --- a/app/backends/audiocpp/models.py +++ b/app/backends/audiocpp/models.py @@ -11,6 +11,11 @@ from typing import Dict, List, Optional, Set, Tuple from backends import common from . import catalog as _catalog +# No-output watchdog for model downloads: huggingface_hub streams steady +# byte progress, so 5 minutes of silence means the transfer wedged. The +# runner kills it and reports exit 124 (see run_console_subprocess). +DOWNLOAD_STALL_TIMEOUT = 300 + def _installed_display_names(audiocpp_dir: Path, model_entries: Optional[List[dict]], install_guidance: List[Tuple[str, str]] @@ -133,7 +138,9 @@ def _install_models(audiocpp_dir: Path, try: rc = common.run_console_subprocess( argv, cwd=str(audiocpp_dir), emit=emit, cancel=cancel, - on_cancel=on_cancel) + on_cancel=on_cancel, + stall_timeout=(DOWNLOAD_STALL_TIMEOUT + if supports_progress else None)) except OSError as exc: print(f"[WARNING] Could not run python {manager} install " f"{install_id}: {exc}") diff --git a/app/backends/audiocpp/wizard.py b/app/backends/audiocpp/wizard.py index 46b6638..c59da8f 100644 --- a/app/backends/audiocpp/wizard.py +++ b/app/backends/audiocpp/wizard.py @@ -574,7 +574,11 @@ def _execute_lanes(settings: dict, def build_step(emit, cancel): rc = _build.build_audiocpp(audiocpp_dir, settings["backend"], emit=emit, cancel=cancel) - if rc != 0: + if rc == 124: + print("[WARNING] build went silent and was stopped; the " + "server.json was still written — build " + "audiocpp_server manually before starting it") + elif rc != 0: print(f"[WARNING] build exited with code {rc}; the server.json " "was still written — build audiocpp_server manually " "before starting it") diff --git a/app/backends/common.py b/app/backends/common.py index 4edfd61..4c768b8 100644 --- a/app/backends/common.py +++ b/app/backends/common.py @@ -347,7 +347,8 @@ def write_prompt_text(wav_dir: Path, def run_console_subprocess(argv: List[str], cwd: Optional[Path] = None, - *, emit=None, cancel=None, on_cancel=None) -> int: + *, emit=None, cancel=None, on_cancel=None, + stall_timeout: Optional[float] = None) -> int: """Run a subprocess, streaming output to the console or to EMIT. With EMIT None the child inherits the real terminal and its output @@ -360,7 +361,16 @@ def run_console_subprocess(argv: List[str], cwd: Optional[Path] = None, CANCEL is an optional ``threading.Event``: once set, ON_CANCEL (if given) is called (e.g. to touch a ``--cancel-file``), then the child's process group is terminated (SIGTERM, escalating to SIGKILL after a grace - period) and 130 is returned. Returns the process exit code. + period) and 130 is returned. + + STALL_TIMEOUT (EMIT path only) is a no-output watchdog in seconds: when + the child produces no new output line for that long, it is treated as + wedged (a build whose compiler hung, a download that stopped moving) — + the process group is terminated, an [ERROR] line is emitted, and 124 is + returned so callers can report a stall distinctly from a plain failure. + None (the default) waits forever, as before. + + Returns the process exit code. """ import subprocess if emit is None: @@ -387,12 +397,18 @@ def run_console_subprocess(argv: List[str], cwd: Optional[Path] = None, return 1 cancelled = False + stalled = False + # Written by the reader thread, read by the poll loop below: a plain + # float assignment is atomic enough under the GIL (no torn reads). + last_output = time.monotonic() def _reader() -> None: + nonlocal last_output try: for raw in iter(proc.stdout.readline, b""): if not raw: break + last_output = time.monotonic() text = raw.decode("utf-8", errors="replace") for line in text.splitlines(): if line: @@ -421,6 +437,13 @@ def run_console_subprocess(argv: List[str], cwd: Optional[Path] = None, break if proc.poll() is not None: break + if (stall_timeout is not None + and time.monotonic() - last_output > stall_timeout): + stalled = True + emit(f"[ERROR] No output for {int(stall_timeout)}s — assuming " + "the process hung; stopping it.") + _terminate_process_group(proc) + break time.sleep(0.1) try: reader.join(timeout=5) @@ -429,6 +452,8 @@ def run_console_subprocess(argv: List[str], cwd: Optional[Path] = None, reader.join(timeout=0) if cancelled: return 130 + if stalled: + return 124 return proc.returncode @@ -577,20 +602,29 @@ def _origin_default_branch(checkout: Path) -> str: def run_console_subprocess_quiet(argv: List[str], - cwd: Optional[Path] = None): + cwd: Optional[Path] = None, + timeout: Optional[float] = None): """Run ARGV silently and return the completed result. Unlike run_console_subprocess (which streams or returns only an exit code) this captures stdout and needs the process object itself, for the small git probes (rev-parse, symbolic-ref) whose *output* matters and - whose failure is a normal, non-fatal outcome. Returns None when the - process could not be started. + whose failure is a normal, non-fatal outcome. TIMEOUT bounds the wait + (e.g. for hardware probes like nvidia-smi that can hang on a wedged + driver); a timeout kills the child and returns a failed result, not an + exception. Returns None when the process could not be started. """ import subprocess try: return subprocess.run( argv, capture_output=True, - cwd=str(cwd) if cwd is not None else None, check=False) + cwd=str(cwd) if cwd is not None else None, check=False, + timeout=timeout) + except subprocess.TimeoutExpired: + class _TimedOut: + returncode = -1 + stdout = b"" + return _TimedOut() except OSError: return None |