feat: cuda arch detection for shorter builds, build failure detection

3 files changed, 141 insertions, 4 deletions

diff --git a/app/backends/audiocpp/build.py b/app/backends/audiocpp/build.py
index 22c72a1..9ffe429 100644
--- a/app/backends/audiocpp/build.py
+++ b/app/backends/audiocpp/build.py
@@ -2,6 +2,7 @@
 
 import contextlib
 import io
+import os
 import re
 import shlex
 import shutil
@@ -260,6 +261,119 @@ GGML_PATCHES = [
     },
 ]
 
+# No-output watchdog for the build: ninja prints a line per completed
+# compile, so 15 minutes of silence means a compiler job wedged (ptxas
+# hangs are the known failure mode of a buggy CUDA toolkit). The runner
+# kills the build and reports exit 124 (see run_console_subprocess).
+BUILD_STALL_TIMEOUT = 900
+
+# Env override for the CUDA architectures passed to build_linux.sh
+# (--cuda-arch): a ';'-or-comma separated list of compute capabilities,
+# e.g. "86" or "86;89". Set it when GPU detection cannot run.
+CUDA_ARCH_ENV = "AUDIOCPP_CUDA_ARCH"
+
+# Compute capability -> common GPUs, shown when detection is impossible
+# and in ptxas failure guidance. Terse on purpose; one line.
+CUDA_ARCH_GUIDE = ("61 GTX 10xx/P40; 75 RTX 20xx; 80 A100; 86 RTX 30xx "
+                   "(3090)/A6000; 89 RTX 40xx (4090)/L40S; 90 H100; "
+                   "120/121 RTX 50xx (5090)/B200")
+
+
+def detect_cuda_arch() -> Optional[str]:
+    """The CUDA architecture token to build for, or None when unknown.
+
+    ``AUDIOCPP_CUDA_ARCH`` wins verbatim (validated as a ';'-or-comma
+    separated list of compute capabilities like ``86`` or ``86;89``), so a
+    user can pin the arch on machines where detection cannot run. Otherwise
+    ``nvidia-smi`` reports each GPU's compute capability (works on Linux
+    and Windows; it does not exist on macOS, where the CUDA backend is not
+    a choice anyway): ``8.6`` becomes ``86``, several distinct GPUs join
+    as ``86;89``. audio.cpp's CMake upgrades bare new architectures
+    (``120``) to their suffixed forms (``120a``) itself.
+    """
+    override = os.environ.get(CUDA_ARCH_ENV, "").strip()
+    if override:
+        parts = [part.strip() for part in
+                 override.replace(",", ";").split(";") if part.strip()]
+        if parts and all(re.fullmatch(r"\d+(-real|-virtual)?", part)
+                         for part in parts):
+            return ";".join(parts)
+        print(f"[WARNING] {CUDA_ARCH_ENV}={override!r} is not an arch list "
+              "(e.g. \"86\" or \"86;89\"); ignoring it")
+    proc = common.run_console_subprocess_quiet(
+        ["nvidia-smi", "--query-gpu=compute_cap", "--format=csv,noheader"],
+        timeout=10)
+    if proc is None or proc.returncode != 0:
+        return None
+    arches: List[str] = []
+    for line in proc.stdout.decode("utf-8", errors="replace").splitlines():
+        cap = line.strip()
+        if not re.fullmatch(r"\d+\.\d+", cap):
+            continue
+        arch = cap.replace(".", "")
+        if arch not in arches:
+            arches.append(arch)
+    return ";".join(arches) if arches else None
+
+
+def _cuda_arch_argv(backend: str, emit=None) -> List[str]:
+    """The ``--cuda-arch`` flags for a CUDA build, plus a status line.
+
+    EMIT is the in-TUI line sink when building from the task view (the
+    line lands in the view, not the real terminal behind curses); without
+    it the line prints to the console. Detection failure is not an error:
+    the build then uses audio.cpp's portable default arch list, which is
+    slower to compile but runs on any GPU.
+    """
+    if backend != "cuda":
+        return []
+    arch = detect_cuda_arch()
+    say = emit if emit is not None else print
+    if arch is None:
+        say(f"[INFO] CUDA architecture: portable default list (could not "
+            f"detect a GPU; set {CUDA_ARCH_ENV}=<arch> to build only for "
+            "this machine's GPU — much faster)")
+        return []
+    say(f"[INFO] CUDA architecture: {arch} (detected via nvidia-smi; "
+        f"override with {CUDA_ARCH_ENV})")
+    return ["--cuda-arch", arch]
+
+
+def _ptxas_failure_hint(log_path: Path) -> str:
+    """Guidance appended when the build log shows a ptxas failure.
+
+    ``ptxas fatal`` / ``nvcc error`` lines mean the CUDA toolkit's
+    assembler (or nvcc itself) failed — an internal compiler error is a
+    toolkit bug, not a broken checkout, and newer ggml template code on
+    newer toolkit releases trips it. Building only for the local GPU's
+    architecture skips most of the codegen paths ptxas chokes on, so the
+    hint points at ``AUDIOCPP_CUDA_ARCH`` (with the detected arch, or the
+    GPU table when detection cannot run); a different toolkit version is
+    the remaining fix when narrowing the arch is not enough.
+    """
+    try:
+        text = log_path.read_text(encoding="utf-8", errors="ignore")
+    except OSError:
+        return ""
+    if "ptxas fatal" not in text and "nvcc error" not in text:
+        return ""
+    arch = detect_cuda_arch()
+    lines = [
+        "  ptxas (the CUDA toolkit's GPU assembler) failed — with an "
+        "internal compiler error this is a CUDA toolkit bug, not your "
+        "sources.",
+        f"  Rebuild for this machine's GPU only: set {CUDA_ARCH_ENV}=<arch> "
+        "(semicolon-separated for several GPUs) and re-run the build.",
+    ]
+    if arch:
+        lines.append(f"  Detected arch for this machine: {arch}")
+    else:
+        lines.append(f"  Arch per GPU: {CUDA_ARCH_GUIDE}")
+    lines.append("  If narrowing the arch still fails, a different CUDA "
+                 "toolkit version usually does (ptxas bugs are fixed in "
+                 "toolkit updates).")
+    return "\n".join(lines)
+
 
 def apply_ggml_patches(audiocpp_dir: Path, *, emit=None, cancel=None) -> int:
     """Apply the shipped ggml build patches to an audio.cpp checkout.
@@ -348,6 +462,7 @@ def build_audiocpp(audiocpp_dir: Path, backend: str, *,
         return 1
     argv = ["sh", str(script), "--backend", backend, "--target",
             "audiocpp_server", "--deployment-build"]
+    argv += _cuda_arch_argv(backend, emit=emit)
     command = f"cd {audiocpp_dir} && {shlex.join(argv)}"
     if emit is None:
         print(f"[INFO] Building audiocpp_server for {backend} ({command})...")
@@ -409,12 +524,23 @@ def _build_audiocpp_tui(emit, cancel, argv: List[str], command: str,
             return patch_rc
         tee(f"[INFO] Building audiocpp_server ({command})...")
         rc = common.run_console_subprocess(
-            argv, cwd=audiocpp_dir, emit=tee, cancel=cancel)
+            argv, cwd=audiocpp_dir, emit=tee, cancel=cancel,
+            stall_timeout=BUILD_STALL_TIMEOUT)
         if rc != 0 and (cancel is None or not cancel.is_set()):
-            notice = (f"[ERROR] audio.cpp build failed (exit code {rc}).\n"
+            if rc == 124:
+                head = (f"[ERROR] audio.cpp build stalled — no output for "
+                        f"{BUILD_STALL_TIMEOUT // 60} minutes, so it was "
+                        "stopped (a wedged compiler job; often a ptxas "
+                        "hang from a buggy CUDA toolkit).")
+            else:
+                head = f"[ERROR] audio.cpp build failed (exit code {rc})."
+            notice = (f"{head}\n"
                       f"  Build log: {log_path}\n"
                       f"  Troubleshoot by re-running this command:\n"
                       f"    {command}")
+            ptxas_hint = _ptxas_failure_hint(log_path)
+            if ptxas_hint:
+                notice += "\n" + ptxas_hint
             for line in notice.splitlines():
                 tee(line)
             common.record_post_tui_notice(notice)
diff --git a/app/backends/audiocpp/models.py b/app/backends/audiocpp/models.py
index 75bc06a..772ae8d 100644
--- a/app/backends/audiocpp/models.py
+++ b/app/backends/audiocpp/models.py
@@ -11,6 +11,11 @@ from typing import Dict, List, Optional, Set, Tuple
 from backends import common
 from . import catalog as _catalog
 
+# No-output watchdog for model downloads: huggingface_hub streams steady
+# byte progress, so 5 minutes of silence means the transfer wedged. The
+# runner kills it and reports exit 124 (see run_console_subprocess).
+DOWNLOAD_STALL_TIMEOUT = 300
+
 def _installed_display_names(audiocpp_dir: Path,
                              model_entries: Optional[List[dict]],
                              install_guidance: List[Tuple[str, str]]
@@ -133,7 +138,9 @@ def _install_models(audiocpp_dir: Path,
         try:
             rc = common.run_console_subprocess(
                 argv, cwd=str(audiocpp_dir), emit=emit, cancel=cancel,
-                on_cancel=on_cancel)
+                on_cancel=on_cancel,
+                stall_timeout=(DOWNLOAD_STALL_TIMEOUT
+                               if supports_progress else None))
         except OSError as exc:
             print(f"[WARNING] Could not run python {manager} install "
                   f"{install_id}: {exc}")
diff --git a/app/backends/audiocpp/wizard.py b/app/backends/audiocpp/wizard.py
index 46b6638..c59da8f 100644
--- a/app/backends/audiocpp/wizard.py
+++ b/app/backends/audiocpp/wizard.py
@@ -574,7 +574,11 @@ def _execute_lanes(settings: dict,
         def build_step(emit, cancel):
             rc = _build.build_audiocpp(audiocpp_dir, settings["backend"],
                                 emit=emit, cancel=cancel)
-            if rc != 0:
+            if rc == 124:
+                print("[WARNING] build went silent and was stopped; the "
+                      "server.json was still written — build "
+                      "audiocpp_server manually before starting it")
+            elif rc != 0:
                 print(f"[WARNING] build exited with code {rc}; the server.json "
                       "was still written — build audiocpp_server manually "
                       "before starting it")