feat: cuda arch detection for shorter builds, build failure detection

1 files changed, 40 insertions, 6 deletions

diff --git a/app/backends/common.py b/app/backends/common.py
index 4edfd61..4c768b8 100644
--- a/app/backends/common.py
+++ b/app/backends/common.py
@@ -347,7 +347,8 @@ def write_prompt_text(wav_dir: Path,
 
 
 def run_console_subprocess(argv: List[str], cwd: Optional[Path] = None,
-                           *, emit=None, cancel=None, on_cancel=None) -> int:
+                           *, emit=None, cancel=None, on_cancel=None,
+                           stall_timeout: Optional[float] = None) -> int:
     """Run a subprocess, streaming output to the console or to EMIT.
 
     With EMIT None the child inherits the real terminal and its output
@@ -360,7 +361,16 @@ def run_console_subprocess(argv: List[str], cwd: Optional[Path] = None,
     CANCEL is an optional ``threading.Event``: once set, ON_CANCEL (if given)
     is called (e.g. to touch a ``--cancel-file``), then the child's process
     group is terminated (SIGTERM, escalating to SIGKILL after a grace
-    period) and 130 is returned. Returns the process exit code.
+    period) and 130 is returned.
+
+    STALL_TIMEOUT (EMIT path only) is a no-output watchdog in seconds: when
+    the child produces no new output line for that long, it is treated as
+    wedged (a build whose compiler hung, a download that stopped moving) —
+    the process group is terminated, an [ERROR] line is emitted, and 124 is
+    returned so callers can report a stall distinctly from a plain failure.
+    None (the default) waits forever, as before.
+
+    Returns the process exit code.
     """
     import subprocess
     if emit is None:
@@ -387,12 +397,18 @@ def run_console_subprocess(argv: List[str], cwd: Optional[Path] = None,
         return 1
 
     cancelled = False
+    stalled = False
+    # Written by the reader thread, read by the poll loop below: a plain
+    # float assignment is atomic enough under the GIL (no torn reads).
+    last_output = time.monotonic()
 
     def _reader() -> None:
+        nonlocal last_output
         try:
             for raw in iter(proc.stdout.readline, b""):
                 if not raw:
                     break
+                last_output = time.monotonic()
                 text = raw.decode("utf-8", errors="replace")
                 for line in text.splitlines():
                     if line:
@@ -421,6 +437,13 @@ def run_console_subprocess(argv: List[str], cwd: Optional[Path] = None,
             break
         if proc.poll() is not None:
             break
+        if (stall_timeout is not None
+                and time.monotonic() - last_output > stall_timeout):
+            stalled = True
+            emit(f"[ERROR] No output for {int(stall_timeout)}s — assuming "
+                 "the process hung; stopping it.")
+            _terminate_process_group(proc)
+            break
         time.sleep(0.1)
     try:
         reader.join(timeout=5)
@@ -429,6 +452,8 @@ def run_console_subprocess(argv: List[str], cwd: Optional[Path] = None,
             reader.join(timeout=0)
     if cancelled:
         return 130
+    if stalled:
+        return 124
     return proc.returncode
 
 
@@ -577,20 +602,29 @@ def _origin_default_branch(checkout: Path) -> str:
 
 
 def run_console_subprocess_quiet(argv: List[str],
-                                 cwd: Optional[Path] = None):
+                                 cwd: Optional[Path] = None,
+                                 timeout: Optional[float] = None):
     """Run ARGV silently and return the completed result.
 
     Unlike run_console_subprocess (which streams or returns only an exit
     code) this captures stdout and needs the process object itself, for the
     small git probes (rev-parse, symbolic-ref) whose *output* matters and
-    whose failure is a normal, non-fatal outcome. Returns None when the
-    process could not be started.
+    whose failure is a normal, non-fatal outcome. TIMEOUT bounds the wait
+    (e.g. for hardware probes like nvidia-smi that can hang on a wedged
+    driver); a timeout kills the child and returns a failed result, not an
+    exception. Returns None when the process could not be started.
     """
     import subprocess
     try:
         return subprocess.run(
             argv, capture_output=True,
-            cwd=str(cwd) if cwd is not None else None, check=False)
+            cwd=str(cwd) if cwd is not None else None, check=False,
+            timeout=timeout)
+    except subprocess.TimeoutExpired:
+        class _TimedOut:
+            returncode = -1
+            stdout = b""
+        return _TimedOut()
     except OSError:
         return None