feat: cuda arch detection for shorter builds, build failure detection

8 files changed, 555 insertions, 16 deletions

diff --git a/app/backends/audiocpp/build.py b/app/backends/audiocpp/build.py
index 22c72a1..9ffe429 100644
--- a/app/backends/audiocpp/build.py
+++ b/app/backends/audiocpp/build.py
@@ -2,6 +2,7 @@
 
 import contextlib
 import io
+import os
 import re
 import shlex
 import shutil
@@ -260,6 +261,119 @@ GGML_PATCHES = [
     },
 ]
 
+# No-output watchdog for the build: ninja prints a line per completed
+# compile, so 15 minutes of silence means a compiler job wedged (ptxas
+# hangs are the known failure mode of a buggy CUDA toolkit). The runner
+# kills the build and reports exit 124 (see run_console_subprocess).
+BUILD_STALL_TIMEOUT = 900
+
+# Env override for the CUDA architectures passed to build_linux.sh
+# (--cuda-arch): a ';'-or-comma separated list of compute capabilities,
+# e.g. "86" or "86;89". Set it when GPU detection cannot run.
+CUDA_ARCH_ENV = "AUDIOCPP_CUDA_ARCH"
+
+# Compute capability -> common GPUs, shown when detection is impossible
+# and in ptxas failure guidance. Terse on purpose; one line.
+CUDA_ARCH_GUIDE = ("61 GTX 10xx/P40; 75 RTX 20xx; 80 A100; 86 RTX 30xx "
+                   "(3090)/A6000; 89 RTX 40xx (4090)/L40S; 90 H100; "
+                   "120/121 RTX 50xx (5090)/B200")
+
+
+def detect_cuda_arch() -> Optional[str]:
+    """The CUDA architecture token to build for, or None when unknown.
+
+    ``AUDIOCPP_CUDA_ARCH`` wins verbatim (validated as a ';'-or-comma
+    separated list of compute capabilities like ``86`` or ``86;89``), so a
+    user can pin the arch on machines where detection cannot run. Otherwise
+    ``nvidia-smi`` reports each GPU's compute capability (works on Linux
+    and Windows; it does not exist on macOS, where the CUDA backend is not
+    a choice anyway): ``8.6`` becomes ``86``, several distinct GPUs join
+    as ``86;89``. audio.cpp's CMake upgrades bare new architectures
+    (``120``) to their suffixed forms (``120a``) itself.
+    """
+    override = os.environ.get(CUDA_ARCH_ENV, "").strip()
+    if override:
+        parts = [part.strip() for part in
+                 override.replace(",", ";").split(";") if part.strip()]
+        if parts and all(re.fullmatch(r"\d+(-real|-virtual)?", part)
+                         for part in parts):
+            return ";".join(parts)
+        print(f"[WARNING] {CUDA_ARCH_ENV}={override!r} is not an arch list "
+              "(e.g. \"86\" or \"86;89\"); ignoring it")
+    proc = common.run_console_subprocess_quiet(
+        ["nvidia-smi", "--query-gpu=compute_cap", "--format=csv,noheader"],
+        timeout=10)
+    if proc is None or proc.returncode != 0:
+        return None
+    arches: List[str] = []
+    for line in proc.stdout.decode("utf-8", errors="replace").splitlines():
+        cap = line.strip()
+        if not re.fullmatch(r"\d+\.\d+", cap):
+            continue
+        arch = cap.replace(".", "")
+        if arch not in arches:
+            arches.append(arch)
+    return ";".join(arches) if arches else None
+
+
+def _cuda_arch_argv(backend: str, emit=None) -> List[str]:
+    """The ``--cuda-arch`` flags for a CUDA build, plus a status line.
+
+    EMIT is the in-TUI line sink when building from the task view (the
+    line lands in the view, not the real terminal behind curses); without
+    it the line prints to the console. Detection failure is not an error:
+    the build then uses audio.cpp's portable default arch list, which is
+    slower to compile but runs on any GPU.
+    """
+    if backend != "cuda":
+        return []
+    arch = detect_cuda_arch()
+    say = emit if emit is not None else print
+    if arch is None:
+        say(f"[INFO] CUDA architecture: portable default list (could not "
+            f"detect a GPU; set {CUDA_ARCH_ENV}=<arch> to build only for "
+            "this machine's GPU — much faster)")
+        return []
+    say(f"[INFO] CUDA architecture: {arch} (detected via nvidia-smi; "
+        f"override with {CUDA_ARCH_ENV})")
+    return ["--cuda-arch", arch]
+
+
+def _ptxas_failure_hint(log_path: Path) -> str:
+    """Guidance appended when the build log shows a ptxas failure.
+
+    ``ptxas fatal`` / ``nvcc error`` lines mean the CUDA toolkit's
+    assembler (or nvcc itself) failed — an internal compiler error is a
+    toolkit bug, not a broken checkout, and newer ggml template code on
+    newer toolkit releases trips it. Building only for the local GPU's
+    architecture skips most of the codegen paths ptxas chokes on, so the
+    hint points at ``AUDIOCPP_CUDA_ARCH`` (with the detected arch, or the
+    GPU table when detection cannot run); a different toolkit version is
+    the remaining fix when narrowing the arch is not enough.
+    """
+    try:
+        text = log_path.read_text(encoding="utf-8", errors="ignore")
+    except OSError:
+        return ""
+    if "ptxas fatal" not in text and "nvcc error" not in text:
+        return ""
+    arch = detect_cuda_arch()
+    lines = [
+        "  ptxas (the CUDA toolkit's GPU assembler) failed — with an "
+        "internal compiler error this is a CUDA toolkit bug, not your "
+        "sources.",
+        f"  Rebuild for this machine's GPU only: set {CUDA_ARCH_ENV}=<arch> "
+        "(semicolon-separated for several GPUs) and re-run the build.",
+    ]
+    if arch:
+        lines.append(f"  Detected arch for this machine: {arch}")
+    else:
+        lines.append(f"  Arch per GPU: {CUDA_ARCH_GUIDE}")
+    lines.append("  If narrowing the arch still fails, a different CUDA "
+                 "toolkit version usually does (ptxas bugs are fixed in "
+                 "toolkit updates).")
+    return "\n".join(lines)
+
 
 def apply_ggml_patches(audiocpp_dir: Path, *, emit=None, cancel=None) -> int:
     """Apply the shipped ggml build patches to an audio.cpp checkout.
@@ -348,6 +462,7 @@ def build_audiocpp(audiocpp_dir: Path, backend: str, *,
         return 1
     argv = ["sh", str(script), "--backend", backend, "--target",
             "audiocpp_server", "--deployment-build"]
+    argv += _cuda_arch_argv(backend, emit=emit)
     command = f"cd {audiocpp_dir} && {shlex.join(argv)}"
     if emit is None:
         print(f"[INFO] Building audiocpp_server for {backend} ({command})...")
@@ -409,12 +524,23 @@ def _build_audiocpp_tui(emit, cancel, argv: List[str], command: str,
             return patch_rc
         tee(f"[INFO] Building audiocpp_server ({command})...")
         rc = common.run_console_subprocess(
-            argv, cwd=audiocpp_dir, emit=tee, cancel=cancel)
+            argv, cwd=audiocpp_dir, emit=tee, cancel=cancel,
+            stall_timeout=BUILD_STALL_TIMEOUT)
         if rc != 0 and (cancel is None or not cancel.is_set()):
-            notice = (f"[ERROR] audio.cpp build failed (exit code {rc}).\n"
+            if rc == 124:
+                head = (f"[ERROR] audio.cpp build stalled — no output for "
+                        f"{BUILD_STALL_TIMEOUT // 60} minutes, so it was "
+                        "stopped (a wedged compiler job; often a ptxas "
+                        "hang from a buggy CUDA toolkit).")
+            else:
+                head = f"[ERROR] audio.cpp build failed (exit code {rc})."
+            notice = (f"{head}\n"
                       f"  Build log: {log_path}\n"
                       f"  Troubleshoot by re-running this command:\n"
                       f"    {command}")
+            ptxas_hint = _ptxas_failure_hint(log_path)
+            if ptxas_hint:
+                notice += "\n" + ptxas_hint
             for line in notice.splitlines():
                 tee(line)
             common.record_post_tui_notice(notice)
diff --git a/app/backends/audiocpp/models.py b/app/backends/audiocpp/models.py
index 75bc06a..772ae8d 100644
--- a/app/backends/audiocpp/models.py
+++ b/app/backends/audiocpp/models.py
@@ -11,6 +11,11 @@ from typing import Dict, List, Optional, Set, Tuple
 from backends import common
 from . import catalog as _catalog
 
+# No-output watchdog for model downloads: huggingface_hub streams steady
+# byte progress, so 5 minutes of silence means the transfer wedged. The
+# runner kills it and reports exit 124 (see run_console_subprocess).
+DOWNLOAD_STALL_TIMEOUT = 300
+
 def _installed_display_names(audiocpp_dir: Path,
                              model_entries: Optional[List[dict]],
                              install_guidance: List[Tuple[str, str]]
@@ -133,7 +138,9 @@ def _install_models(audiocpp_dir: Path,
         try:
             rc = common.run_console_subprocess(
                 argv, cwd=str(audiocpp_dir), emit=emit, cancel=cancel,
-                on_cancel=on_cancel)
+                on_cancel=on_cancel,
+                stall_timeout=(DOWNLOAD_STALL_TIMEOUT
+                               if supports_progress else None))
         except OSError as exc:
             print(f"[WARNING] Could not run python {manager} install "
                   f"{install_id}: {exc}")
diff --git a/app/backends/audiocpp/wizard.py b/app/backends/audiocpp/wizard.py
index 46b6638..c59da8f 100644
--- a/app/backends/audiocpp/wizard.py
+++ b/app/backends/audiocpp/wizard.py
@@ -574,7 +574,11 @@ def _execute_lanes(settings: dict,
         def build_step(emit, cancel):
             rc = _build.build_audiocpp(audiocpp_dir, settings["backend"],
                                 emit=emit, cancel=cancel)
-            if rc != 0:
+            if rc == 124:
+                print("[WARNING] build went silent and was stopped; the "
+                      "server.json was still written — build "
+                      "audiocpp_server manually before starting it")
+            elif rc != 0:
                 print(f"[WARNING] build exited with code {rc}; the server.json "
                       "was still written — build audiocpp_server manually "
                       "before starting it")
diff --git a/app/backends/common.py b/app/backends/common.py
index 4edfd61..4c768b8 100644
--- a/app/backends/common.py
+++ b/app/backends/common.py
@@ -347,7 +347,8 @@ def write_prompt_text(wav_dir: Path,
 
 
 def run_console_subprocess(argv: List[str], cwd: Optional[Path] = None,
-                           *, emit=None, cancel=None, on_cancel=None) -> int:
+                           *, emit=None, cancel=None, on_cancel=None,
+                           stall_timeout: Optional[float] = None) -> int:
     """Run a subprocess, streaming output to the console or to EMIT.
 
     With EMIT None the child inherits the real terminal and its output
@@ -360,7 +361,16 @@ def run_console_subprocess(argv: List[str], cwd: Optional[Path] = None,
     CANCEL is an optional ``threading.Event``: once set, ON_CANCEL (if given)
     is called (e.g. to touch a ``--cancel-file``), then the child's process
     group is terminated (SIGTERM, escalating to SIGKILL after a grace
-    period) and 130 is returned. Returns the process exit code.
+    period) and 130 is returned.
+
+    STALL_TIMEOUT (EMIT path only) is a no-output watchdog in seconds: when
+    the child produces no new output line for that long, it is treated as
+    wedged (a build whose compiler hung, a download that stopped moving) —
+    the process group is terminated, an [ERROR] line is emitted, and 124 is
+    returned so callers can report a stall distinctly from a plain failure.
+    None (the default) waits forever, as before.
+
+    Returns the process exit code.
     """
     import subprocess
     if emit is None:
@@ -387,12 +397,18 @@ def run_console_subprocess(argv: List[str], cwd: Optional[Path] = None,
         return 1
 
     cancelled = False
+    stalled = False
+    # Written by the reader thread, read by the poll loop below: a plain
+    # float assignment is atomic enough under the GIL (no torn reads).
+    last_output = time.monotonic()
 
     def _reader() -> None:
+        nonlocal last_output
         try:
             for raw in iter(proc.stdout.readline, b""):
                 if not raw:
                     break
+                last_output = time.monotonic()
                 text = raw.decode("utf-8", errors="replace")
                 for line in text.splitlines():
                     if line:
@@ -421,6 +437,13 @@ def run_console_subprocess(argv: List[str], cwd: Optional[Path] = None,
             break
         if proc.poll() is not None:
             break
+        if (stall_timeout is not None
+                and time.monotonic() - last_output > stall_timeout):
+            stalled = True
+            emit(f"[ERROR] No output for {int(stall_timeout)}s — assuming "
+                 "the process hung; stopping it.")
+            _terminate_process_group(proc)
+            break
         time.sleep(0.1)
     try:
         reader.join(timeout=5)
@@ -429,6 +452,8 @@ def run_console_subprocess(argv: List[str], cwd: Optional[Path] = None,
             reader.join(timeout=0)
     if cancelled:
         return 130
+    if stalled:
+        return 124
     return proc.returncode
 
 
@@ -577,20 +602,29 @@ def _origin_default_branch(checkout: Path) -> str:
 
 
 def run_console_subprocess_quiet(argv: List[str],
-                                 cwd: Optional[Path] = None):
+                                 cwd: Optional[Path] = None,
+                                 timeout: Optional[float] = None):
     """Run ARGV silently and return the completed result.
 
     Unlike run_console_subprocess (which streams or returns only an exit
     code) this captures stdout and needs the process object itself, for the
     small git probes (rev-parse, symbolic-ref) whose *output* matters and
-    whose failure is a normal, non-fatal outcome. Returns None when the
-    process could not be started.
+    whose failure is a normal, non-fatal outcome. TIMEOUT bounds the wait
+    (e.g. for hardware probes like nvidia-smi that can hang on a wedged
+    driver); a timeout kills the child and returns a failed result, not an
+    exception. Returns None when the process could not be started.
     """
     import subprocess
     try:
         return subprocess.run(
             argv, capture_output=True,
-            cwd=str(cwd) if cwd is not None else None, check=False)
+            cwd=str(cwd) if cwd is not None else None, check=False,
+            timeout=timeout)
+    except subprocess.TimeoutExpired:
+        class _TimedOut:
+            returncode = -1
+            stdout = b""
+        return _TimedOut()
     except OSError:
         return None
 
diff --git a/app/tests/test_backends_audiocpp.py b/app/tests/test_backends_audiocpp.py
index 6f97438..4a8ee5f 100644
--- a/app/tests/test_backends_audiocpp.py
+++ b/app/tests/test_backends_audiocpp.py
@@ -1407,6 +1407,189 @@ class BuildAudiocppTests(unittest.TestCase):
         self.assertEqual(len(notices), 1)
         self.assertIn("No build script found", notices[0])
 
+    def test_cuda_build_appends_detected_arch_flag(self):
+        with patch.object(common, "run_console_subprocess",
+                          return_value=0) as run, \
+                patch.object(make_server.build, "detect_cuda_arch",
+                             return_value="86"):
+            make_server.build.build_audiocpp(self.checkout, "cuda")
+        argv = run.call_args[0][0]
+        self.assertIn("--cuda-arch", argv)
+        self.assertEqual(argv[argv.index("--cuda-arch") + 1], "86")
+
+    def test_tui_stall_notice_reports_the_hang(self):
+        emitted, emit = self._emit()
+        with patch.object(common, "LOG_DIR", self.log_dir), \
+                patch.object(common, "run_console_subprocess",
+                             return_value=124):
+            rc = make_server.build.build_audiocpp(self.checkout, "cuda",
+                                            emit=emit)
+        self.assertEqual(rc, 124)
+        log_text = self._log_files()[0].read_text(encoding="utf-8")
+        self.assertIn("stalled", log_text)
+        notices = common.drain_post_tui_notices()
+        self.assertEqual(len(notices), 1)
+        self.assertIn("stalled", notices[0])
+        self.assertIn("stopped", notices[0])
+
+    def test_tui_ptxas_failure_appends_detected_arch_guidance(self):
+        emitted, emit = self._emit()
+
+        def fake_run(argv, cwd=None, emit=None, cancel=None, **kwargs):
+            emit("ptxas fatal   : (C7907) Internal compiler error.")
+            return 1
+
+        with patch.object(common, "LOG_DIR", self.log_dir), \
+                patch.object(common, "run_console_subprocess",
+                             side_effect=fake_run), \
+                patch.object(make_server.build, "detect_cuda_arch",
+                             return_value="86"):
+            rc = make_server.build.build_audiocpp(self.checkout, "cuda",
+                                            emit=emit)
+        self.assertEqual(rc, 1)
+        notice = common.drain_post_tui_notices()[0]
+        self.assertIn("CUDA toolkit bug", notice)
+        self.assertIn("AUDIOCPP_CUDA_ARCH", notice)
+        self.assertIn("Detected arch for this machine: 86", notice)
+        # The undetected-GPU table is not needed when detection worked.
+        self.assertNotIn("Arch per GPU", notice)
+
+    def test_tui_ptxas_failure_lists_gpu_table_when_undetected(self):
+        emitted, emit = self._emit()
+
+        def fake_run(argv, cwd=None, emit=None, cancel=None, **kwargs):
+            emit("ptxas fatal   : (C7907) Internal compiler error.")
+            return 1
+
+        with patch.object(common, "LOG_DIR", self.log_dir), \
+                patch.object(common, "run_console_subprocess",
+                             side_effect=fake_run), \
+                patch.object(make_server.build, "detect_cuda_arch",
+                             return_value=None):
+            rc = make_server.build.build_audiocpp(self.checkout, "cuda",
+                                            emit=emit)
+        self.assertEqual(rc, 1)
+        notice = common.drain_post_tui_notices()[0]
+        self.assertIn("Arch per GPU", notice)
+        self.assertIn("3090", notice)
+        self.assertIn("4090", notice)
+
+    def test_tui_plain_failure_has_no_ptxas_guidance(self):
+        emitted, emit = self._emit()
+        with patch.object(common, "LOG_DIR", self.log_dir), \
+                patch.object(common, "run_console_subprocess",
+                             return_value=3):
+            rc = make_server.build.build_audiocpp(self.checkout, "cuda",
+                                            emit=emit)
+        notice = common.drain_post_tui_notices()[0]
+        self.assertIn("failed (exit code 3)", notice)
+        self.assertNotIn("AUDIOCPP_CUDA_ARCH", notice)
+
+
+class DetectCudaArchTests(unittest.TestCase):
+    """detect_cuda_arch: env override, nvidia-smi probe, None fallbacks."""
+
+    def setUp(self):
+        # Run every case without an AUDIOCPP_CUDA_ARCH leak from the
+        # developer's own shell.
+        clean = {k: v for k, v in os.environ.items()
+                 if k != make_server.build.CUDA_ARCH_ENV}
+        patcher = patch.dict(os.environ, clean, clear=True)
+        patcher.start()
+        self.addCleanup(patcher.stop)
+
+    def test_env_override_wins_without_probing(self):
+        with patch.dict(os.environ,
+                        {make_server.build.CUDA_ARCH_ENV: "86"}), \
+                patch.object(common, "run_console_subprocess_quiet") as run:
+            self.assertEqual(make_server.build.detect_cuda_arch(), "86")
+        run.assert_not_called()
+
+    def test_env_override_multi_gpu_and_commas(self):
+        with patch.dict(os.environ,
+                        {make_server.build.CUDA_ARCH_ENV: "86, 89;75"}):
+            self.assertEqual(make_server.build.detect_cuda_arch(),
+                             "86;89;75")
+
+    def test_env_override_real_virtual_suffixes_allowed(self):
+        with patch.dict(os.environ,
+                        {make_server.build.CUDA_ARCH_ENV: "86-real"}):
+            self.assertEqual(make_server.build.detect_cuda_arch(), "86-real")
+
+    def test_invalid_env_override_ignored_and_probe_runs(self):
+        probe = MagicMock(returncode=0, stdout=b"8.6\n")
+        with patch.dict(os.environ,
+                        {make_server.build.CUDA_ARCH_ENV: "rtx"}), \
+                patch.object(common, "run_console_subprocess_quiet",
+                             return_value=probe) as run:
+            self.assertEqual(make_server.build.detect_cuda_arch(), "86")
+        self.assertEqual(run.call_args[0][0][0], "nvidia-smi")
+
+    def test_compute_caps_parsed_and_deduped(self):
+        probe = MagicMock(returncode=0, stdout=b"8.6\n8.6\n12.0\n")
+        with patch.object(common, "run_console_subprocess_quiet",
+                          return_value=probe):
+            self.assertEqual(make_server.build.detect_cuda_arch(), "86;120")
+
+    def test_nvidia_smi_failure_yields_none(self):
+        probe = MagicMock(returncode=1, stdout=b"")
+        with patch.object(common, "run_console_subprocess_quiet",
+                          return_value=probe):
+            self.assertIsNone(make_server.build.detect_cuda_arch())
+
+    def test_unparsable_output_yields_none(self):
+        probe = MagicMock(returncode=0,
+                          stdout=b"NVIDIA-SMI has failed because...\n")
+        with patch.object(common, "run_console_subprocess_quiet",
+                          return_value=probe):
+            self.assertIsNone(make_server.build.detect_cuda_arch())
+
+    def test_unstartable_probe_yields_none(self):
+        with patch.object(common, "run_console_subprocess_quiet",
+                          return_value=None):
+            self.assertIsNone(make_server.build.detect_cuda_arch())
+
+    def test_probe_is_bounded_by_a_timeout(self):
+        probe = MagicMock(returncode=0, stdout=b"8.6\n")
+        with patch.object(common, "run_console_subprocess_quiet",
+                          return_value=probe) as run:
+            make_server.build.detect_cuda_arch()
+        self.assertIsNotNone(run.call_args[1].get("timeout"))
+
+
+class CudaArchArgvTests(unittest.TestCase):
+    """_cuda_arch_argv: the --cuda-arch flags and their status line."""
+
+    def _argv(self, backend, arch, emit=None):
+        with patch.object(make_server.build, "detect_cuda_arch",
+                          return_value=arch):
+            return make_server.build._cuda_arch_argv(backend, emit=emit)
+
+    def test_cuda_build_gets_the_detected_arch(self):
+        self.assertEqual(self._argv("cuda", "86"),
+                         ["--cuda-arch", "86"])
+
+    def test_multi_gpu_arch_passed_verbatim(self):
+        self.assertEqual(self._argv("cuda", "86;89"),
+                         ["--cuda-arch", "86;89"])
+
+    def test_detection_failure_means_no_flag(self):
+        self.assertEqual(self._argv("cuda", None), [])
+
+    def test_non_cuda_backends_never_get_the_flag(self):
+        for backend in ("cpu", "vulkan", "hip"):
+            with self.subTest(backend=backend):
+                self.assertEqual(self._argv(backend, "86"), [])
+
+    def test_status_lines_report_the_outcome(self):
+        detected, emit = [], lambda line: detected.append(line)
+        self._argv("cuda", "86", emit=emit)
+        self.assertIn("CUDA architecture: 86", detected[0])
+        undetected, emit = [], lambda line: undetected.append(line)
+        self._argv("cuda", None, emit=emit)
+        self.assertIn("portable default list", undetected[0])
+        self.assertIn(make_server.build.CUDA_ARCH_ENV, undetected[0])
+
 
 class AudiocppUpdateTests(unittest.TestCase):
     """update: stop the server, refresh the checkout, rebuild when stale.
diff --git a/app/tests/test_backends_common.py b/app/tests/test_backends_common.py
index a94994b..08ffc2c 100644
--- a/app/tests/test_backends_common.py
+++ b/app/tests/test_backends_common.py
@@ -2,8 +2,9 @@
 
 The streaming mode of ``run_console_subprocess`` (used by the in-TUI task
 view) is exercised with a real child process: output lines are captured and
-forwarded, cancellation kills the child and returns 130, and an on_cancel
-hook runs first.
+forwarded, cancellation kills the child and returns 130, an on_cancel hook
+runs first, and the no-output stall watchdog kills a wedged child and
+returns 124.
 """
 
 import sys
@@ -51,6 +52,58 @@ class RunConsoleSubprocessStreamingTests(unittest.TestCase):
         self.assertEqual(touched, [True])
 
 
+class RunConsoleSubprocessStallTests(unittest.TestCase):
+    """The no-output watchdog: a silent child is killed and reported 124."""
+
+    def test_stall_kills_a_silent_child_and_returns_124(self):
+        lines = []
+        rc = common.run_console_subprocess(
+            [sys.executable, "-c",
+             "import sys, time; print('start', flush=True); "
+             "time.sleep(60)"],
+            emit=lines.append, stall_timeout=0.5)
+        self.assertEqual(rc, 124)
+        self.assertEqual(lines[0], "start")
+        # The stall is announced to the view before the kill.
+        self.assertTrue(any("[ERROR]" in line and "No output" in line
+                            for line in lines), lines)
+
+    def test_no_stall_while_output_keeps_flowing(self):
+        lines = []
+        rc = common.run_console_subprocess(
+            [sys.executable, "-c",
+             "import sys, time\n"
+             "for _ in range(6):\n"
+             "    print('tick', flush=True)\n"
+             "    time.sleep(0.2)\n"],
+            emit=lines.append, stall_timeout=1.0)
+        self.assertEqual(rc, 0)
+        self.assertEqual(lines, ["tick"] * 6)
+
+    def test_console_path_has_no_watchdog(self):
+        # Without emit the child inherits the terminal; stall_timeout is
+        # a no-op there (the caller sees raw output and can Ctrl-C).
+        rc = common.run_console_subprocess(
+            [sys.executable, "-c", "print('hi')"], stall_timeout=0.001)
+        self.assertEqual(rc, 0)
+
+
+class RunConsoleSubprocessQuietTimeoutTests(unittest.TestCase):
+    """A timed-out quiet probe returns a failed result, never raises."""
+
+    def test_timeout_returns_failed_result(self):
+        proc = common.run_console_subprocess_quiet(
+            [sys.executable, "-c", "import time; time.sleep(30)"],
+            timeout=0.5)
+        self.assertIsNotNone(proc)
+        self.assertEqual(proc.returncode, -1)
+
+    def test_untimed_probe_still_reports_the_exit_code(self):
+        proc = common.run_console_subprocess_quiet(
+            [sys.executable, "-c", "import sys; sys.exit(5)"])
+        self.assertEqual(proc.returncode, 5)
+
+
 class GitCloneTests(unittest.TestCase):
     def test_git_clone_console_passes_through(self):
         with mock.patch.object(common, "run_console_subprocess",
diff --git a/app/tests/test_taskview.py b/app/tests/test_taskview.py
index b7f4389..646fccb 100644
--- a/app/tests/test_taskview.py
+++ b/app/tests/test_taskview.py
@@ -592,5 +592,91 @@ class LanesViewTests(_FakeTui, unittest.TestCase):
         self.assertLessEqual(x + len(label) - 1, 1 + pane_w - 2)
 
 
+class SilenceCueTests(_FakeTui, unittest.TestCase):
+    """The "(no output 6m)" cue for a running step that stopped emitting."""
+
+    def test_silence_cue_text_format(self):
+        self.assertEqual(taskview._silence_cue_text(90), "(no output 1m)")
+        self.assertEqual(taskview._silence_cue_text(3700), "(no output 61m)")
+
+    def test_no_cue_while_output_is_recent(self):
+        view, _ = self.make_view(steps=[_step("one")])
+        view.handle_event({"kind": "step_start", "index": 0, "title": "one"})
+        view._ingest_line("working")
+        self.assertIsNone(taskview._silent_secs(view.last_line_at,
+                                                view.last_line_at + 59))
+
+    def test_cue_after_the_threshold(self):
+        view, _ = self.make_view(steps=[_step("one")])
+        view.handle_event({"kind": "step_start", "index": 0, "title": "one"})
+        view._ingest_line("working")
+        self.assertEqual(taskview._silent_secs(view.last_line_at,
+                                               view.last_line_at + 60), 60)
+
+    def test_step_start_resets_and_done_clears_the_tracking(self):
+        view, _ = self.make_view(steps=[_step("one"), _step("two")])
+        view.handle_event({"kind": "step_start", "index": 0, "title": "one"})
+        start = view.last_line_at
+        view.handle_event({"kind": "step_done", "index": 0, "rc": 0})
+        self.assertIsNone(view.last_line_at)
+        view.handle_event({"kind": "step_start", "index": 1, "title": "two"})
+        self.assertGreaterEqual(view.last_line_at, start)
+
+    def test_render_shows_the_cue_for_a_silent_step(self):
+        now = [1000.0]
+
+        def clock():
+            return now[0]
+
+        screen = FakeScreen(width=80, height=24)
+        with patch.object(taskview.TaskView, "_worker_main",
+                          lambda self: None):
+            view = taskview.TaskView(screen, "Setup", [_step("one")],
+                                     clock=clock)
+        view.handle_event({"kind": "step_start", "index": 0, "title": "one"})
+        view._ingest_line("working")
+        now[0] = view.last_line_at + 300
+        view.render()
+        text = " ".join(t for _, _, t, _ in screen.strings)
+        self.assertIn("(no output 5m)", text)
+
+    def test_render_hides_the_cue_when_output_is_fresh(self):
+        now = [1000.0]
+
+        def clock():
+            return now[0]
+
+        screen = FakeScreen(width=80, height=24)
+        with patch.object(taskview.TaskView, "_worker_main",
+                          lambda self: None):
+            view = taskview.TaskView(screen, "Setup", [_step("one")],
+                                     clock=clock)
+        view.handle_event({"kind": "step_start", "index": 0, "title": "one"})
+        view._ingest_line("working")
+        now[0] = view.last_line_at + 5
+        view.render()
+        text = " ".join(t for _, _, t, _ in screen.strings)
+        self.assertNotIn("(no output", text)
+
+    def test_lane_pane_shows_the_cue_for_a_silent_lane(self):
+        now = [1000.0]
+
+        def clock():
+            return now[0]
+
+        screen = FakeScreen(width=80, height=24)
+        view = taskview.LanesView(
+            screen, "Setup",
+            [taskview.TaskLane("Build", [_step("one")])], clock=clock)
+        lane = view._lanes[0]
+        view._handle_lane_event(
+            lane, {"kind": "step_start", "index": 0, "title": "one"})
+        view._ingest_lane_line(lane, "working")
+        now[0] = lane.last_line_at + 120
+        view.render()
+        text = " ".join(t for _, _, t, _ in screen.strings)
+        self.assertIn("(no output 2m)", text)
+
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/app/ui/taskview.py b/app/ui/taskview.py
index c5c2ff7..d7aed96 100644
--- a/app/ui/taskview.py
+++ b/app/ui/taskview.py
@@ -63,6 +63,11 @@ _PROGRESS_COUNT = re.compile(r"\[(\d+)/(\d+)\]")
 # A spinner frame set for the running step marker.
 _SPINNER = ("|", "/", "-", "\\")
 
+# How long a running step may stay silent (no output lines) before the
+# view starts saying so next to its elapsed clock — an early "this looks
+# wedged" cue for the no-output watchdog that eventually kills the step.
+_SILENCE_CUE_SECS = 60
+
 
 @dataclass
 class TaskStep:
@@ -117,6 +122,27 @@ def _progress_match(text: str) -> Optional[Tuple[float, float, str]]:
     return None
 
 
+def _silent_secs(last_line_at: Optional[float], now: float
+                 ) -> Optional[float]:
+    """Seconds the running step has been silent, or None (no cue).
+
+    None when nothing is tracked (no output yet is not tracked here — the
+    caller decides), or when the silence is still below the cue threshold.
+    Module-level for testability.
+    """
+    if last_line_at is None:
+        return None
+    silent = now - last_line_at
+    if silent < _SILENCE_CUE_SECS:
+        return None
+    return silent
+
+
+def _silence_cue_text(silent: float) -> str:
+    """The dim text shown next to the elapsed clock for SILENT seconds."""
+    return f"(no output {int(silent // 60)}m)"
+
+
 def _lane_step_mark(current: Optional[int],
                     results: List[Optional[int]],
                     cancelled_step: Optional[int],
@@ -208,6 +234,7 @@ class TaskView(ScreenView):
         self._progress: Optional[Tuple[float, float]] = None  # (done, total)
         self._progress_kind = ""   # "bytes" | "percent" | "count" | ""
         self.step_started: List[Optional[float]] = [None] * len(steps)
+        self.last_line_at: Optional[float] = None  # silence cue (see _SILENCE_CUE_SECS)
         self.finished_at: Optional[float] = None
         self.cancelled = False
         self.cancelling = False
@@ -265,6 +292,7 @@ class TaskView(ScreenView):
         if kind == "step_start":
             self.current = event["index"]
             self.step_started[self.current] = self._now()
+            self.last_line_at = self._now()
             self._progress = None
             self._progress_kind = ""
         elif kind == "line":
@@ -274,11 +302,13 @@ class TaskView(ScreenView):
             index = event["index"]
             self.results[index] = event.get("rc") or 0
             self.current = None
+            self.last_line_at = None
             self._progress = None
             self._progress_kind = ""
         elif kind == "step_cancelled":
             self.cancelled_step = event["index"]
             self.current = None
+            self.last_line_at = None
             self._progress = None
             self._progress_kind = ""
         elif kind == "finish":
@@ -290,6 +320,7 @@ class TaskView(ScreenView):
     def _ingest_line(self, text: str) -> None:
         """Fold one output line into the log tail and progress bar."""
         line = text.rstrip("\r\n")
+        self.last_line_at = self._now()
         if not line:
             return
         match = _progress_match(line)
@@ -371,9 +402,14 @@ class TaskView(ScreenView):
             _text(scr, theme, y, inner_x + 5, label, theme["body"])
             if index == self.current and self.phase not in _TERMINAL:
                 started = self.step_started[index] or self._now()
+                elapsed_text = f" {_format_elapsed(self._now() - started)}"
                 _text(scr, theme, y, inner_x + 5 + len(label) + 1,
-                      f" {_format_elapsed(self._now() - started)}",
-                      theme["dim"])
+                      elapsed_text, theme["dim"])
+                silent = _silent_secs(self.last_line_at, self._now())
+                if silent is not None:
+                    _text(scr, theme, y,
+                          inner_x + 5 + len(label) + 1 + len(elapsed_text),
+                          f" {_silence_cue_text(silent)}", theme["dim"])
             y += 1
 
         y += 1
@@ -581,6 +617,7 @@ class _LaneState:
         self.progress: Optional[Tuple[float, float]] = None
         self.progress_kind = ""
         self.step_started: List[Optional[float]] = [None] * len(self.steps)
+        self.last_line_at: Optional[float] = None  # silence cue
         self.cancelled_step: Optional[int] = None
         self.rc = 0
         self.finished = False
@@ -664,6 +701,7 @@ class LanesView(_GetchModes):
         if kind == "step_start":
             lane.current = event["index"]
             lane.step_started[lane.current] = self._now()
+            lane.last_line_at = self._now()
             lane.progress = None
             lane.progress_kind = ""
         elif kind == "line":
@@ -671,11 +709,13 @@ class LanesView(_GetchModes):
         elif kind == "step_done":
             lane.results[event["index"]] = event.get("rc") or 0
             lane.current = None
+            lane.last_line_at = None
             lane.progress = None
             lane.progress_kind = ""
         elif kind == "step_cancelled":
             lane.cancelled_step = event["index"]
             lane.current = None
+            lane.last_line_at = None
             lane.progress = None
             lane.progress_kind = ""
         elif kind == "lane_finish":
@@ -685,6 +725,7 @@ class LanesView(_GetchModes):
     def _ingest_lane_line(self, lane: _LaneState, text: str) -> None:
         """Fold one output line into LANE's log tail and progress bar."""
         line = text.rstrip("\r\n")
+        lane.last_line_at = self._now()
         if not line:
             return
         match = _progress_match(line)
@@ -862,9 +903,14 @@ class LanesView(_GetchModes):
             _text(scr, theme, row, x + 6, label, theme["body"])
             if index == lane.current and not terminal:
                 started = lane.step_started[index] or self._now()
+                elapsed_text = f" {_format_elapsed(self._now() - started)}"
                 _text(scr, theme, row, x + 6 + len(label) + 1,
-                      f" {_format_elapsed(self._now() - started)}",
-                      theme["dim"])
+                      elapsed_text, theme["dim"])
+                silent = _silent_secs(lane.last_line_at, self._now())
+                if silent is not None:
+                    _text(scr, theme, row,
+                          x + 6 + len(label) + 1 + len(elapsed_text),
+                          f" {_silence_cue_text(silent)}", theme["dim"])
             row += 1
 
         row += 1