From 5c3db8f500ff206f3a675d8f4184cb0d61f94804 Mon Sep 17 00:00:00 2001 From: historia Date: Fri, 28 Aug 2026 02:43:27 -0400 Subject: feat: cuda arch detection for shorter builds, build failure detection --- app/backends/audiocpp/build.py | 130 ++++++++++++++++++++++++- app/backends/audiocpp/models.py | 9 +- app/backends/audiocpp/wizard.py | 6 +- app/backends/common.py | 46 +++++++-- app/tests/test_backends_audiocpp.py | 183 ++++++++++++++++++++++++++++++++++++ app/tests/test_backends_common.py | 57 ++++++++++- app/tests/test_taskview.py | 86 +++++++++++++++++ app/ui/taskview.py | 54 ++++++++++- 8 files changed, 555 insertions(+), 16 deletions(-) (limited to 'app') diff --git a/app/backends/audiocpp/build.py b/app/backends/audiocpp/build.py index 22c72a1..9ffe429 100644 --- a/app/backends/audiocpp/build.py +++ b/app/backends/audiocpp/build.py @@ -2,6 +2,7 @@ import contextlib import io +import os import re import shlex import shutil @@ -260,6 +261,119 @@ GGML_PATCHES = [ }, ] +# No-output watchdog for the build: ninja prints a line per completed +# compile, so 15 minutes of silence means a compiler job wedged (ptxas +# hangs are the known failure mode of a buggy CUDA toolkit). The runner +# kills the build and reports exit 124 (see run_console_subprocess). +BUILD_STALL_TIMEOUT = 900 + +# Env override for the CUDA architectures passed to build_linux.sh +# (--cuda-arch): a ';'-or-comma separated list of compute capabilities, +# e.g. "86" or "86;89". Set it when GPU detection cannot run. +CUDA_ARCH_ENV = "AUDIOCPP_CUDA_ARCH" + +# Compute capability -> common GPUs, shown when detection is impossible +# and in ptxas failure guidance. Terse on purpose; one line. +CUDA_ARCH_GUIDE = ("61 GTX 10xx/P40; 75 RTX 20xx; 80 A100; 86 RTX 30xx " + "(3090)/A6000; 89 RTX 40xx (4090)/L40S; 90 H100; " + "120/121 RTX 50xx (5090)/B200") + + +def detect_cuda_arch() -> Optional[str]: + """The CUDA architecture token to build for, or None when unknown. + + ``AUDIOCPP_CUDA_ARCH`` wins verbatim (validated as a ';'-or-comma + separated list of compute capabilities like ``86`` or ``86;89``), so a + user can pin the arch on machines where detection cannot run. Otherwise + ``nvidia-smi`` reports each GPU's compute capability (works on Linux + and Windows; it does not exist on macOS, where the CUDA backend is not + a choice anyway): ``8.6`` becomes ``86``, several distinct GPUs join + as ``86;89``. audio.cpp's CMake upgrades bare new architectures + (``120``) to their suffixed forms (``120a``) itself. + """ + override = os.environ.get(CUDA_ARCH_ENV, "").strip() + if override: + parts = [part.strip() for part in + override.replace(",", ";").split(";") if part.strip()] + if parts and all(re.fullmatch(r"\d+(-real|-virtual)?", part) + for part in parts): + return ";".join(parts) + print(f"[WARNING] {CUDA_ARCH_ENV}={override!r} is not an arch list " + "(e.g. \"86\" or \"86;89\"); ignoring it") + proc = common.run_console_subprocess_quiet( + ["nvidia-smi", "--query-gpu=compute_cap", "--format=csv,noheader"], + timeout=10) + if proc is None or proc.returncode != 0: + return None + arches: List[str] = [] + for line in proc.stdout.decode("utf-8", errors="replace").splitlines(): + cap = line.strip() + if not re.fullmatch(r"\d+\.\d+", cap): + continue + arch = cap.replace(".", "") + if arch not in arches: + arches.append(arch) + return ";".join(arches) if arches else None + + +def _cuda_arch_argv(backend: str, emit=None) -> List[str]: + """The ``--cuda-arch`` flags for a CUDA build, plus a status line. + + EMIT is the in-TUI line sink when building from the task view (the + line lands in the view, not the real terminal behind curses); without + it the line prints to the console. Detection failure is not an error: + the build then uses audio.cpp's portable default arch list, which is + slower to compile but runs on any GPU. + """ + if backend != "cuda": + return [] + arch = detect_cuda_arch() + say = emit if emit is not None else print + if arch is None: + say(f"[INFO] CUDA architecture: portable default list (could not " + f"detect a GPU; set {CUDA_ARCH_ENV}= to build only for " + "this machine's GPU — much faster)") + return [] + say(f"[INFO] CUDA architecture: {arch} (detected via nvidia-smi; " + f"override with {CUDA_ARCH_ENV})") + return ["--cuda-arch", arch] + + +def _ptxas_failure_hint(log_path: Path) -> str: + """Guidance appended when the build log shows a ptxas failure. + + ``ptxas fatal`` / ``nvcc error`` lines mean the CUDA toolkit's + assembler (or nvcc itself) failed — an internal compiler error is a + toolkit bug, not a broken checkout, and newer ggml template code on + newer toolkit releases trips it. Building only for the local GPU's + architecture skips most of the codegen paths ptxas chokes on, so the + hint points at ``AUDIOCPP_CUDA_ARCH`` (with the detected arch, or the + GPU table when detection cannot run); a different toolkit version is + the remaining fix when narrowing the arch is not enough. + """ + try: + text = log_path.read_text(encoding="utf-8", errors="ignore") + except OSError: + return "" + if "ptxas fatal" not in text and "nvcc error" not in text: + return "" + arch = detect_cuda_arch() + lines = [ + " ptxas (the CUDA toolkit's GPU assembler) failed — with an " + "internal compiler error this is a CUDA toolkit bug, not your " + "sources.", + f" Rebuild for this machine's GPU only: set {CUDA_ARCH_ENV}= " + "(semicolon-separated for several GPUs) and re-run the build.", + ] + if arch: + lines.append(f" Detected arch for this machine: {arch}") + else: + lines.append(f" Arch per GPU: {CUDA_ARCH_GUIDE}") + lines.append(" If narrowing the arch still fails, a different CUDA " + "toolkit version usually does (ptxas bugs are fixed in " + "toolkit updates).") + return "\n".join(lines) + def apply_ggml_patches(audiocpp_dir: Path, *, emit=None, cancel=None) -> int: """Apply the shipped ggml build patches to an audio.cpp checkout. @@ -348,6 +462,7 @@ def build_audiocpp(audiocpp_dir: Path, backend: str, *, return 1 argv = ["sh", str(script), "--backend", backend, "--target", "audiocpp_server", "--deployment-build"] + argv += _cuda_arch_argv(backend, emit=emit) command = f"cd {audiocpp_dir} && {shlex.join(argv)}" if emit is None: print(f"[INFO] Building audiocpp_server for {backend} ({command})...") @@ -409,12 +524,23 @@ def _build_audiocpp_tui(emit, cancel, argv: List[str], command: str, return patch_rc tee(f"[INFO] Building audiocpp_server ({command})...") rc = common.run_console_subprocess( - argv, cwd=audiocpp_dir, emit=tee, cancel=cancel) + argv, cwd=audiocpp_dir, emit=tee, cancel=cancel, + stall_timeout=BUILD_STALL_TIMEOUT) if rc != 0 and (cancel is None or not cancel.is_set()): - notice = (f"[ERROR] audio.cpp build failed (exit code {rc}).\n" + if rc == 124: + head = (f"[ERROR] audio.cpp build stalled — no output for " + f"{BUILD_STALL_TIMEOUT // 60} minutes, so it was " + "stopped (a wedged compiler job; often a ptxas " + "hang from a buggy CUDA toolkit).") + else: + head = f"[ERROR] audio.cpp build failed (exit code {rc})." + notice = (f"{head}\n" f" Build log: {log_path}\n" f" Troubleshoot by re-running this command:\n" f" {command}") + ptxas_hint = _ptxas_failure_hint(log_path) + if ptxas_hint: + notice += "\n" + ptxas_hint for line in notice.splitlines(): tee(line) common.record_post_tui_notice(notice) diff --git a/app/backends/audiocpp/models.py b/app/backends/audiocpp/models.py index 75bc06a..772ae8d 100644 --- a/app/backends/audiocpp/models.py +++ b/app/backends/audiocpp/models.py @@ -11,6 +11,11 @@ from typing import Dict, List, Optional, Set, Tuple from backends import common from . import catalog as _catalog +# No-output watchdog for model downloads: huggingface_hub streams steady +# byte progress, so 5 minutes of silence means the transfer wedged. The +# runner kills it and reports exit 124 (see run_console_subprocess). +DOWNLOAD_STALL_TIMEOUT = 300 + def _installed_display_names(audiocpp_dir: Path, model_entries: Optional[List[dict]], install_guidance: List[Tuple[str, str]] @@ -133,7 +138,9 @@ def _install_models(audiocpp_dir: Path, try: rc = common.run_console_subprocess( argv, cwd=str(audiocpp_dir), emit=emit, cancel=cancel, - on_cancel=on_cancel) + on_cancel=on_cancel, + stall_timeout=(DOWNLOAD_STALL_TIMEOUT + if supports_progress else None)) except OSError as exc: print(f"[WARNING] Could not run python {manager} install " f"{install_id}: {exc}") diff --git a/app/backends/audiocpp/wizard.py b/app/backends/audiocpp/wizard.py index 46b6638..c59da8f 100644 --- a/app/backends/audiocpp/wizard.py +++ b/app/backends/audiocpp/wizard.py @@ -574,7 +574,11 @@ def _execute_lanes(settings: dict, def build_step(emit, cancel): rc = _build.build_audiocpp(audiocpp_dir, settings["backend"], emit=emit, cancel=cancel) - if rc != 0: + if rc == 124: + print("[WARNING] build went silent and was stopped; the " + "server.json was still written — build " + "audiocpp_server manually before starting it") + elif rc != 0: print(f"[WARNING] build exited with code {rc}; the server.json " "was still written — build audiocpp_server manually " "before starting it") diff --git a/app/backends/common.py b/app/backends/common.py index 4edfd61..4c768b8 100644 --- a/app/backends/common.py +++ b/app/backends/common.py @@ -347,7 +347,8 @@ def write_prompt_text(wav_dir: Path, def run_console_subprocess(argv: List[str], cwd: Optional[Path] = None, - *, emit=None, cancel=None, on_cancel=None) -> int: + *, emit=None, cancel=None, on_cancel=None, + stall_timeout: Optional[float] = None) -> int: """Run a subprocess, streaming output to the console or to EMIT. With EMIT None the child inherits the real terminal and its output @@ -360,7 +361,16 @@ def run_console_subprocess(argv: List[str], cwd: Optional[Path] = None, CANCEL is an optional ``threading.Event``: once set, ON_CANCEL (if given) is called (e.g. to touch a ``--cancel-file``), then the child's process group is terminated (SIGTERM, escalating to SIGKILL after a grace - period) and 130 is returned. Returns the process exit code. + period) and 130 is returned. + + STALL_TIMEOUT (EMIT path only) is a no-output watchdog in seconds: when + the child produces no new output line for that long, it is treated as + wedged (a build whose compiler hung, a download that stopped moving) — + the process group is terminated, an [ERROR] line is emitted, and 124 is + returned so callers can report a stall distinctly from a plain failure. + None (the default) waits forever, as before. + + Returns the process exit code. """ import subprocess if emit is None: @@ -387,12 +397,18 @@ def run_console_subprocess(argv: List[str], cwd: Optional[Path] = None, return 1 cancelled = False + stalled = False + # Written by the reader thread, read by the poll loop below: a plain + # float assignment is atomic enough under the GIL (no torn reads). + last_output = time.monotonic() def _reader() -> None: + nonlocal last_output try: for raw in iter(proc.stdout.readline, b""): if not raw: break + last_output = time.monotonic() text = raw.decode("utf-8", errors="replace") for line in text.splitlines(): if line: @@ -421,6 +437,13 @@ def run_console_subprocess(argv: List[str], cwd: Optional[Path] = None, break if proc.poll() is not None: break + if (stall_timeout is not None + and time.monotonic() - last_output > stall_timeout): + stalled = True + emit(f"[ERROR] No output for {int(stall_timeout)}s — assuming " + "the process hung; stopping it.") + _terminate_process_group(proc) + break time.sleep(0.1) try: reader.join(timeout=5) @@ -429,6 +452,8 @@ def run_console_subprocess(argv: List[str], cwd: Optional[Path] = None, reader.join(timeout=0) if cancelled: return 130 + if stalled: + return 124 return proc.returncode @@ -577,20 +602,29 @@ def _origin_default_branch(checkout: Path) -> str: def run_console_subprocess_quiet(argv: List[str], - cwd: Optional[Path] = None): + cwd: Optional[Path] = None, + timeout: Optional[float] = None): """Run ARGV silently and return the completed result. Unlike run_console_subprocess (which streams or returns only an exit code) this captures stdout and needs the process object itself, for the small git probes (rev-parse, symbolic-ref) whose *output* matters and - whose failure is a normal, non-fatal outcome. Returns None when the - process could not be started. + whose failure is a normal, non-fatal outcome. TIMEOUT bounds the wait + (e.g. for hardware probes like nvidia-smi that can hang on a wedged + driver); a timeout kills the child and returns a failed result, not an + exception. Returns None when the process could not be started. """ import subprocess try: return subprocess.run( argv, capture_output=True, - cwd=str(cwd) if cwd is not None else None, check=False) + cwd=str(cwd) if cwd is not None else None, check=False, + timeout=timeout) + except subprocess.TimeoutExpired: + class _TimedOut: + returncode = -1 + stdout = b"" + return _TimedOut() except OSError: return None diff --git a/app/tests/test_backends_audiocpp.py b/app/tests/test_backends_audiocpp.py index 6f97438..4a8ee5f 100644 --- a/app/tests/test_backends_audiocpp.py +++ b/app/tests/test_backends_audiocpp.py @@ -1407,6 +1407,189 @@ class BuildAudiocppTests(unittest.TestCase): self.assertEqual(len(notices), 1) self.assertIn("No build script found", notices[0]) + def test_cuda_build_appends_detected_arch_flag(self): + with patch.object(common, "run_console_subprocess", + return_value=0) as run, \ + patch.object(make_server.build, "detect_cuda_arch", + return_value="86"): + make_server.build.build_audiocpp(self.checkout, "cuda") + argv = run.call_args[0][0] + self.assertIn("--cuda-arch", argv) + self.assertEqual(argv[argv.index("--cuda-arch") + 1], "86") + + def test_tui_stall_notice_reports_the_hang(self): + emitted, emit = self._emit() + with patch.object(common, "LOG_DIR", self.log_dir), \ + patch.object(common, "run_console_subprocess", + return_value=124): + rc = make_server.build.build_audiocpp(self.checkout, "cuda", + emit=emit) + self.assertEqual(rc, 124) + log_text = self._log_files()[0].read_text(encoding="utf-8") + self.assertIn("stalled", log_text) + notices = common.drain_post_tui_notices() + self.assertEqual(len(notices), 1) + self.assertIn("stalled", notices[0]) + self.assertIn("stopped", notices[0]) + + def test_tui_ptxas_failure_appends_detected_arch_guidance(self): + emitted, emit = self._emit() + + def fake_run(argv, cwd=None, emit=None, cancel=None, **kwargs): + emit("ptxas fatal : (C7907) Internal compiler error.") + return 1 + + with patch.object(common, "LOG_DIR", self.log_dir), \ + patch.object(common, "run_console_subprocess", + side_effect=fake_run), \ + patch.object(make_server.build, "detect_cuda_arch", + return_value="86"): + rc = make_server.build.build_audiocpp(self.checkout, "cuda", + emit=emit) + self.assertEqual(rc, 1) + notice = common.drain_post_tui_notices()[0] + self.assertIn("CUDA toolkit bug", notice) + self.assertIn("AUDIOCPP_CUDA_ARCH", notice) + self.assertIn("Detected arch for this machine: 86", notice) + # The undetected-GPU table is not needed when detection worked. + self.assertNotIn("Arch per GPU", notice) + + def test_tui_ptxas_failure_lists_gpu_table_when_undetected(self): + emitted, emit = self._emit() + + def fake_run(argv, cwd=None, emit=None, cancel=None, **kwargs): + emit("ptxas fatal : (C7907) Internal compiler error.") + return 1 + + with patch.object(common, "LOG_DIR", self.log_dir), \ + patch.object(common, "run_console_subprocess", + side_effect=fake_run), \ + patch.object(make_server.build, "detect_cuda_arch", + return_value=None): + rc = make_server.build.build_audiocpp(self.checkout, "cuda", + emit=emit) + self.assertEqual(rc, 1) + notice = common.drain_post_tui_notices()[0] + self.assertIn("Arch per GPU", notice) + self.assertIn("3090", notice) + self.assertIn("4090", notice) + + def test_tui_plain_failure_has_no_ptxas_guidance(self): + emitted, emit = self._emit() + with patch.object(common, "LOG_DIR", self.log_dir), \ + patch.object(common, "run_console_subprocess", + return_value=3): + rc = make_server.build.build_audiocpp(self.checkout, "cuda", + emit=emit) + notice = common.drain_post_tui_notices()[0] + self.assertIn("failed (exit code 3)", notice) + self.assertNotIn("AUDIOCPP_CUDA_ARCH", notice) + + +class DetectCudaArchTests(unittest.TestCase): + """detect_cuda_arch: env override, nvidia-smi probe, None fallbacks.""" + + def setUp(self): + # Run every case without an AUDIOCPP_CUDA_ARCH leak from the + # developer's own shell. + clean = {k: v for k, v in os.environ.items() + if k != make_server.build.CUDA_ARCH_ENV} + patcher = patch.dict(os.environ, clean, clear=True) + patcher.start() + self.addCleanup(patcher.stop) + + def test_env_override_wins_without_probing(self): + with patch.dict(os.environ, + {make_server.build.CUDA_ARCH_ENV: "86"}), \ + patch.object(common, "run_console_subprocess_quiet") as run: + self.assertEqual(make_server.build.detect_cuda_arch(), "86") + run.assert_not_called() + + def test_env_override_multi_gpu_and_commas(self): + with patch.dict(os.environ, + {make_server.build.CUDA_ARCH_ENV: "86, 89;75"}): + self.assertEqual(make_server.build.detect_cuda_arch(), + "86;89;75") + + def test_env_override_real_virtual_suffixes_allowed(self): + with patch.dict(os.environ, + {make_server.build.CUDA_ARCH_ENV: "86-real"}): + self.assertEqual(make_server.build.detect_cuda_arch(), "86-real") + + def test_invalid_env_override_ignored_and_probe_runs(self): + probe = MagicMock(returncode=0, stdout=b"8.6\n") + with patch.dict(os.environ, + {make_server.build.CUDA_ARCH_ENV: "rtx"}), \ + patch.object(common, "run_console_subprocess_quiet", + return_value=probe) as run: + self.assertEqual(make_server.build.detect_cuda_arch(), "86") + self.assertEqual(run.call_args[0][0][0], "nvidia-smi") + + def test_compute_caps_parsed_and_deduped(self): + probe = MagicMock(returncode=0, stdout=b"8.6\n8.6\n12.0\n") + with patch.object(common, "run_console_subprocess_quiet", + return_value=probe): + self.assertEqual(make_server.build.detect_cuda_arch(), "86;120") + + def test_nvidia_smi_failure_yields_none(self): + probe = MagicMock(returncode=1, stdout=b"") + with patch.object(common, "run_console_subprocess_quiet", + return_value=probe): + self.assertIsNone(make_server.build.detect_cuda_arch()) + + def test_unparsable_output_yields_none(self): + probe = MagicMock(returncode=0, + stdout=b"NVIDIA-SMI has failed because...\n") + with patch.object(common, "run_console_subprocess_quiet", + return_value=probe): + self.assertIsNone(make_server.build.detect_cuda_arch()) + + def test_unstartable_probe_yields_none(self): + with patch.object(common, "run_console_subprocess_quiet", + return_value=None): + self.assertIsNone(make_server.build.detect_cuda_arch()) + + def test_probe_is_bounded_by_a_timeout(self): + probe = MagicMock(returncode=0, stdout=b"8.6\n") + with patch.object(common, "run_console_subprocess_quiet", + return_value=probe) as run: + make_server.build.detect_cuda_arch() + self.assertIsNotNone(run.call_args[1].get("timeout")) + + +class CudaArchArgvTests(unittest.TestCase): + """_cuda_arch_argv: the --cuda-arch flags and their status line.""" + + def _argv(self, backend, arch, emit=None): + with patch.object(make_server.build, "detect_cuda_arch", + return_value=arch): + return make_server.build._cuda_arch_argv(backend, emit=emit) + + def test_cuda_build_gets_the_detected_arch(self): + self.assertEqual(self._argv("cuda", "86"), + ["--cuda-arch", "86"]) + + def test_multi_gpu_arch_passed_verbatim(self): + self.assertEqual(self._argv("cuda", "86;89"), + ["--cuda-arch", "86;89"]) + + def test_detection_failure_means_no_flag(self): + self.assertEqual(self._argv("cuda", None), []) + + def test_non_cuda_backends_never_get_the_flag(self): + for backend in ("cpu", "vulkan", "hip"): + with self.subTest(backend=backend): + self.assertEqual(self._argv(backend, "86"), []) + + def test_status_lines_report_the_outcome(self): + detected, emit = [], lambda line: detected.append(line) + self._argv("cuda", "86", emit=emit) + self.assertIn("CUDA architecture: 86", detected[0]) + undetected, emit = [], lambda line: undetected.append(line) + self._argv("cuda", None, emit=emit) + self.assertIn("portable default list", undetected[0]) + self.assertIn(make_server.build.CUDA_ARCH_ENV, undetected[0]) + class AudiocppUpdateTests(unittest.TestCase): """update: stop the server, refresh the checkout, rebuild when stale. diff --git a/app/tests/test_backends_common.py b/app/tests/test_backends_common.py index a94994b..08ffc2c 100644 --- a/app/tests/test_backends_common.py +++ b/app/tests/test_backends_common.py @@ -2,8 +2,9 @@ The streaming mode of ``run_console_subprocess`` (used by the in-TUI task view) is exercised with a real child process: output lines are captured and -forwarded, cancellation kills the child and returns 130, and an on_cancel -hook runs first. +forwarded, cancellation kills the child and returns 130, an on_cancel hook +runs first, and the no-output stall watchdog kills a wedged child and +returns 124. """ import sys @@ -51,6 +52,58 @@ class RunConsoleSubprocessStreamingTests(unittest.TestCase): self.assertEqual(touched, [True]) +class RunConsoleSubprocessStallTests(unittest.TestCase): + """The no-output watchdog: a silent child is killed and reported 124.""" + + def test_stall_kills_a_silent_child_and_returns_124(self): + lines = [] + rc = common.run_console_subprocess( + [sys.executable, "-c", + "import sys, time; print('start', flush=True); " + "time.sleep(60)"], + emit=lines.append, stall_timeout=0.5) + self.assertEqual(rc, 124) + self.assertEqual(lines[0], "start") + # The stall is announced to the view before the kill. + self.assertTrue(any("[ERROR]" in line and "No output" in line + for line in lines), lines) + + def test_no_stall_while_output_keeps_flowing(self): + lines = [] + rc = common.run_console_subprocess( + [sys.executable, "-c", + "import sys, time\n" + "for _ in range(6):\n" + " print('tick', flush=True)\n" + " time.sleep(0.2)\n"], + emit=lines.append, stall_timeout=1.0) + self.assertEqual(rc, 0) + self.assertEqual(lines, ["tick"] * 6) + + def test_console_path_has_no_watchdog(self): + # Without emit the child inherits the terminal; stall_timeout is + # a no-op there (the caller sees raw output and can Ctrl-C). + rc = common.run_console_subprocess( + [sys.executable, "-c", "print('hi')"], stall_timeout=0.001) + self.assertEqual(rc, 0) + + +class RunConsoleSubprocessQuietTimeoutTests(unittest.TestCase): + """A timed-out quiet probe returns a failed result, never raises.""" + + def test_timeout_returns_failed_result(self): + proc = common.run_console_subprocess_quiet( + [sys.executable, "-c", "import time; time.sleep(30)"], + timeout=0.5) + self.assertIsNotNone(proc) + self.assertEqual(proc.returncode, -1) + + def test_untimed_probe_still_reports_the_exit_code(self): + proc = common.run_console_subprocess_quiet( + [sys.executable, "-c", "import sys; sys.exit(5)"]) + self.assertEqual(proc.returncode, 5) + + class GitCloneTests(unittest.TestCase): def test_git_clone_console_passes_through(self): with mock.patch.object(common, "run_console_subprocess", diff --git a/app/tests/test_taskview.py b/app/tests/test_taskview.py index b7f4389..646fccb 100644 --- a/app/tests/test_taskview.py +++ b/app/tests/test_taskview.py @@ -592,5 +592,91 @@ class LanesViewTests(_FakeTui, unittest.TestCase): self.assertLessEqual(x + len(label) - 1, 1 + pane_w - 2) +class SilenceCueTests(_FakeTui, unittest.TestCase): + """The "(no output 6m)" cue for a running step that stopped emitting.""" + + def test_silence_cue_text_format(self): + self.assertEqual(taskview._silence_cue_text(90), "(no output 1m)") + self.assertEqual(taskview._silence_cue_text(3700), "(no output 61m)") + + def test_no_cue_while_output_is_recent(self): + view, _ = self.make_view(steps=[_step("one")]) + view.handle_event({"kind": "step_start", "index": 0, "title": "one"}) + view._ingest_line("working") + self.assertIsNone(taskview._silent_secs(view.last_line_at, + view.last_line_at + 59)) + + def test_cue_after_the_threshold(self): + view, _ = self.make_view(steps=[_step("one")]) + view.handle_event({"kind": "step_start", "index": 0, "title": "one"}) + view._ingest_line("working") + self.assertEqual(taskview._silent_secs(view.last_line_at, + view.last_line_at + 60), 60) + + def test_step_start_resets_and_done_clears_the_tracking(self): + view, _ = self.make_view(steps=[_step("one"), _step("two")]) + view.handle_event({"kind": "step_start", "index": 0, "title": "one"}) + start = view.last_line_at + view.handle_event({"kind": "step_done", "index": 0, "rc": 0}) + self.assertIsNone(view.last_line_at) + view.handle_event({"kind": "step_start", "index": 1, "title": "two"}) + self.assertGreaterEqual(view.last_line_at, start) + + def test_render_shows_the_cue_for_a_silent_step(self): + now = [1000.0] + + def clock(): + return now[0] + + screen = FakeScreen(width=80, height=24) + with patch.object(taskview.TaskView, "_worker_main", + lambda self: None): + view = taskview.TaskView(screen, "Setup", [_step("one")], + clock=clock) + view.handle_event({"kind": "step_start", "index": 0, "title": "one"}) + view._ingest_line("working") + now[0] = view.last_line_at + 300 + view.render() + text = " ".join(t for _, _, t, _ in screen.strings) + self.assertIn("(no output 5m)", text) + + def test_render_hides_the_cue_when_output_is_fresh(self): + now = [1000.0] + + def clock(): + return now[0] + + screen = FakeScreen(width=80, height=24) + with patch.object(taskview.TaskView, "_worker_main", + lambda self: None): + view = taskview.TaskView(screen, "Setup", [_step("one")], + clock=clock) + view.handle_event({"kind": "step_start", "index": 0, "title": "one"}) + view._ingest_line("working") + now[0] = view.last_line_at + 5 + view.render() + text = " ".join(t for _, _, t, _ in screen.strings) + self.assertNotIn("(no output", text) + + def test_lane_pane_shows_the_cue_for_a_silent_lane(self): + now = [1000.0] + + def clock(): + return now[0] + + screen = FakeScreen(width=80, height=24) + view = taskview.LanesView( + screen, "Setup", + [taskview.TaskLane("Build", [_step("one")])], clock=clock) + lane = view._lanes[0] + view._handle_lane_event( + lane, {"kind": "step_start", "index": 0, "title": "one"}) + view._ingest_lane_line(lane, "working") + now[0] = lane.last_line_at + 120 + view.render() + text = " ".join(t for _, _, t, _ in screen.strings) + self.assertIn("(no output 2m)", text) + + if __name__ == "__main__": unittest.main() diff --git a/app/ui/taskview.py b/app/ui/taskview.py index c5c2ff7..d7aed96 100644 --- a/app/ui/taskview.py +++ b/app/ui/taskview.py @@ -63,6 +63,11 @@ _PROGRESS_COUNT = re.compile(r"\[(\d+)/(\d+)\]") # A spinner frame set for the running step marker. _SPINNER = ("|", "/", "-", "\\") +# How long a running step may stay silent (no output lines) before the +# view starts saying so next to its elapsed clock — an early "this looks +# wedged" cue for the no-output watchdog that eventually kills the step. +_SILENCE_CUE_SECS = 60 + @dataclass class TaskStep: @@ -117,6 +122,27 @@ def _progress_match(text: str) -> Optional[Tuple[float, float, str]]: return None +def _silent_secs(last_line_at: Optional[float], now: float + ) -> Optional[float]: + """Seconds the running step has been silent, or None (no cue). + + None when nothing is tracked (no output yet is not tracked here — the + caller decides), or when the silence is still below the cue threshold. + Module-level for testability. + """ + if last_line_at is None: + return None + silent = now - last_line_at + if silent < _SILENCE_CUE_SECS: + return None + return silent + + +def _silence_cue_text(silent: float) -> str: + """The dim text shown next to the elapsed clock for SILENT seconds.""" + return f"(no output {int(silent // 60)}m)" + + def _lane_step_mark(current: Optional[int], results: List[Optional[int]], cancelled_step: Optional[int], @@ -208,6 +234,7 @@ class TaskView(ScreenView): self._progress: Optional[Tuple[float, float]] = None # (done, total) self._progress_kind = "" # "bytes" | "percent" | "count" | "" self.step_started: List[Optional[float]] = [None] * len(steps) + self.last_line_at: Optional[float] = None # silence cue (see _SILENCE_CUE_SECS) self.finished_at: Optional[float] = None self.cancelled = False self.cancelling = False @@ -265,6 +292,7 @@ class TaskView(ScreenView): if kind == "step_start": self.current = event["index"] self.step_started[self.current] = self._now() + self.last_line_at = self._now() self._progress = None self._progress_kind = "" elif kind == "line": @@ -274,11 +302,13 @@ class TaskView(ScreenView): index = event["index"] self.results[index] = event.get("rc") or 0 self.current = None + self.last_line_at = None self._progress = None self._progress_kind = "" elif kind == "step_cancelled": self.cancelled_step = event["index"] self.current = None + self.last_line_at = None self._progress = None self._progress_kind = "" elif kind == "finish": @@ -290,6 +320,7 @@ class TaskView(ScreenView): def _ingest_line(self, text: str) -> None: """Fold one output line into the log tail and progress bar.""" line = text.rstrip("\r\n") + self.last_line_at = self._now() if not line: return match = _progress_match(line) @@ -371,9 +402,14 @@ class TaskView(ScreenView): _text(scr, theme, y, inner_x + 5, label, theme["body"]) if index == self.current and self.phase not in _TERMINAL: started = self.step_started[index] or self._now() + elapsed_text = f" {_format_elapsed(self._now() - started)}" _text(scr, theme, y, inner_x + 5 + len(label) + 1, - f" {_format_elapsed(self._now() - started)}", - theme["dim"]) + elapsed_text, theme["dim"]) + silent = _silent_secs(self.last_line_at, self._now()) + if silent is not None: + _text(scr, theme, y, + inner_x + 5 + len(label) + 1 + len(elapsed_text), + f" {_silence_cue_text(silent)}", theme["dim"]) y += 1 y += 1 @@ -581,6 +617,7 @@ class _LaneState: self.progress: Optional[Tuple[float, float]] = None self.progress_kind = "" self.step_started: List[Optional[float]] = [None] * len(self.steps) + self.last_line_at: Optional[float] = None # silence cue self.cancelled_step: Optional[int] = None self.rc = 0 self.finished = False @@ -664,6 +701,7 @@ class LanesView(_GetchModes): if kind == "step_start": lane.current = event["index"] lane.step_started[lane.current] = self._now() + lane.last_line_at = self._now() lane.progress = None lane.progress_kind = "" elif kind == "line": @@ -671,11 +709,13 @@ class LanesView(_GetchModes): elif kind == "step_done": lane.results[event["index"]] = event.get("rc") or 0 lane.current = None + lane.last_line_at = None lane.progress = None lane.progress_kind = "" elif kind == "step_cancelled": lane.cancelled_step = event["index"] lane.current = None + lane.last_line_at = None lane.progress = None lane.progress_kind = "" elif kind == "lane_finish": @@ -685,6 +725,7 @@ class LanesView(_GetchModes): def _ingest_lane_line(self, lane: _LaneState, text: str) -> None: """Fold one output line into LANE's log tail and progress bar.""" line = text.rstrip("\r\n") + lane.last_line_at = self._now() if not line: return match = _progress_match(line) @@ -862,9 +903,14 @@ class LanesView(_GetchModes): _text(scr, theme, row, x + 6, label, theme["body"]) if index == lane.current and not terminal: started = lane.step_started[index] or self._now() + elapsed_text = f" {_format_elapsed(self._now() - started)}" _text(scr, theme, row, x + 6 + len(label) + 1, - f" {_format_elapsed(self._now() - started)}", - theme["dim"]) + elapsed_text, theme["dim"]) + silent = _silent_secs(lane.last_line_at, self._now()) + if silent is not None: + _text(scr, theme, row, + x + 6 + len(label) + 1 + len(elapsed_text), + f" {_silence_cue_text(silent)}", theme["dim"]) row += 1 row += 1 -- cgit v1.2.3