diff options
Diffstat (limited to 'app/backends/common.py')
| -rw-r--r-- | app/backends/common.py | 46 |
1 files changed, 40 insertions, 6 deletions
diff --git a/app/backends/common.py b/app/backends/common.py index 4edfd61..4c768b8 100644 --- a/app/backends/common.py +++ b/app/backends/common.py @@ -347,7 +347,8 @@ def write_prompt_text(wav_dir: Path, def run_console_subprocess(argv: List[str], cwd: Optional[Path] = None, - *, emit=None, cancel=None, on_cancel=None) -> int: + *, emit=None, cancel=None, on_cancel=None, + stall_timeout: Optional[float] = None) -> int: """Run a subprocess, streaming output to the console or to EMIT. With EMIT None the child inherits the real terminal and its output @@ -360,7 +361,16 @@ def run_console_subprocess(argv: List[str], cwd: Optional[Path] = None, CANCEL is an optional ``threading.Event``: once set, ON_CANCEL (if given) is called (e.g. to touch a ``--cancel-file``), then the child's process group is terminated (SIGTERM, escalating to SIGKILL after a grace - period) and 130 is returned. Returns the process exit code. + period) and 130 is returned. + + STALL_TIMEOUT (EMIT path only) is a no-output watchdog in seconds: when + the child produces no new output line for that long, it is treated as + wedged (a build whose compiler hung, a download that stopped moving) — + the process group is terminated, an [ERROR] line is emitted, and 124 is + returned so callers can report a stall distinctly from a plain failure. + None (the default) waits forever, as before. + + Returns the process exit code. """ import subprocess if emit is None: @@ -387,12 +397,18 @@ def run_console_subprocess(argv: List[str], cwd: Optional[Path] = None, return 1 cancelled = False + stalled = False + # Written by the reader thread, read by the poll loop below: a plain + # float assignment is atomic enough under the GIL (no torn reads). + last_output = time.monotonic() def _reader() -> None: + nonlocal last_output try: for raw in iter(proc.stdout.readline, b""): if not raw: break + last_output = time.monotonic() text = raw.decode("utf-8", errors="replace") for line in text.splitlines(): if line: @@ -421,6 +437,13 @@ def run_console_subprocess(argv: List[str], cwd: Optional[Path] = None, break if proc.poll() is not None: break + if (stall_timeout is not None + and time.monotonic() - last_output > stall_timeout): + stalled = True + emit(f"[ERROR] No output for {int(stall_timeout)}s — assuming " + "the process hung; stopping it.") + _terminate_process_group(proc) + break time.sleep(0.1) try: reader.join(timeout=5) @@ -429,6 +452,8 @@ def run_console_subprocess(argv: List[str], cwd: Optional[Path] = None, reader.join(timeout=0) if cancelled: return 130 + if stalled: + return 124 return proc.returncode @@ -577,20 +602,29 @@ def _origin_default_branch(checkout: Path) -> str: def run_console_subprocess_quiet(argv: List[str], - cwd: Optional[Path] = None): + cwd: Optional[Path] = None, + timeout: Optional[float] = None): """Run ARGV silently and return the completed result. Unlike run_console_subprocess (which streams or returns only an exit code) this captures stdout and needs the process object itself, for the small git probes (rev-parse, symbolic-ref) whose *output* matters and - whose failure is a normal, non-fatal outcome. Returns None when the - process could not be started. + whose failure is a normal, non-fatal outcome. TIMEOUT bounds the wait + (e.g. for hardware probes like nvidia-smi that can hang on a wedged + driver); a timeout kills the child and returns a failed result, not an + exception. Returns None when the process could not be started. """ import subprocess try: return subprocess.run( argv, capture_output=True, - cwd=str(cwd) if cwd is not None else None, check=False) + cwd=str(cwd) if cwd is not None else None, check=False, + timeout=timeout) + except subprocess.TimeoutExpired: + class _TimedOut: + returncode = -1 + stdout = b"" + return _TimedOut() except OSError: return None |