feat: tui backend server progress and generate script progress

1 files changed, 149 insertions, 45 deletions

diff --git a/app/backends/servers.py b/app/backends/servers.py
index 912c09d..62eadb5 100644
--- a/app/backends/servers.py
+++ b/app/backends/servers.py
@@ -1,16 +1,25 @@
 """Start and stop TTS backend servers from the TUI hub.
 
 Each backend's ``detect()`` returns a list of ``ServerSpec`` — the exact argv
-(absolute binaries in the managed venv, no shell activation needed) and the
-URL to probe for readiness. This module turns those specs into running
-processes: ``start`` spawns the server, streams its output to
-``app/logs/<name>-server.log``, records its pid, and polls the URL until it
-accepts connections (model loads are slow, so the timeout is generous);
-``stop`` terminates the process group the hub started.
-
-Everything here runs in the plain console tail after the curses TUI returns
-(matching the wizards' build/pip streaming), so progress and log tails appear
-normally. Pid/log files live under ``app/logs/`` which is already gitignored.
+(absolute binaries in the managed venv, no shell activation needed), the
+working directory to spawn it in, and the URL to probe for readiness. This
+module turns those specs into running processes: ``start`` spawns the server
+(streaming its output to ``app/logs/<name>-server.log``, in the spec's cwd
+when it has one — audio.cpp resolves model_specs/ relative to its process
+working directory), records its pid, and polls the URL until it accepts
+connections (model loads are slow, so the timeout is generous). With a spec
+``identity`` the poll also verifies the server answers HTTP as that backend,
+so readiness means "serving", not just "listening".
+
+Progress reporting goes through an optional ``progress`` callback (see
+``_console_progress`` for the event shapes); the default callback prints the
+same lines as before, so the plain-console flow is unchanged. ``stop``
+terminates the process group the hub started.
+
+Everything here can run in the plain console tail after the curses TUI
+returns (matching the wizards' build/pip streaming) or behind the run view's
+boot screen, which renders the same events. Pid/log files live under
+``app/logs/`` which is already gitignored.
 """
 
 import os
@@ -19,9 +28,9 @@ import subprocess
 import sys
 import time
 from pathlib import Path
-from typing import List
+from typing import Callable, List, Optional
 
-from backends import common
+from backends import common, probe
 from backends.common import APP_DIR
 
 LOG_DIR = APP_DIR / "logs"
@@ -34,6 +43,48 @@ SERVER_START_TIMEOUT = 600
 # Grace period after SIGTERM before escalating to SIGKILL (POSIX).
 STOP_GRACE_SECONDS = 10
 
+# How often the start poll re-checks readiness (seconds).
+POLL_INTERVAL = 1
+
+# Progress callback: called with an event dict. KIND is one of:
+#   "starting"  {name, argv, cwd, log_path, pid}  spawned, waiting for boot
+#   "elapsed"   {name, seconds}                   heartbeat while waiting
+#   "ready"     {name, url}                       server is up and answering
+#   "exited"    {name, returncode, log_tail}      process exited while booting
+#   "timeout"   {name, seconds, log_tail}         readiness deadline elapsed
+#   "running"   {name, url}                       already up (no spawn)
+#   "cancelled" {name}                            boot aborted via cancel
+#   "error"     {message}                         could not spawn the executable
+ProgressCallback = Optional[Callable[[dict], None]]
+
+
+def _console_progress(event: dict) -> None:
+    """Print PROGRESS events as the plain-console output (the old behavior)."""
+    kind = event.get("kind")
+    if kind == "starting":
+        print(f"[INFO] starting {event['name']} server: {event['argv']}")
+        if event.get("cwd"):
+            print(f"[INFO] working directory: {event['cwd']}")
+        print(f"[INFO] pid {event['pid']}; logs: {event['log_path']}")
+    elif kind == "elapsed":
+        print(f"[INFO] still waiting for the server ({int(event['seconds'])}s)...")
+    elif kind == "ready":
+        print(f"[OK] {event['name']} server is up on {event['url']}")
+    elif kind == "running":
+        print(f"[INFO] {event['name']} server already running on {event['url']}")
+    elif kind == "cancelled":
+        print(f"[INFO] {event['name']} server start cancelled")
+    elif kind == "exited":
+        print(f"[ERROR] {event['name']} server exited with code "
+              f"{event['returncode']}")
+        _print_tail(event.get("log_tail"))
+    elif kind == "timeout":
+        print(f"[ERROR] {event['name']} server did not start within "
+              f"{int(event['seconds'])}s")
+        _print_tail(event.get("log_tail"))
+    elif kind == "error":
+        print(f"[ERROR] {event['message']}")
+
 
 def _log_path(name: str) -> Path:
     return LOG_DIR / f"{name}-server.log"
@@ -43,17 +94,20 @@ def _pid_path(name: str) -> Path:
     return LOG_DIR / f"{name}-server.pid"
 
 
-def _tail_log(name: str, lines: int = 20) -> None:
-    """Print the last LINES of the server's log (best-effort)."""
-    path = _log_path(name)
+def _read_log_tail(name: str, lines: int = 20) -> List[str]:
+    """Return the last LINES of the server's log (best-effort)."""
     try:
-        text = path.read_text(encoding="utf-8", errors="replace")
+        text = _log_path(name).read_text(encoding="utf-8", errors="replace")
     except OSError:
-        return
-    tail = "\n".join(text.splitlines()[-lines:])
+        return []
+    return text.splitlines()[-lines:]
+
+
+def _print_tail(tail: List[str]) -> None:
+    """Print a log-tail event payload (used by the console callback)."""
     if tail:
-        print(f"--- last {lines} lines of {path} ---")
-        print(tail)
+        print(f"--- last {len(tail)} lines of the server log ---")
+        print("\n".join(tail))
         print("---")
 
 
@@ -123,24 +177,53 @@ def _kill_pid(pid: int) -> bool:
     return True
 
 
-def start(spec) -> bool:
+def _server_ready(spec) -> bool:
+    """True when the server described by SPEC is usable, not just listening.
+
+    Without an IDENTITY this is the plain TCP-connect check. With one, the
+    server must also answer HTTP as that backend (``probe.identify_server``);
+    for the faster backend (whose model loads after the port opens) the
+    ``/health`` model_loaded flag must additionally be true.
+    """
+    if not common.server_running(spec.url):
+        return False
+    identity = getattr(spec, "identity", None)
+    if identity is None:
+        return True
+    if probe.identify_server(spec.url) != identity:
+        return False
+    if identity == probe.IDENTITY_FASTER:
+        return probe.faster_model_loaded(spec.url)
+    return True
+
+
+def start(spec, progress: ProgressCallback = None,
+          cancel=None) -> bool:
     """Start the server described by SPEC (a ``backends.ServerSpec``).
 
-    Spawns its argv with stdout/stderr to ``logs/<name>-server.log``, records
-    the pid, and polls ``common.server_running(spec.url)`` until it accepts
-    connections or ``SERVER_START_TIMEOUT`` elapses. Returns True when the
-    server is up; on timeout or early exit, prints the log tail and returns
+    Spawns its argv with stdout/stderr to ``logs/<name>-server.log``, in the
+    spec's CWD when it has one (audio.cpp discovers model_specs/ from its
+    process working directory), records the pid, and polls readiness —
+    ``_server_ready``, so an IDENTITY spec must actually answer HTTP — until
+    it is up or ``SERVER_START_TIMEOUT`` elapses. Returns True when the
+    server is up; on timeout or early exit reports the log tail and returns
     False. A no-op (True) when the server is already running.
+
+    PROGRESS, when given, receives each boot event (see ProgressCallback);
+    the default ``_console_progress`` prints them, preserving the old
+    console output. CANCEL (a threading.Event) aborts the boot: the spawned
+    process is terminated and False is reported (event kind "cancelled").
     """
+    report = progress if progress is not None else _console_progress
     argv: List[str] = list(spec.argv)
     exe = Path(argv[0])
     if not exe.exists():
-        print(f"[ERROR] server executable not found: {exe}")
-        print("        run 'Set up a backend' for "
-              f"{spec.name!r} first.")
+        report({"kind": "error",
+                "message": f"server executable not found: {exe}. Run 'Set "
+                           "up a backend' first."})
         return False
-    if common.server_running(spec.url):
-        print(f"[INFO] {spec.name} server already running on {spec.url}")
+    if _server_ready(spec):
+        report({"kind": "running", "name": spec.name, "url": spec.url})
         return True
 
     LOG_DIR.mkdir(parents=True, exist_ok=True)
@@ -151,10 +234,11 @@ def start(spec) -> bool:
         except OSError:
             pass
 
-    print(f"[INFO] starting {spec.name} server: "
-          + " ".join(str(a) for a in argv))
+    cwd = getattr(spec, "cwd", None)
     log_handle = _log_path(spec.name).open("w", encoding="utf-8")
     popen_kwargs = {"stdout": log_handle, "stderr": subprocess.STDOUT}
+    if cwd is not None:
+        popen_kwargs["cwd"] = str(cwd)
     if sys.platform == "win32":
         popen_kwargs["creationflags"] = \
             subprocess.CREATE_NEW_PROCESS_GROUP  # type: ignore[attr-defined]
@@ -163,31 +247,51 @@ def start(spec) -> bool:
     try:
         proc = subprocess.Popen(argv, **popen_kwargs)
     except OSError as exc:
-        print(f"[ERROR] could not start server: {exc}")
+        report({"kind": "error",
+                "message": f"could not start server: {exc}"})
         log_handle.close()
         return False
 
     pid_file.write_text(str(proc.pid), encoding="utf-8")
-    print(f"[INFO] pid {proc.pid}; logs: {_log_path(spec.name)}")
-
-    deadline = time.time() + SERVER_START_TIMEOUT
+    report({"kind": "starting", "name": spec.name,
+            "argv": " ".join(str(a) for a in argv),
+            "cwd": str(cwd) if cwd is not None else None,
+            "log_path": str(_log_path(spec.name)), "pid": proc.pid})
+
+    started = time.time()
+    next_heartbeat = started + 15
+    deadline = started + SERVER_START_TIMEOUT
     while time.time() < deadline:
+        if cancel is not None and cancel.is_set():
+            # User cancelled while booting: kill what we spawned (the
+            # server we started is not left loading in the background).
+            _kill_pid(proc.pid)
+            try:
+                pid_file.unlink()
+            except OSError:
+                pass
+            report({"kind": "cancelled", "name": spec.name})
+            return False
         if proc.poll() is not None:
-            print(f"[ERROR] {spec.name} server exited with code "
-                  f"{proc.returncode}")
-            _tail_log(spec.name)
+            report({"kind": "exited", "name": spec.name,
+                    "returncode": proc.returncode,
+                    "log_tail": _read_log_tail(spec.name)})
             try:
                 pid_file.unlink()
             except OSError:
                 pass
             return False
-        if common.server_running(spec.url):
-            print(f"[OK] {spec.name} server is up on {spec.url}")
+        if _server_ready(spec):
+            report({"kind": "ready", "name": spec.name, "url": spec.url})
             return True
-        time.sleep(1)
-    print(f"[ERROR] {spec.name} server did not start within "
-          f"{SERVER_START_TIMEOUT}s")
-    _tail_log(spec.name)
+        if time.time() >= next_heartbeat:
+            report({"kind": "elapsed",
+                    "seconds": time.time() - started})
+            next_heartbeat += 15
+        time.sleep(POLL_INTERVAL)
+    report({"kind": "timeout", "name": spec.name,
+            "seconds": SERVER_START_TIMEOUT,
+            "log_tail": _read_log_tail(spec.name)})
     # Leave the pid file in place so stop() can kill it (it may still load).
     return False