feat: tui backend server progress and generate script progress

6 files changed, 307 insertions, 58 deletions

diff --git a/app/backends/__init__.py b/app/backends/__init__.py
index 9e8bf31..488c36e 100644
--- a/app/backends/__init__.py
+++ b/app/backends/__init__.py
@@ -24,6 +24,7 @@ automatically.
 
 import shlex
 from dataclasses import dataclass, field
+from pathlib import Path
 from typing import Callable, Dict, List, Optional
 
 
@@ -35,10 +36,24 @@ class ServerSpec:
     on separate ports). ARGV is the exact command line the hub spawns (using
     the managed venv's absolute binaries, so no shell activation is needed);
     URL is the endpoint ``common.server_running`` probes to decide readiness.
+
+    CWD is the working directory the server is spawned in. It matters for
+    servers that discover resources relative to their process working
+    directory (audio.cpp resolves ``model_specs/<family>.json`` by walking up
+    from its cwd), so the hub starts them from their checkout root. None
+    inherits the hub's cwd (today's behavior).
+
+    IDENTITY is the ``backends.probe.IDENTITY_*`` constant the server is
+    expected to answer as once it is truly ready. When set, ``servers.start``
+    waits for the server to answer HTTP with that identity — not merely to
+    accept TCP connections — so "listening but still starting" servers are
+    caught. None keeps the plain TCP-connect readiness check.
     """
     name: str
     url: str
     argv: List[str]
+    cwd: Optional[Path] = None
+    identity: Optional[str] = None
 
 
 @dataclass
@@ -72,6 +87,12 @@ class BackendStatus:
     RUNNING_MODELS names which of a multi-server backend's models answered
     (qwen: "Base" and/or "CustomVoice", local and remote combined), shown in
     parentheses in the hub's status table.
+
+    MODELS_MISSING says the server config references model files that are not
+    on disk (e.g. an audio.cpp server.json entry whose ``path`` was never
+    downloaded); DETAILS then names them. The backend still counts as ready
+    (the hub surfaces the warning), but a conversion would fail until the
+    models are installed.
     """
     key: str
     label: str
@@ -86,6 +107,7 @@ class BackendStatus:
     remote: bool = False
     remote_urls: Dict[str, str] = field(default_factory=dict)
     remote_models: List[str] = field(default_factory=list)
+    models_missing: bool = False
 
     @property
     def ready(self) -> bool:
@@ -94,8 +116,18 @@ class BackendStatus:
 
 
 def format_launch_hint(servers: List[ServerSpec]) -> str:
-    """Join a backend's server argvs into a copy-pasteable launch hint."""
-    return " ; ".join(shlex.join(s.argv) for s in servers)
+    """Join a backend's server argvs into a copy-pasteable launch hint.
+
+    A server with a CWD is prefixed with ``cd <cwd> &&`` so the hint works
+    pasted into a shell (the server relies on that working directory).
+    """
+    parts = []
+    for server in servers:
+        command = shlex.join(server.argv)
+        if server.cwd is not None:
+            command = f"cd {shlex.quote(str(server.cwd))} && {command}"
+        parts.append(command)
+    return " ; ".join(parts)
 
 
 @dataclass
diff --git a/app/backends/audiocpp.py b/app/backends/audiocpp.py
index ee0ee21..a502c8f 100755
--- a/app/backends/audiocpp.py
+++ b/app/backends/audiocpp.py
@@ -1276,6 +1276,78 @@ def _wizard(stdscr, args: argparse.Namespace, parser: argparse.ArgumentParser
             }
 
 
+def _model_path_present(path: Path) -> bool:
+    """True when a server.json model path holds actual model files.
+
+    A present path is either a file (a single-model package) or a non-empty
+    directory (the usual GGUF package target directory; an empty one means a
+    download that never ran or was cleaned up halfway).
+    """
+    try:
+        if path.is_file():
+            return True
+        if path.is_dir():
+            return any(path.iterdir())
+    except OSError:
+        return False
+    return False
+
+
+def missing_model_entries(server_json: Path) -> List[dict]:
+    """Return the server.json model entries whose files are not on disk.
+
+    Paths resolve exactly like audiocpp_server resolves them (relative paths
+    against the server.json's directory). Each returned entry carries the
+    entry ``id`` and ``rel`` (the configured path string); used by ``detect``
+    to warn that a conversion would fail until the models are installed.
+    """
+    try:
+        data = json.loads(server_json.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return []
+    if not isinstance(data, dict):
+        return []
+    base = server_json.parent
+    missing: List[dict] = []
+    for entry in data.get("models") or []:
+        if not isinstance(entry, dict):
+            continue
+        rel = entry.get("path")
+        if not isinstance(rel, str) or not rel:
+            continue
+        path = Path(rel) if Path(rel).is_absolute() else base / rel
+        if _model_path_present(path):
+            continue
+        missing.append({"id": str(entry.get("id") or rel), "rel": rel})
+    return missing
+
+
+def model_install_hints(audiocpp_dir: Path,
+                        missing: List[dict]) -> List[str]:
+    """Remediation lines for MISSING model entries (see missing_model_entries).
+
+    Maps each entry's configured path back to the catalog package that
+    installs it (``models/<target_directory>`` -> install id) so the line
+    carries the exact ``model_manager_v2.py install`` command; entries whose
+    directory matches no catalog package just name the path.
+    """
+    by_path: Dict[str, str] = {}
+    try:
+        for entry in load_model_catalog(audiocpp_dir):
+            by_path[entry["default_path"]] = entry["install_id"]
+    except (NotADirectoryError, OSError):
+        pass
+    hints: List[str] = []
+    for item in missing:
+        install_id = by_path.get(item["rel"])
+        hint = f"model not downloaded: {item['id']} ({item['rel']})"
+        if install_id:
+            hint += (f" — install with: python tools/model_manager_v2.py "
+                     f"install {install_id}")
+        hints.append(hint)
+    return hints
+
+
 def find_local_checkout() -> Optional[Path]:
     """Best-effort location of an audio.cpp checkout with model_specs.
 
@@ -1421,20 +1493,24 @@ def build_audiocpp(audiocpp_dir: Path, backend: str) -> int:
 
 
 def _print_launch_hint(audiocpp_dir: Path, output_path: Path) -> None:
-    """Print the exact command to start the server (or build guidance)."""
+    """Print the exact command to start the server (or build guidance).
+
+    The command is prefixed with ``cd <checkout> &&`` because the server
+    discovers model_specs/<family>.json relative to its working directory.
+    """
     binary = find_audiocpp_server_bin(audiocpp_dir)
     print()
     if binary is not None:
         print("Start the server with:")
-        print(f"  {binary} --config {output_path}")
+        print(f"  cd {audiocpp_dir} && {binary} --config {output_path}")
     else:
         print("[INFO] audiocpp_server binary not found. Build it first, e.g.:")
         script = find_build_script(audiocpp_dir)
         if script is not None:
             print(f"  sh {script} --backend <cuda|vulkan|hip|cpu> "
                   "--target audiocpp_server")
-        print(f"  then run: ./build/<platform>-<backend>-release/bin/"
-              f"audiocpp_server --config {output_path}")
+        print(f"  then run: cd {audiocpp_dir} && ./build/<platform>-<backend>"
+              f"-release/bin/audiocpp_server --config {output_path}")
 
 
 def _execute(settings: dict, args: argparse.Namespace) -> int:
@@ -1767,15 +1843,23 @@ def detect() -> BackendStatus:
     server_json = checkout / "server.json"
     configured = server_json.exists()
     specs: List[ServerSpec] = []
+    missing = missing_model_entries(server_json) if configured else []
     if configured:
         details.append(f"config: {server_json}")
+        if missing:
+            # The config references model files that are not on disk; a
+            # conversion would fail at model-load time, so say so now.
+            details.extend(model_install_hints(checkout, missing))
         if built:
+            # Spawned from the checkout: audiocpp_server discovers
+            # model_specs/<family>.json relative to its working directory.
             specs = [ServerSpec(
                 "audiocpp", config.AUDIOCPP_API_URL,
-                [str(binary), "--config", str(server_json)])]
+                [str(binary), "--config", str(server_json)],
+                cwd=checkout, identity=probe.IDENTITY_AUDIOCPP)]
         else:
-            launch = (f"./build/<platform>-<backend>-release/bin/"
-                      f"audiocpp_server --config {server_json}")
+            launch = (f"cd {checkout} && ./build/<platform>-<backend>-release"
+                      f"/bin/audiocpp_server --config {server_json}")
     else:
         details.append("no server.json — run setup to configure models")
     if specs:
@@ -1787,7 +1871,8 @@ def detect() -> BackendStatus:
                          running=managed or remote_running,
                          details=details, launch_hint=launch,
                          servers=specs, managed=managed,
-                         remote=remote_running, remote_urls=remote_urls)
+                         remote=remote_running, remote_urls=remote_urls,
+                         models_missing=bool(missing))
 
 
 def _detect_remote(managed: bool = False) -> Tuple[bool, dict]:
diff --git a/app/backends/faster.py b/app/backends/faster.py
index 3dc30dd..e1249ca 100755
--- a/app/backends/faster.py
+++ b/app/backends/faster.py
@@ -362,7 +362,10 @@ def detect() -> BackendStatus:
         argv = [str(envs.env_python()),
                 str(_checkout() / "examples" / "openai_server.py"),
                 "--voices", str(voices_json), "--port", str(_config_port())]
-        specs = [ServerSpec("faster", config.FASTER_API_URL, argv)]
+        # identity: /health must report model_loaded before the server is
+        # really usable (the model loads after the port opens).
+        specs = [ServerSpec("faster", config.FASTER_API_URL, argv,
+                            identity=probe.IDENTITY_FASTER)]
         launch = format_launch_hint(specs)
     managed = servers.manages(specs)
     remote_running, remote_urls = _detect_remote(managed)
diff --git a/app/backends/probe.py b/app/backends/probe.py
index ebef86e..efa2963 100644
--- a/app/backends/probe.py
+++ b/app/backends/probe.py
@@ -126,3 +126,26 @@ def same_endpoint(url_a: str, url_b: str) -> bool:
     port_a = a.port or (443 if (a.scheme or "http") == "https" else 80)
     port_b = b.port or (443 if (b.scheme or "http") == "https" else 80)
     return host_a == host_b and port_a == port_b
+
+
+def health_payload(url: str, timeout: float = DEFAULT_TIMEOUT) -> Optional[dict]:
+    """Return the server's ``/health`` JSON document, or None.
+
+    A cheaper, raw check than ``identify_server``: used by the run view's
+    background poll to tell "process alive" from "server answering" without
+    probing every identity endpoint.
+    """
+    if not url:
+        return None
+    return _get_json(f"{url.rstrip('/')}/health", timeout)
+
+
+def faster_model_loaded(url: str, timeout: float = DEFAULT_TIMEOUT) -> bool:
+    """True when a faster server at URL reports its model loaded.
+
+    faster's ``/health`` answers with a ``model_loaded`` flag only after the
+    weights are resident, so this is the "truly ready" signal used while
+    waiting for a started server to become usable.
+    """
+    payload = health_payload(url, timeout)
+    return bool(payload and payload.get("model_loaded"))
diff --git a/app/backends/qwen.py b/app/backends/qwen.py
index 160c0f1..7f821fa 100644
--- a/app/backends/qwen.py
+++ b/app/backends/qwen.py
@@ -227,10 +227,12 @@ def detect() -> BackendStatus:
     specs = [
         ServerSpec("qwen-custom", config.QWEN_API_URL,
                    [demo, QWEN_CUSTOMVOICE_MODEL, "--ip", "127.0.0.1",
-                    "--port", str(custom_port)]),
+                    "--port", str(custom_port)],
+                   identity=probe.IDENTITY_QWEN_CUSTOM),
         ServerSpec("qwen-clone", config.CLONE_API_URL,
                    [demo, QWEN_BASE_MODEL, "--ip", "127.0.0.1",
-                    "--port", str(clone_port)]),
+                    "--port", str(clone_port)],
+                   identity=probe.IDENTITY_QWEN_CLONE),
     ]
     managed = servers.manages(specs)
     # Which local servers this tool started (pid alive) name the running
diff --git a/app/backends/servers.py b/app/backends/servers.py
index 912c09d..62eadb5 100644
--- a/app/backends/servers.py
+++ b/app/backends/servers.py
@@ -1,16 +1,25 @@
 """Start and stop TTS backend servers from the TUI hub.
 
 Each backend's ``detect()`` returns a list of ``ServerSpec`` — the exact argv
-(absolute binaries in the managed venv, no shell activation needed) and the
-URL to probe for readiness. This module turns those specs into running
-processes: ``start`` spawns the server, streams its output to
-``app/logs/<name>-server.log``, records its pid, and polls the URL until it
-accepts connections (model loads are slow, so the timeout is generous);
-``stop`` terminates the process group the hub started.
-
-Everything here runs in the plain console tail after the curses TUI returns
-(matching the wizards' build/pip streaming), so progress and log tails appear
-normally. Pid/log files live under ``app/logs/`` which is already gitignored.
+(absolute binaries in the managed venv, no shell activation needed), the
+working directory to spawn it in, and the URL to probe for readiness. This
+module turns those specs into running processes: ``start`` spawns the server
+(streaming its output to ``app/logs/<name>-server.log``, in the spec's cwd
+when it has one — audio.cpp resolves model_specs/ relative to its process
+working directory), records its pid, and polls the URL until it accepts
+connections (model loads are slow, so the timeout is generous). With a spec
+``identity`` the poll also verifies the server answers HTTP as that backend,
+so readiness means "serving", not just "listening".
+
+Progress reporting goes through an optional ``progress`` callback (see
+``_console_progress`` for the event shapes); the default callback prints the
+same lines as before, so the plain-console flow is unchanged. ``stop``
+terminates the process group the hub started.
+
+Everything here can run in the plain console tail after the curses TUI
+returns (matching the wizards' build/pip streaming) or behind the run view's
+boot screen, which renders the same events. Pid/log files live under
+``app/logs/`` which is already gitignored.
 """
 
 import os
@@ -19,9 +28,9 @@ import subprocess
 import sys
 import time
 from pathlib import Path
-from typing import List
+from typing import Callable, List, Optional
 
-from backends import common
+from backends import common, probe
 from backends.common import APP_DIR
 
 LOG_DIR = APP_DIR / "logs"
@@ -34,6 +43,48 @@ SERVER_START_TIMEOUT = 600
 # Grace period after SIGTERM before escalating to SIGKILL (POSIX).
 STOP_GRACE_SECONDS = 10
 
+# How often the start poll re-checks readiness (seconds).
+POLL_INTERVAL = 1
+
+# Progress callback: called with an event dict. KIND is one of:
+#   "starting"  {name, argv, cwd, log_path, pid}  spawned, waiting for boot
+#   "elapsed"   {name, seconds}                   heartbeat while waiting
+#   "ready"     {name, url}                       server is up and answering
+#   "exited"    {name, returncode, log_tail}      process exited while booting
+#   "timeout"   {name, seconds, log_tail}         readiness deadline elapsed
+#   "running"   {name, url}                       already up (no spawn)
+#   "cancelled" {name}                            boot aborted via cancel
+#   "error"     {message}                         could not spawn the executable
+ProgressCallback = Optional[Callable[[dict], None]]
+
+
+def _console_progress(event: dict) -> None:
+    """Print PROGRESS events as the plain-console output (the old behavior)."""
+    kind = event.get("kind")
+    if kind == "starting":
+        print(f"[INFO] starting {event['name']} server: {event['argv']}")
+        if event.get("cwd"):
+            print(f"[INFO] working directory: {event['cwd']}")
+        print(f"[INFO] pid {event['pid']}; logs: {event['log_path']}")
+    elif kind == "elapsed":
+        print(f"[INFO] still waiting for the server ({int(event['seconds'])}s)...")
+    elif kind == "ready":
+        print(f"[OK] {event['name']} server is up on {event['url']}")
+    elif kind == "running":
+        print(f"[INFO] {event['name']} server already running on {event['url']}")
+    elif kind == "cancelled":
+        print(f"[INFO] {event['name']} server start cancelled")
+    elif kind == "exited":
+        print(f"[ERROR] {event['name']} server exited with code "
+              f"{event['returncode']}")
+        _print_tail(event.get("log_tail"))
+    elif kind == "timeout":
+        print(f"[ERROR] {event['name']} server did not start within "
+              f"{int(event['seconds'])}s")
+        _print_tail(event.get("log_tail"))
+    elif kind == "error":
+        print(f"[ERROR] {event['message']}")
+
 
 def _log_path(name: str) -> Path:
     return LOG_DIR / f"{name}-server.log"
@@ -43,17 +94,20 @@ def _pid_path(name: str) -> Path:
     return LOG_DIR / f"{name}-server.pid"
 
 
-def _tail_log(name: str, lines: int = 20) -> None:
-    """Print the last LINES of the server's log (best-effort)."""
-    path = _log_path(name)
+def _read_log_tail(name: str, lines: int = 20) -> List[str]:
+    """Return the last LINES of the server's log (best-effort)."""
     try:
-        text = path.read_text(encoding="utf-8", errors="replace")
+        text = _log_path(name).read_text(encoding="utf-8", errors="replace")
     except OSError:
-        return
-    tail = "\n".join(text.splitlines()[-lines:])
+        return []
+    return text.splitlines()[-lines:]
+
+
+def _print_tail(tail: List[str]) -> None:
+    """Print a log-tail event payload (used by the console callback)."""
     if tail:
-        print(f"--- last {lines} lines of {path} ---")
-        print(tail)
+        print(f"--- last {len(tail)} lines of the server log ---")
+        print("\n".join(tail))
         print("---")
 
 
@@ -123,24 +177,53 @@ def _kill_pid(pid: int) -> bool:
     return True
 
 
-def start(spec) -> bool:
+def _server_ready(spec) -> bool:
+    """True when the server described by SPEC is usable, not just listening.
+
+    Without an IDENTITY this is the plain TCP-connect check. With one, the
+    server must also answer HTTP as that backend (``probe.identify_server``);
+    for the faster backend (whose model loads after the port opens) the
+    ``/health`` model_loaded flag must additionally be true.
+    """
+    if not common.server_running(spec.url):
+        return False
+    identity = getattr(spec, "identity", None)
+    if identity is None:
+        return True
+    if probe.identify_server(spec.url) != identity:
+        return False
+    if identity == probe.IDENTITY_FASTER:
+        return probe.faster_model_loaded(spec.url)
+    return True
+
+
+def start(spec, progress: ProgressCallback = None,
+          cancel=None) -> bool:
     """Start the server described by SPEC (a ``backends.ServerSpec``).
 
-    Spawns its argv with stdout/stderr to ``logs/<name>-server.log``, records
-    the pid, and polls ``common.server_running(spec.url)`` until it accepts
-    connections or ``SERVER_START_TIMEOUT`` elapses. Returns True when the
-    server is up; on timeout or early exit, prints the log tail and returns
+    Spawns its argv with stdout/stderr to ``logs/<name>-server.log``, in the
+    spec's CWD when it has one (audio.cpp discovers model_specs/ from its
+    process working directory), records the pid, and polls readiness —
+    ``_server_ready``, so an IDENTITY spec must actually answer HTTP — until
+    it is up or ``SERVER_START_TIMEOUT`` elapses. Returns True when the
+    server is up; on timeout or early exit reports the log tail and returns
     False. A no-op (True) when the server is already running.
+
+    PROGRESS, when given, receives each boot event (see ProgressCallback);
+    the default ``_console_progress`` prints them, preserving the old
+    console output. CANCEL (a threading.Event) aborts the boot: the spawned
+    process is terminated and False is reported (event kind "cancelled").
     """
+    report = progress if progress is not None else _console_progress
     argv: List[str] = list(spec.argv)
     exe = Path(argv[0])
     if not exe.exists():
-        print(f"[ERROR] server executable not found: {exe}")
-        print("        run 'Set up a backend' for "
-              f"{spec.name!r} first.")
+        report({"kind": "error",
+                "message": f"server executable not found: {exe}. Run 'Set "
+                           "up a backend' first."})
         return False
-    if common.server_running(spec.url):
-        print(f"[INFO] {spec.name} server already running on {spec.url}")
+    if _server_ready(spec):
+        report({"kind": "running", "name": spec.name, "url": spec.url})
         return True
 
     LOG_DIR.mkdir(parents=True, exist_ok=True)
@@ -151,10 +234,11 @@ def start(spec) -> bool:
         except OSError:
             pass
 
-    print(f"[INFO] starting {spec.name} server: "
-          + " ".join(str(a) for a in argv))
+    cwd = getattr(spec, "cwd", None)
     log_handle = _log_path(spec.name).open("w", encoding="utf-8")
     popen_kwargs = {"stdout": log_handle, "stderr": subprocess.STDOUT}
+    if cwd is not None:
+        popen_kwargs["cwd"] = str(cwd)
     if sys.platform == "win32":
         popen_kwargs["creationflags"] = \
             subprocess.CREATE_NEW_PROCESS_GROUP  # type: ignore[attr-defined]
@@ -163,31 +247,51 @@ def start(spec) -> bool:
     try:
         proc = subprocess.Popen(argv, **popen_kwargs)
     except OSError as exc:
-        print(f"[ERROR] could not start server: {exc}")
+        report({"kind": "error",
+                "message": f"could not start server: {exc}"})
         log_handle.close()
         return False
 
     pid_file.write_text(str(proc.pid), encoding="utf-8")
-    print(f"[INFO] pid {proc.pid}; logs: {_log_path(spec.name)}")
-
-    deadline = time.time() + SERVER_START_TIMEOUT
+    report({"kind": "starting", "name": spec.name,
+            "argv": " ".join(str(a) for a in argv),
+            "cwd": str(cwd) if cwd is not None else None,
+            "log_path": str(_log_path(spec.name)), "pid": proc.pid})
+
+    started = time.time()
+    next_heartbeat = started + 15
+    deadline = started + SERVER_START_TIMEOUT
     while time.time() < deadline:
+        if cancel is not None and cancel.is_set():
+            # User cancelled while booting: kill what we spawned (the
+            # server we started is not left loading in the background).
+            _kill_pid(proc.pid)
+            try:
+                pid_file.unlink()
+            except OSError:
+                pass
+            report({"kind": "cancelled", "name": spec.name})
+            return False
         if proc.poll() is not None:
-            print(f"[ERROR] {spec.name} server exited with code "
-                  f"{proc.returncode}")
-            _tail_log(spec.name)
+            report({"kind": "exited", "name": spec.name,
+                    "returncode": proc.returncode,
+                    "log_tail": _read_log_tail(spec.name)})
             try:
                 pid_file.unlink()
             except OSError:
                 pass
             return False
-        if common.server_running(spec.url):
-            print(f"[OK] {spec.name} server is up on {spec.url}")
+        if _server_ready(spec):
+            report({"kind": "ready", "name": spec.name, "url": spec.url})
             return True
-        time.sleep(1)
-    print(f"[ERROR] {spec.name} server did not start within "
-          f"{SERVER_START_TIMEOUT}s")
-    _tail_log(spec.name)
+        if time.time() >= next_heartbeat:
+            report({"kind": "elapsed",
+                    "seconds": time.time() - started})
+            next_heartbeat += 15
+        time.sleep(POLL_INTERVAL)
+    report({"kind": "timeout", "name": spec.name,
+            "seconds": SERVER_START_TIMEOUT,
+            "log_tail": _read_log_tail(spec.name)})
     # Leave the pid file in place so stop() can kill it (it may still load).
     return False