diff options
Diffstat (limited to 'app/backends')
| -rw-r--r-- | app/backends/__init__.py | 36 | ||||
| -rwxr-xr-x | app/backends/audiocpp.py | 101 | ||||
| -rwxr-xr-x | app/backends/faster.py | 5 | ||||
| -rw-r--r-- | app/backends/probe.py | 23 | ||||
| -rw-r--r-- | app/backends/qwen.py | 6 | ||||
| -rw-r--r-- | app/backends/servers.py | 194 |
6 files changed, 307 insertions, 58 deletions
diff --git a/app/backends/__init__.py b/app/backends/__init__.py index 9e8bf31..488c36e 100644 --- a/app/backends/__init__.py +++ b/app/backends/__init__.py @@ -24,6 +24,7 @@ automatically. import shlex from dataclasses import dataclass, field +from pathlib import Path from typing import Callable, Dict, List, Optional @@ -35,10 +36,24 @@ class ServerSpec: on separate ports). ARGV is the exact command line the hub spawns (using the managed venv's absolute binaries, so no shell activation is needed); URL is the endpoint ``common.server_running`` probes to decide readiness. + + CWD is the working directory the server is spawned in. It matters for + servers that discover resources relative to their process working + directory (audio.cpp resolves ``model_specs/<family>.json`` by walking up + from its cwd), so the hub starts them from their checkout root. None + inherits the hub's cwd (today's behavior). + + IDENTITY is the ``backends.probe.IDENTITY_*`` constant the server is + expected to answer as once it is truly ready. When set, ``servers.start`` + waits for the server to answer HTTP with that identity — not merely to + accept TCP connections — so "listening but still starting" servers are + caught. None keeps the plain TCP-connect readiness check. """ name: str url: str argv: List[str] + cwd: Optional[Path] = None + identity: Optional[str] = None @dataclass @@ -72,6 +87,12 @@ class BackendStatus: RUNNING_MODELS names which of a multi-server backend's models answered (qwen: "Base" and/or "CustomVoice", local and remote combined), shown in parentheses in the hub's status table. + + MODELS_MISSING says the server config references model files that are not + on disk (e.g. an audio.cpp server.json entry whose ``path`` was never + downloaded); DETAILS then names them. The backend still counts as ready + (the hub surfaces the warning), but a conversion would fail until the + models are installed. """ key: str label: str @@ -86,6 +107,7 @@ class BackendStatus: remote: bool = False remote_urls: Dict[str, str] = field(default_factory=dict) remote_models: List[str] = field(default_factory=list) + models_missing: bool = False @property def ready(self) -> bool: @@ -94,8 +116,18 @@ class BackendStatus: def format_launch_hint(servers: List[ServerSpec]) -> str: - """Join a backend's server argvs into a copy-pasteable launch hint.""" - return " ; ".join(shlex.join(s.argv) for s in servers) + """Join a backend's server argvs into a copy-pasteable launch hint. + + A server with a CWD is prefixed with ``cd <cwd> &&`` so the hint works + pasted into a shell (the server relies on that working directory). + """ + parts = [] + for server in servers: + command = shlex.join(server.argv) + if server.cwd is not None: + command = f"cd {shlex.quote(str(server.cwd))} && {command}" + parts.append(command) + return " ; ".join(parts) @dataclass diff --git a/app/backends/audiocpp.py b/app/backends/audiocpp.py index ee0ee21..a502c8f 100755 --- a/app/backends/audiocpp.py +++ b/app/backends/audiocpp.py @@ -1276,6 +1276,78 @@ def _wizard(stdscr, args: argparse.Namespace, parser: argparse.ArgumentParser } +def _model_path_present(path: Path) -> bool: + """True when a server.json model path holds actual model files. + + A present path is either a file (a single-model package) or a non-empty + directory (the usual GGUF package target directory; an empty one means a + download that never ran or was cleaned up halfway). + """ + try: + if path.is_file(): + return True + if path.is_dir(): + return any(path.iterdir()) + except OSError: + return False + return False + + +def missing_model_entries(server_json: Path) -> List[dict]: + """Return the server.json model entries whose files are not on disk. + + Paths resolve exactly like audiocpp_server resolves them (relative paths + against the server.json's directory). Each returned entry carries the + entry ``id`` and ``rel`` (the configured path string); used by ``detect`` + to warn that a conversion would fail until the models are installed. + """ + try: + data = json.loads(server_json.read_text(encoding="utf-8")) + except (OSError, ValueError): + return [] + if not isinstance(data, dict): + return [] + base = server_json.parent + missing: List[dict] = [] + for entry in data.get("models") or []: + if not isinstance(entry, dict): + continue + rel = entry.get("path") + if not isinstance(rel, str) or not rel: + continue + path = Path(rel) if Path(rel).is_absolute() else base / rel + if _model_path_present(path): + continue + missing.append({"id": str(entry.get("id") or rel), "rel": rel}) + return missing + + +def model_install_hints(audiocpp_dir: Path, + missing: List[dict]) -> List[str]: + """Remediation lines for MISSING model entries (see missing_model_entries). + + Maps each entry's configured path back to the catalog package that + installs it (``models/<target_directory>`` -> install id) so the line + carries the exact ``model_manager_v2.py install`` command; entries whose + directory matches no catalog package just name the path. + """ + by_path: Dict[str, str] = {} + try: + for entry in load_model_catalog(audiocpp_dir): + by_path[entry["default_path"]] = entry["install_id"] + except (NotADirectoryError, OSError): + pass + hints: List[str] = [] + for item in missing: + install_id = by_path.get(item["rel"]) + hint = f"model not downloaded: {item['id']} ({item['rel']})" + if install_id: + hint += (f" — install with: python tools/model_manager_v2.py " + f"install {install_id}") + hints.append(hint) + return hints + + def find_local_checkout() -> Optional[Path]: """Best-effort location of an audio.cpp checkout with model_specs. @@ -1421,20 +1493,24 @@ def build_audiocpp(audiocpp_dir: Path, backend: str) -> int: def _print_launch_hint(audiocpp_dir: Path, output_path: Path) -> None: - """Print the exact command to start the server (or build guidance).""" + """Print the exact command to start the server (or build guidance). + + The command is prefixed with ``cd <checkout> &&`` because the server + discovers model_specs/<family>.json relative to its working directory. + """ binary = find_audiocpp_server_bin(audiocpp_dir) print() if binary is not None: print("Start the server with:") - print(f" {binary} --config {output_path}") + print(f" cd {audiocpp_dir} && {binary} --config {output_path}") else: print("[INFO] audiocpp_server binary not found. Build it first, e.g.:") script = find_build_script(audiocpp_dir) if script is not None: print(f" sh {script} --backend <cuda|vulkan|hip|cpu> " "--target audiocpp_server") - print(f" then run: ./build/<platform>-<backend>-release/bin/" - f"audiocpp_server --config {output_path}") + print(f" then run: cd {audiocpp_dir} && ./build/<platform>-<backend>" + f"-release/bin/audiocpp_server --config {output_path}") def _execute(settings: dict, args: argparse.Namespace) -> int: @@ -1767,15 +1843,23 @@ def detect() -> BackendStatus: server_json = checkout / "server.json" configured = server_json.exists() specs: List[ServerSpec] = [] + missing = missing_model_entries(server_json) if configured else [] if configured: details.append(f"config: {server_json}") + if missing: + # The config references model files that are not on disk; a + # conversion would fail at model-load time, so say so now. + details.extend(model_install_hints(checkout, missing)) if built: + # Spawned from the checkout: audiocpp_server discovers + # model_specs/<family>.json relative to its working directory. specs = [ServerSpec( "audiocpp", config.AUDIOCPP_API_URL, - [str(binary), "--config", str(server_json)])] + [str(binary), "--config", str(server_json)], + cwd=checkout, identity=probe.IDENTITY_AUDIOCPP)] else: - launch = (f"./build/<platform>-<backend>-release/bin/" - f"audiocpp_server --config {server_json}") + launch = (f"cd {checkout} && ./build/<platform>-<backend>-release" + f"/bin/audiocpp_server --config {server_json}") else: details.append("no server.json — run setup to configure models") if specs: @@ -1787,7 +1871,8 @@ def detect() -> BackendStatus: running=managed or remote_running, details=details, launch_hint=launch, servers=specs, managed=managed, - remote=remote_running, remote_urls=remote_urls) + remote=remote_running, remote_urls=remote_urls, + models_missing=bool(missing)) def _detect_remote(managed: bool = False) -> Tuple[bool, dict]: diff --git a/app/backends/faster.py b/app/backends/faster.py index 3dc30dd..e1249ca 100755 --- a/app/backends/faster.py +++ b/app/backends/faster.py @@ -362,7 +362,10 @@ def detect() -> BackendStatus: argv = [str(envs.env_python()), str(_checkout() / "examples" / "openai_server.py"), "--voices", str(voices_json), "--port", str(_config_port())] - specs = [ServerSpec("faster", config.FASTER_API_URL, argv)] + # identity: /health must report model_loaded before the server is + # really usable (the model loads after the port opens). + specs = [ServerSpec("faster", config.FASTER_API_URL, argv, + identity=probe.IDENTITY_FASTER)] launch = format_launch_hint(specs) managed = servers.manages(specs) remote_running, remote_urls = _detect_remote(managed) diff --git a/app/backends/probe.py b/app/backends/probe.py index ebef86e..efa2963 100644 --- a/app/backends/probe.py +++ b/app/backends/probe.py @@ -126,3 +126,26 @@ def same_endpoint(url_a: str, url_b: str) -> bool: port_a = a.port or (443 if (a.scheme or "http") == "https" else 80) port_b = b.port or (443 if (b.scheme or "http") == "https" else 80) return host_a == host_b and port_a == port_b + + +def health_payload(url: str, timeout: float = DEFAULT_TIMEOUT) -> Optional[dict]: + """Return the server's ``/health`` JSON document, or None. + + A cheaper, raw check than ``identify_server``: used by the run view's + background poll to tell "process alive" from "server answering" without + probing every identity endpoint. + """ + if not url: + return None + return _get_json(f"{url.rstrip('/')}/health", timeout) + + +def faster_model_loaded(url: str, timeout: float = DEFAULT_TIMEOUT) -> bool: + """True when a faster server at URL reports its model loaded. + + faster's ``/health`` answers with a ``model_loaded`` flag only after the + weights are resident, so this is the "truly ready" signal used while + waiting for a started server to become usable. + """ + payload = health_payload(url, timeout) + return bool(payload and payload.get("model_loaded")) diff --git a/app/backends/qwen.py b/app/backends/qwen.py index 160c0f1..7f821fa 100644 --- a/app/backends/qwen.py +++ b/app/backends/qwen.py @@ -227,10 +227,12 @@ def detect() -> BackendStatus: specs = [ ServerSpec("qwen-custom", config.QWEN_API_URL, [demo, QWEN_CUSTOMVOICE_MODEL, "--ip", "127.0.0.1", - "--port", str(custom_port)]), + "--port", str(custom_port)], + identity=probe.IDENTITY_QWEN_CUSTOM), ServerSpec("qwen-clone", config.CLONE_API_URL, [demo, QWEN_BASE_MODEL, "--ip", "127.0.0.1", - "--port", str(clone_port)]), + "--port", str(clone_port)], + identity=probe.IDENTITY_QWEN_CLONE), ] managed = servers.manages(specs) # Which local servers this tool started (pid alive) name the running diff --git a/app/backends/servers.py b/app/backends/servers.py index 912c09d..62eadb5 100644 --- a/app/backends/servers.py +++ b/app/backends/servers.py @@ -1,16 +1,25 @@ """Start and stop TTS backend servers from the TUI hub. Each backend's ``detect()`` returns a list of ``ServerSpec`` — the exact argv -(absolute binaries in the managed venv, no shell activation needed) and the -URL to probe for readiness. This module turns those specs into running -processes: ``start`` spawns the server, streams its output to -``app/logs/<name>-server.log``, records its pid, and polls the URL until it -accepts connections (model loads are slow, so the timeout is generous); -``stop`` terminates the process group the hub started. - -Everything here runs in the plain console tail after the curses TUI returns -(matching the wizards' build/pip streaming), so progress and log tails appear -normally. Pid/log files live under ``app/logs/`` which is already gitignored. +(absolute binaries in the managed venv, no shell activation needed), the +working directory to spawn it in, and the URL to probe for readiness. This +module turns those specs into running processes: ``start`` spawns the server +(streaming its output to ``app/logs/<name>-server.log``, in the spec's cwd +when it has one — audio.cpp resolves model_specs/ relative to its process +working directory), records its pid, and polls the URL until it accepts +connections (model loads are slow, so the timeout is generous). With a spec +``identity`` the poll also verifies the server answers HTTP as that backend, +so readiness means "serving", not just "listening". + +Progress reporting goes through an optional ``progress`` callback (see +``_console_progress`` for the event shapes); the default callback prints the +same lines as before, so the plain-console flow is unchanged. ``stop`` +terminates the process group the hub started. + +Everything here can run in the plain console tail after the curses TUI +returns (matching the wizards' build/pip streaming) or behind the run view's +boot screen, which renders the same events. Pid/log files live under +``app/logs/`` which is already gitignored. """ import os @@ -19,9 +28,9 @@ import subprocess import sys import time from pathlib import Path -from typing import List +from typing import Callable, List, Optional -from backends import common +from backends import common, probe from backends.common import APP_DIR LOG_DIR = APP_DIR / "logs" @@ -34,6 +43,48 @@ SERVER_START_TIMEOUT = 600 # Grace period after SIGTERM before escalating to SIGKILL (POSIX). STOP_GRACE_SECONDS = 10 +# How often the start poll re-checks readiness (seconds). +POLL_INTERVAL = 1 + +# Progress callback: called with an event dict. KIND is one of: +# "starting" {name, argv, cwd, log_path, pid} spawned, waiting for boot +# "elapsed" {name, seconds} heartbeat while waiting +# "ready" {name, url} server is up and answering +# "exited" {name, returncode, log_tail} process exited while booting +# "timeout" {name, seconds, log_tail} readiness deadline elapsed +# "running" {name, url} already up (no spawn) +# "cancelled" {name} boot aborted via cancel +# "error" {message} could not spawn the executable +ProgressCallback = Optional[Callable[[dict], None]] + + +def _console_progress(event: dict) -> None: + """Print PROGRESS events as the plain-console output (the old behavior).""" + kind = event.get("kind") + if kind == "starting": + print(f"[INFO] starting {event['name']} server: {event['argv']}") + if event.get("cwd"): + print(f"[INFO] working directory: {event['cwd']}") + print(f"[INFO] pid {event['pid']}; logs: {event['log_path']}") + elif kind == "elapsed": + print(f"[INFO] still waiting for the server ({int(event['seconds'])}s)...") + elif kind == "ready": + print(f"[OK] {event['name']} server is up on {event['url']}") + elif kind == "running": + print(f"[INFO] {event['name']} server already running on {event['url']}") + elif kind == "cancelled": + print(f"[INFO] {event['name']} server start cancelled") + elif kind == "exited": + print(f"[ERROR] {event['name']} server exited with code " + f"{event['returncode']}") + _print_tail(event.get("log_tail")) + elif kind == "timeout": + print(f"[ERROR] {event['name']} server did not start within " + f"{int(event['seconds'])}s") + _print_tail(event.get("log_tail")) + elif kind == "error": + print(f"[ERROR] {event['message']}") + def _log_path(name: str) -> Path: return LOG_DIR / f"{name}-server.log" @@ -43,17 +94,20 @@ def _pid_path(name: str) -> Path: return LOG_DIR / f"{name}-server.pid" -def _tail_log(name: str, lines: int = 20) -> None: - """Print the last LINES of the server's log (best-effort).""" - path = _log_path(name) +def _read_log_tail(name: str, lines: int = 20) -> List[str]: + """Return the last LINES of the server's log (best-effort).""" try: - text = path.read_text(encoding="utf-8", errors="replace") + text = _log_path(name).read_text(encoding="utf-8", errors="replace") except OSError: - return - tail = "\n".join(text.splitlines()[-lines:]) + return [] + return text.splitlines()[-lines:] + + +def _print_tail(tail: List[str]) -> None: + """Print a log-tail event payload (used by the console callback).""" if tail: - print(f"--- last {lines} lines of {path} ---") - print(tail) + print(f"--- last {len(tail)} lines of the server log ---") + print("\n".join(tail)) print("---") @@ -123,24 +177,53 @@ def _kill_pid(pid: int) -> bool: return True -def start(spec) -> bool: +def _server_ready(spec) -> bool: + """True when the server described by SPEC is usable, not just listening. + + Without an IDENTITY this is the plain TCP-connect check. With one, the + server must also answer HTTP as that backend (``probe.identify_server``); + for the faster backend (whose model loads after the port opens) the + ``/health`` model_loaded flag must additionally be true. + """ + if not common.server_running(spec.url): + return False + identity = getattr(spec, "identity", None) + if identity is None: + return True + if probe.identify_server(spec.url) != identity: + return False + if identity == probe.IDENTITY_FASTER: + return probe.faster_model_loaded(spec.url) + return True + + +def start(spec, progress: ProgressCallback = None, + cancel=None) -> bool: """Start the server described by SPEC (a ``backends.ServerSpec``). - Spawns its argv with stdout/stderr to ``logs/<name>-server.log``, records - the pid, and polls ``common.server_running(spec.url)`` until it accepts - connections or ``SERVER_START_TIMEOUT`` elapses. Returns True when the - server is up; on timeout or early exit, prints the log tail and returns + Spawns its argv with stdout/stderr to ``logs/<name>-server.log``, in the + spec's CWD when it has one (audio.cpp discovers model_specs/ from its + process working directory), records the pid, and polls readiness — + ``_server_ready``, so an IDENTITY spec must actually answer HTTP — until + it is up or ``SERVER_START_TIMEOUT`` elapses. Returns True when the + server is up; on timeout or early exit reports the log tail and returns False. A no-op (True) when the server is already running. + + PROGRESS, when given, receives each boot event (see ProgressCallback); + the default ``_console_progress`` prints them, preserving the old + console output. CANCEL (a threading.Event) aborts the boot: the spawned + process is terminated and False is reported (event kind "cancelled"). """ + report = progress if progress is not None else _console_progress argv: List[str] = list(spec.argv) exe = Path(argv[0]) if not exe.exists(): - print(f"[ERROR] server executable not found: {exe}") - print(" run 'Set up a backend' for " - f"{spec.name!r} first.") + report({"kind": "error", + "message": f"server executable not found: {exe}. Run 'Set " + "up a backend' first."}) return False - if common.server_running(spec.url): - print(f"[INFO] {spec.name} server already running on {spec.url}") + if _server_ready(spec): + report({"kind": "running", "name": spec.name, "url": spec.url}) return True LOG_DIR.mkdir(parents=True, exist_ok=True) @@ -151,10 +234,11 @@ def start(spec) -> bool: except OSError: pass - print(f"[INFO] starting {spec.name} server: " - + " ".join(str(a) for a in argv)) + cwd = getattr(spec, "cwd", None) log_handle = _log_path(spec.name).open("w", encoding="utf-8") popen_kwargs = {"stdout": log_handle, "stderr": subprocess.STDOUT} + if cwd is not None: + popen_kwargs["cwd"] = str(cwd) if sys.platform == "win32": popen_kwargs["creationflags"] = \ subprocess.CREATE_NEW_PROCESS_GROUP # type: ignore[attr-defined] @@ -163,31 +247,51 @@ def start(spec) -> bool: try: proc = subprocess.Popen(argv, **popen_kwargs) except OSError as exc: - print(f"[ERROR] could not start server: {exc}") + report({"kind": "error", + "message": f"could not start server: {exc}"}) log_handle.close() return False pid_file.write_text(str(proc.pid), encoding="utf-8") - print(f"[INFO] pid {proc.pid}; logs: {_log_path(spec.name)}") - - deadline = time.time() + SERVER_START_TIMEOUT + report({"kind": "starting", "name": spec.name, + "argv": " ".join(str(a) for a in argv), + "cwd": str(cwd) if cwd is not None else None, + "log_path": str(_log_path(spec.name)), "pid": proc.pid}) + + started = time.time() + next_heartbeat = started + 15 + deadline = started + SERVER_START_TIMEOUT while time.time() < deadline: + if cancel is not None and cancel.is_set(): + # User cancelled while booting: kill what we spawned (the + # server we started is not left loading in the background). + _kill_pid(proc.pid) + try: + pid_file.unlink() + except OSError: + pass + report({"kind": "cancelled", "name": spec.name}) + return False if proc.poll() is not None: - print(f"[ERROR] {spec.name} server exited with code " - f"{proc.returncode}") - _tail_log(spec.name) + report({"kind": "exited", "name": spec.name, + "returncode": proc.returncode, + "log_tail": _read_log_tail(spec.name)}) try: pid_file.unlink() except OSError: pass return False - if common.server_running(spec.url): - print(f"[OK] {spec.name} server is up on {spec.url}") + if _server_ready(spec): + report({"kind": "ready", "name": spec.name, "url": spec.url}) return True - time.sleep(1) - print(f"[ERROR] {spec.name} server did not start within " - f"{SERVER_START_TIMEOUT}s") - _tail_log(spec.name) + if time.time() >= next_heartbeat: + report({"kind": "elapsed", + "seconds": time.time() - started}) + next_heartbeat += 15 + time.sleep(POLL_INTERVAL) + report({"kind": "timeout", "name": spec.name, + "seconds": SERVER_START_TIMEOUT, + "log_tail": _read_log_tail(spec.name)}) # Leave the pid file in place so stop() can kill it (it may still load). return False |