diff options
Diffstat (limited to 'app/backends/managed.py')
| -rw-r--r-- | app/backends/managed.py | 76 |
1 files changed, 59 insertions, 17 deletions
diff --git a/app/backends/managed.py b/app/backends/managed.py index 3b98aea..d7c8164 100644 --- a/app/backends/managed.py +++ b/app/backends/managed.py @@ -29,7 +29,7 @@ from dataclasses import dataclass from typing import Optional from backends import ServerSpec, common, probe, servers -from converter.clients import BACKEND_QWEN +from converter.clients import BACKEND_QWEN, BACKEND_SGLOMNI @dataclass @@ -56,20 +56,22 @@ class ManagedServer: servers.stop(self.spec.name) -def ensure_running(backend: str, voice_mode: str) -> Optional[ManagedServer]: +def ensure_running(backend: str, voice_mode: str, + model: Optional[str] = None) -> Optional[ManagedServer]: """Make the backend's managed server ready for a conversion run. Resolves the server spec for BACKEND (qwen: the demo hosting the - model VOICE_MODE needs; the others: their single configured spec), - then starts it when its port is free — waiting out the boot and - streaming ``servers``' console progress — or reuses the server - already answering there (qwen: restarting a managed server that hosts - another model, refusing a foreign one). Returns the run's - ``ManagedServer`` (call ``shutdown`` when the conversion is over), or - None when the backend is not installed here and nothing can be - started: the caller proceeds unmanaged, since a foreign server at the - configured endpoint may still answer and otherwise the conversion - fails with the converter's own unreachable-server message. + model VOICE_MODE needs; sglomni: the server hosting MODEL — one + model per process; the others: their single configured spec), then + starts it when its port is free — waiting out the boot and streaming + ``servers``' console progress — or reuses the server already answering + there (qwen/sglomni: restarting a managed server that hosts another + model, refusing a foreign one). Returns the run's ``ManagedServer`` + (call ``shutdown`` when the conversion is over), or None when the + backend is not installed here and nothing can be started: the caller + proceeds unmanaged, since a foreign server at the configured endpoint + may still answer and otherwise the conversion fails with the + converter's own unreachable-server message. Raises KeyboardInterrupt when the boot poll is interrupted (after stopping a server this call spawned, so nothing is left loading). @@ -83,12 +85,20 @@ def ensure_running(backend: str, voice_mode: str) -> Optional[ManagedServer]: "automatically; the conversion will use the configured " "endpoint (see the TUI's Configure Backends to install it).") return None - spec = _spec_for(status, backend, voice_mode) - return _boot(spec, backend, voice_mode) + spec = _spec_for(status, backend, voice_mode, model) + return _boot(spec, backend, voice_mode, model) -def _spec_for(status, backend: str, voice_mode: str) -> ServerSpec: +def _spec_for(status, backend: str, voice_mode: str, + model: Optional[str] = None) -> ServerSpec: """The server spec this run needs, from STATUS's detected servers.""" + if backend == BACKEND_SGLOMNI: + # sglomni hosts one model per process: aim the spec at the model + # this run selected (the converter resolves it again; resolving + # here too keeps the boot check and the conversion consistent). + from backends.sglomni import models as sg_models + from backends.sglomni import status as sg_status + return sg_status.build_spec(sg_models.resolve_model(model)) if backend != BACKEND_QWEN: return status.servers[0] # qwen hosts one model per process: aim the spec at the model this @@ -97,18 +107,50 @@ def _spec_for(status, backend: str, voice_mode: str) -> ServerSpec: return qwen.build_spec(qwen.model_for_voice_mode(voice_mode)) -def _boot(spec: ServerSpec, backend: str, voice_mode: str) -> ManagedServer: +def _boot(spec: ServerSpec, backend: str, voice_mode: str, + model: Optional[str] = None) -> ManagedServer: """Start or reuse the server SPEC describes, per the run's needs.""" wanted_model = None + wanted_repo = None if backend == BACKEND_QWEN: from backends import qwen wanted_model = qwen.model_for_voice_mode(voice_mode) + if backend == BACKEND_SGLOMNI: + from backends.sglomni import models as sg_models + entry = sg_models.resolve_model(model) + wanted_repo = entry.repo + # When this GPU cannot run the model's default FP8 pipeline, the + # spec launches the vendored bf16 config — say so before the boot. + from backends.sglomni import status as sg_status + note = sg_status.gpu_fallback_note(entry) + if note: + print(f"[WARNING] {note}") if common.server_running(spec.url): - if wanted_model is None: + if wanted_model is None and wanted_repo is None: print(f"[INFO] using the {spec.name} server already running " f"at {spec.url}") return ManagedServer(spec) + if wanted_repo is not None: + running_repo = probe.sglomni_served_model(spec.url) + if running_repo == wanted_repo: + print(f"[INFO] using the {spec.name} server already " + f"running at {spec.url} (hosting {wanted_repo})") + return ManagedServer(spec) + hosted = running_repo or "an unknown model" + if not servers.alive(spec.name): + print(f"[ERROR] a server this tool did not start is " + f"running at {spec.url} hosting {hosted} — this run " + f"needs {wanted_repo}. Stop that server first, or " + "convert with it by picking that model.") + return ManagedServer(spec, ok=False) + # Ours: stop it and boot the newly-selected model on the same + # port (the TUI's Generate form restarts a managed server the + # same way when the run's model selection changes). + print(f"[INFO] restarting the {spec.name} server to host " + f"{wanted_repo}...") + servers.stop(spec.name) + return _start(spec) running_model = qwen.model_for_identity( probe.identify_server(spec.url)) if running_model == wanted_model: |