feat: reserve 20% vram when running higgs with sglang-omni

4 files changed, 60 insertions, 16 deletions

diff --git a/app/backends/sglomni/catalog.py b/app/backends/sglomni/catalog.py
index 61cb11c..1e4ca2d 100644
--- a/app/backends/sglomni/catalog.py
+++ b/app/backends/sglomni/catalog.py
@@ -11,7 +11,9 @@ The catalog is static knowledge about sglang-omni's supported TTS models
 (v0.1.4-era), not a live query: the server's /v1/models answers only which
 model is currently hosted. Entries carry the HuggingFace repo id verbatim —
 it is both the `--model-path` value and the model name /v1/models reports,
-so a running server is matched back to its entry by that id.
+so a running server is matched back to its entry by that id. ZONOS2 is the
+one entry without a config file (its GPU-conditional bf16 fallback lives in
+`bf16_config`); every other entry launches with a vendored config.
 
 Voice capabilities (mirroring the audio.cpp client's vocabulary):
   speaker  the voice is a named preset shipped with the model (Qwen3-TTS
@@ -61,14 +63,16 @@ class ModelEntry:
     fp8_min_compute_capability: Optional[Tuple[int, int]] = None
     bf16_config: Optional[str] = None
     # max_new_tokens sent with every /v1/audio/speech request (None = send
-    # nothing and leave the server default in charge). Zonos2's AR engine
-    # defaults to 1024 audio frames (44100 Hz / 512 hop = 86.13 fps), which
-    # caps one request at ~12 s of speech and silently truncates longer
-    # text. 12288 frames (~143 s) covers a full 250-word CHUNK_SIZE
-    # sub-chunk (~8600 frames) and stays under the KV-pool admission check
-    # on every GPU that can host the model (all are >= 24 GB with
-    # >= 28083-token pools; the scheduler rejects prompt + max_new_tokens
-    # above it, and zonos2 requests are not auto-clamped).
+    # nothing and leave the server default in charge). Some engines cap one
+    # request below what a full CHUNK_SIZE sub-chunk can narrate — ZONOS2's
+    # AR engine defaults to 1024 audio frames (44100 Hz / 512 hop =
+    # 86.13 fps, ~12 s) and Higgs's to 2048 frames (75 fps, ~27 s, and
+    # per-request values are clamped to the engine cap, so its vendored
+    # config raises the cap too) — and silently truncate longer text.
+    # 12288 frames covers a full 250-word CHUNK_SIZE sub-chunk on both and
+    # stays under the KV-pool admission check on every GPU that can host
+    # the model (the scheduler rejects prompt + max_new_tokens above it,
+    # and these requests are not auto-clamped).
     max_new_tokens: Optional[int] = None
 
 
@@ -164,9 +168,18 @@ ENTRIES: Tuple[ModelEntry, ...] = (
         key="higgs_audio_v3_tts",
         label="Higgs Audio v3 TTS",
         repo="bosonai/higgs-audio-v3-tts-4b",
-        config=None,
+        # The vendored config trims the engine's VRAM budget (the upstream
+        # pipeline budgets 0.98 of the card across its colocated stages,
+        # which OOMs on 24 GB cards) and raises the engine's 2048-frame
+        # generation cap (see the yaml header).
+        config="higgs_audio_v3_tts.yaml",
         capability=CAPABILITY_CLONE,
         requires_reference=False,
+        # The engine's 2048-frame default is ~27 s of speech at the codec's
+        # 75 fps; requests are clamped to the engine cap server-side, so the
+        # yaml raises the cap and every request carries 12288 frames
+        # (~164 s) — enough for a full CHUNK_SIZE sub-chunk.
+        max_new_tokens=12288,
         notes="zero-shot narration, cloning from a reference clip",
     ),
     ModelEntry(
@@ -253,7 +266,7 @@ def entries_by_keys(keys) -> List[ModelEntry]:
 
 def config_path(entry: ModelEntry):
     """The vendored config yaml path for ENTRY, or None when it runs on
-    --model-path alone (Higgs, ZONOS2). A declared-but-missing file means
+    --model-path alone (ZONOS2). A declared-but-missing file means
     a broken install — callers treat that like a missing entry."""
     return _config_file(entry.config)
 
diff --git a/app/backends/sglomni/configs/higgs_audio_v3_tts.yaml b/app/backends/sglomni/configs/higgs_audio_v3_tts.yaml
new file mode 100644
index 0000000..74736fa
--- /dev/null
+++ b/app/backends/sglomni/configs/higgs_audio_v3_tts.yaml
@@ -0,0 +1,30 @@
+# Higgs Audio v3 TTS with VRAM headroom and a raised generation cap.
+#
+# The upstream default pipeline (HiggsTtsPipelineConfig) budgets VRAM as
+# gpu_memory_fraction 0.85 (tts_engine) + 0.10 (vocoder) + 0.03
+# (audio_encoder) = 0.98 of the card. The engine's static pool then fills
+# ~85% of a 24 GB GPU by itself (sglang mem_fraction_static = 0.85), and any
+# other VRAM consumer on the card (desktop, browsers) leaves too little room
+# for transient allocations: the first /v1/audio/speech request aborts with
+# "CUDA out of memory. Tried to allocate 14.00 MiB".
+#
+# 0.80 trims the engine's static pool by ~1.2 GB per 24 GB of VRAM while
+# leaving a KV cache pool (~10 GB on a 24 GB card) far larger than any
+# narration request needs. Cards with heavy other-GPU-process usage can go
+# lower (e.g. 0.75).
+#
+# The tts_engine factory also caps every request at max_new_tokens=2048
+# audio frames, and per-request values are clamped to that cap server-side
+# (make_higgs_scheduler_adapters) — the Higgs codec runs 75 frames per
+# second (24 kHz / 320 downsample), so the default is ~27 s of speech, which
+# silently truncates this tool's full 250-word sub-chunks (~100 s). Raising
+# the factory cap is the only way past it; the catalog also sends
+# max_new_tokens=12288 per request (the same value ZONOS2 uses) so a request
+# may use the room: 12288 frames ≈ 164 s.
+config_cls: HiggsTtsPipelineConfig
+model_path: bosonai/higgs-audio-v3-tts-4b
+stages:
+  tts_engine:
+    gpu_memory_fraction: 0.80
+    factory:
+      max_new_tokens: 12288
diff --git a/app/backends/sglomni/constants.py b/app/backends/sglomni/constants.py
index 97a4f3d..94a07bd 100644
--- a/app/backends/sglomni/constants.py
+++ b/app/backends/sglomni/constants.py
@@ -15,8 +15,9 @@ UV_PIP_PKG = "uv"
 # launching Python's, see pythonenv.prepare_env.
 
 # The vendored copies of upstream examples/configs/*.yaml (the `sgl-omni
-# serve --config` argument). Two catalog models run without one (Higgs,
-# ZONOS2) and are launched from --model-path alone.
+# serve --config` argument). ZONOS2 is the one catalog model without a
+# config (it runs from --model-path alone, with a GPU-conditional bf16
+# fallback config in its catalog entry).
 CONFIGS_DIR = Path(__file__).resolve().parent / "configs"
 
 # The managed server's port when the configured URL names none.
diff --git a/app/backends/sglomni/status.py b/app/backends/sglomni/status.py
index 099817e..2b77307 100644
--- a/app/backends/sglomni/status.py
+++ b/app/backends/sglomni/status.py
@@ -27,9 +27,9 @@ def build_spec(entry: ModelEntry) -> ServerSpec:
     Public because the run preparation (hub) and the CLI's managed-server
     bootstrap need to boot exactly the model their run selected, which can
     differ from detect()'s default. The config yaml is the vendored copy
-    from the catalog; models without one (Higgs, ZONOS2) run from
-    --model-path alone — except that a GPU the FP8 kernels cannot run
-    launches ZONOS2's vendored bf16 config instead (launch_config_path).
+    from the catalog; ZONOS2 alone runs from --model-path alone — except
+    that a GPU the FP8 kernels cannot run launches its vendored bf16 config
+    instead (launch_config_path).
     """
     url = config.SGLOMNI_API_URL
     argv: List[str] = [