feat: reserve 20% vram when running higgs with sglang-omni

1 files changed, 24 insertions, 11 deletions

diff --git a/app/backends/sglomni/catalog.py b/app/backends/sglomni/catalog.py
index 61cb11c..1e4ca2d 100644
--- a/app/backends/sglomni/catalog.py
+++ b/app/backends/sglomni/catalog.py
@@ -11,7 +11,9 @@ The catalog is static knowledge about sglang-omni's supported TTS models
 (v0.1.4-era), not a live query: the server's /v1/models answers only which
 model is currently hosted. Entries carry the HuggingFace repo id verbatim —
 it is both the `--model-path` value and the model name /v1/models reports,
-so a running server is matched back to its entry by that id.
+so a running server is matched back to its entry by that id. ZONOS2 is the
+one entry without a config file (its GPU-conditional bf16 fallback lives in
+`bf16_config`); every other entry launches with a vendored config.
 
 Voice capabilities (mirroring the audio.cpp client's vocabulary):
   speaker  the voice is a named preset shipped with the model (Qwen3-TTS
@@ -61,14 +63,16 @@ class ModelEntry:
     fp8_min_compute_capability: Optional[Tuple[int, int]] = None
     bf16_config: Optional[str] = None
     # max_new_tokens sent with every /v1/audio/speech request (None = send
-    # nothing and leave the server default in charge). Zonos2's AR engine
-    # defaults to 1024 audio frames (44100 Hz / 512 hop = 86.13 fps), which
-    # caps one request at ~12 s of speech and silently truncates longer
-    # text. 12288 frames (~143 s) covers a full 250-word CHUNK_SIZE
-    # sub-chunk (~8600 frames) and stays under the KV-pool admission check
-    # on every GPU that can host the model (all are >= 24 GB with
-    # >= 28083-token pools; the scheduler rejects prompt + max_new_tokens
-    # above it, and zonos2 requests are not auto-clamped).
+    # nothing and leave the server default in charge). Some engines cap one
+    # request below what a full CHUNK_SIZE sub-chunk can narrate — ZONOS2's
+    # AR engine defaults to 1024 audio frames (44100 Hz / 512 hop =
+    # 86.13 fps, ~12 s) and Higgs's to 2048 frames (75 fps, ~27 s, and
+    # per-request values are clamped to the engine cap, so its vendored
+    # config raises the cap too) — and silently truncate longer text.
+    # 12288 frames covers a full 250-word CHUNK_SIZE sub-chunk on both and
+    # stays under the KV-pool admission check on every GPU that can host
+    # the model (the scheduler rejects prompt + max_new_tokens above it,
+    # and these requests are not auto-clamped).
     max_new_tokens: Optional[int] = None
 
 
@@ -164,9 +168,18 @@ ENTRIES: Tuple[ModelEntry, ...] = (
         key="higgs_audio_v3_tts",
         label="Higgs Audio v3 TTS",
         repo="bosonai/higgs-audio-v3-tts-4b",
-        config=None,
+        # The vendored config trims the engine's VRAM budget (the upstream
+        # pipeline budgets 0.98 of the card across its colocated stages,
+        # which OOMs on 24 GB cards) and raises the engine's 2048-frame
+        # generation cap (see the yaml header).
+        config="higgs_audio_v3_tts.yaml",
         capability=CAPABILITY_CLONE,
         requires_reference=False,
+        # The engine's 2048-frame default is ~27 s of speech at the codec's
+        # 75 fps; requests are clamped to the engine cap server-side, so the
+        # yaml raises the cap and every request carries 12288 frames
+        # (~164 s) — enough for a full CHUNK_SIZE sub-chunk.
+        max_new_tokens=12288,
         notes="zero-shot narration, cloning from a reference clip",
     ),
     ModelEntry(
@@ -253,7 +266,7 @@ def entries_by_keys(keys) -> List[ModelEntry]:
 
 def config_path(entry: ModelEntry):
     """The vendored config yaml path for ENTRY, or None when it runs on
-    --model-path alone (Higgs, ZONOS2). A declared-but-missing file means
+    --model-path alone (ZONOS2). A declared-but-missing file means
     a broken install — callers treat that like a missing entry."""
     return _config_file(entry.config)