feat: reserve 20% vram when running higgs with sglang-omni

7 files changed, 103 insertions, 22 deletions

diff --git a/app/backends/sglomni/catalog.py b/app/backends/sglomni/catalog.py
index 61cb11c..1e4ca2d 100644
--- a/app/backends/sglomni/catalog.py
+++ b/app/backends/sglomni/catalog.py
@@ -11,7 +11,9 @@ The catalog is static knowledge about sglang-omni's supported TTS models
 (v0.1.4-era), not a live query: the server's /v1/models answers only which
 model is currently hosted. Entries carry the HuggingFace repo id verbatim —
 it is both the `--model-path` value and the model name /v1/models reports,
-so a running server is matched back to its entry by that id.
+so a running server is matched back to its entry by that id. ZONOS2 is the
+one entry without a config file (its GPU-conditional bf16 fallback lives in
+`bf16_config`); every other entry launches with a vendored config.
 
 Voice capabilities (mirroring the audio.cpp client's vocabulary):
   speaker  the voice is a named preset shipped with the model (Qwen3-TTS
@@ -61,14 +63,16 @@ class ModelEntry:
     fp8_min_compute_capability: Optional[Tuple[int, int]] = None
     bf16_config: Optional[str] = None
     # max_new_tokens sent with every /v1/audio/speech request (None = send
-    # nothing and leave the server default in charge). Zonos2's AR engine
-    # defaults to 1024 audio frames (44100 Hz / 512 hop = 86.13 fps), which
-    # caps one request at ~12 s of speech and silently truncates longer
-    # text. 12288 frames (~143 s) covers a full 250-word CHUNK_SIZE
-    # sub-chunk (~8600 frames) and stays under the KV-pool admission check
-    # on every GPU that can host the model (all are >= 24 GB with
-    # >= 28083-token pools; the scheduler rejects prompt + max_new_tokens
-    # above it, and zonos2 requests are not auto-clamped).
+    # nothing and leave the server default in charge). Some engines cap one
+    # request below what a full CHUNK_SIZE sub-chunk can narrate — ZONOS2's
+    # AR engine defaults to 1024 audio frames (44100 Hz / 512 hop =
+    # 86.13 fps, ~12 s) and Higgs's to 2048 frames (75 fps, ~27 s, and
+    # per-request values are clamped to the engine cap, so its vendored
+    # config raises the cap too) — and silently truncate longer text.
+    # 12288 frames covers a full 250-word CHUNK_SIZE sub-chunk on both and
+    # stays under the KV-pool admission check on every GPU that can host
+    # the model (the scheduler rejects prompt + max_new_tokens above it,
+    # and these requests are not auto-clamped).
     max_new_tokens: Optional[int] = None
 
 
@@ -164,9 +168,18 @@ ENTRIES: Tuple[ModelEntry, ...] = (
         key="higgs_audio_v3_tts",
         label="Higgs Audio v3 TTS",
         repo="bosonai/higgs-audio-v3-tts-4b",
-        config=None,
+        # The vendored config trims the engine's VRAM budget (the upstream
+        # pipeline budgets 0.98 of the card across its colocated stages,
+        # which OOMs on 24 GB cards) and raises the engine's 2048-frame
+        # generation cap (see the yaml header).
+        config="higgs_audio_v3_tts.yaml",
         capability=CAPABILITY_CLONE,
         requires_reference=False,
+        # The engine's 2048-frame default is ~27 s of speech at the codec's
+        # 75 fps; requests are clamped to the engine cap server-side, so the
+        # yaml raises the cap and every request carries 12288 frames
+        # (~164 s) — enough for a full CHUNK_SIZE sub-chunk.
+        max_new_tokens=12288,
         notes="zero-shot narration, cloning from a reference clip",
     ),
     ModelEntry(
@@ -253,7 +266,7 @@ def entries_by_keys(keys) -> List[ModelEntry]:
 
 def config_path(entry: ModelEntry):
     """The vendored config yaml path for ENTRY, or None when it runs on
-    --model-path alone (Higgs, ZONOS2). A declared-but-missing file means
+    --model-path alone (ZONOS2). A declared-but-missing file means
     a broken install — callers treat that like a missing entry."""
     return _config_file(entry.config)
 
diff --git a/app/backends/sglomni/configs/higgs_audio_v3_tts.yaml b/app/backends/sglomni/configs/higgs_audio_v3_tts.yaml
new file mode 100644
index 0000000..74736fa
--- /dev/null
+++ b/app/backends/sglomni/configs/higgs_audio_v3_tts.yaml
@@ -0,0 +1,30 @@
+# Higgs Audio v3 TTS with VRAM headroom and a raised generation cap.
+#
+# The upstream default pipeline (HiggsTtsPipelineConfig) budgets VRAM as
+# gpu_memory_fraction 0.85 (tts_engine) + 0.10 (vocoder) + 0.03
+# (audio_encoder) = 0.98 of the card. The engine's static pool then fills
+# ~85% of a 24 GB GPU by itself (sglang mem_fraction_static = 0.85), and any
+# other VRAM consumer on the card (desktop, browsers) leaves too little room
+# for transient allocations: the first /v1/audio/speech request aborts with
+# "CUDA out of memory. Tried to allocate 14.00 MiB".
+#
+# 0.80 trims the engine's static pool by ~1.2 GB per 24 GB of VRAM while
+# leaving a KV cache pool (~10 GB on a 24 GB card) far larger than any
+# narration request needs. Cards with heavy other-GPU-process usage can go
+# lower (e.g. 0.75).
+#
+# The tts_engine factory also caps every request at max_new_tokens=2048
+# audio frames, and per-request values are clamped to that cap server-side
+# (make_higgs_scheduler_adapters) — the Higgs codec runs 75 frames per
+# second (24 kHz / 320 downsample), so the default is ~27 s of speech, which
+# silently truncates this tool's full 250-word sub-chunks (~100 s). Raising
+# the factory cap is the only way past it; the catalog also sends
+# max_new_tokens=12288 per request (the same value ZONOS2 uses) so a request
+# may use the room: 12288 frames ≈ 164 s.
+config_cls: HiggsTtsPipelineConfig
+model_path: bosonai/higgs-audio-v3-tts-4b
+stages:
+  tts_engine:
+    gpu_memory_fraction: 0.80
+    factory:
+      max_new_tokens: 12288
diff --git a/app/backends/sglomni/constants.py b/app/backends/sglomni/constants.py
index 97a4f3d..94a07bd 100644
--- a/app/backends/sglomni/constants.py
+++ b/app/backends/sglomni/constants.py
@@ -15,8 +15,9 @@ UV_PIP_PKG = "uv"
 # launching Python's, see pythonenv.prepare_env.
 
 # The vendored copies of upstream examples/configs/*.yaml (the `sgl-omni
-# serve --config` argument). Two catalog models run without one (Higgs,
-# ZONOS2) and are launched from --model-path alone.
+# serve --config` argument). ZONOS2 is the one catalog model without a
+# config (it runs from --model-path alone, with a GPU-conditional bf16
+# fallback config in its catalog entry).
 CONFIGS_DIR = Path(__file__).resolve().parent / "configs"
 
 # The managed server's port when the configured URL names none.
diff --git a/app/backends/sglomni/status.py b/app/backends/sglomni/status.py
index 099817e..2b77307 100644
--- a/app/backends/sglomni/status.py
+++ b/app/backends/sglomni/status.py
@@ -27,9 +27,9 @@ def build_spec(entry: ModelEntry) -> ServerSpec:
     Public because the run preparation (hub) and the CLI's managed-server
     bootstrap need to boot exactly the model their run selected, which can
     differ from detect()'s default. The config yaml is the vendored copy
-    from the catalog; models without one (Higgs, ZONOS2) run from
-    --model-path alone — except that a GPU the FP8 kernels cannot run
-    launches ZONOS2's vendored bf16 config instead (launch_config_path).
+    from the catalog; ZONOS2 alone runs from --model-path alone — except
+    that a GPU the FP8 kernels cannot run launches its vendored bf16 config
+    instead (launch_config_path).
     """
     url = config.SGLOMNI_API_URL
     argv: List[str] = [
diff --git a/app/docs/backend-sglomni.md b/app/docs/backend-sglomni.md
index bf7d3a9..4f553f9 100644
--- a/app/docs/backend-sglomni.md
+++ b/app/docs/backend-sglomni.md
@@ -60,7 +60,7 @@ inside the start timeout.
 | `qwen3_tts_0_6b_base` | Qwen3-TTS 0.6B Base | clone (reference required) | |
 | `qwen3_tts_1_7b_base` | Qwen3-TTS 1.7B Base | clone (reference required) | higher quality |
 | `qwen3_tts_1_7b_voicedesign` | Qwen3-TTS 1.7B VoiceDesign | `--instructions` | |
-| `higgs_audio_v3_tts` | Higgs Audio v3 TTS | default voice or clone | no config file needed |
+| `higgs_audio_v3_tts` | Higgs Audio v3 TTS | default voice or clone | launches with a vendored config: VRAM headroom for 24 GB cards + raised generation cap (requests carry `max_new_tokens=12288`; the engine's 2048-frame default is ~27 s and clamps per-request values) |
 | `moss_tts` | MOSS-TTS v1.5 | clone (reference required) | |
 | `moss_tts_local` | MOSS-TTS Local v1.5 | default voice or clone | 48 kHz |
 | `voxtral_tts` | Voxtral TTS 4B | preset named voices | e.g. `default`, `casual_male` |
@@ -155,6 +155,7 @@ app/envs/sglomni/bin/hf download bosonai/higgs-audio-v3-tts-4b
 # 5. Start a server manually (the managed flow does this for you)
 app/envs/sglomni/bin/sgl-omni serve \
   --model-path bosonai/higgs-audio-v3-tts-4b \
+  --config app/backends/sglomni/configs/higgs_audio_v3_tts.yaml \
   --port 8100
 
 # 6. Convert
@@ -162,9 +163,12 @@ python audiobook.py --backend sglomni --model higgs_audio_v3_tts \
   --api-url http://127.0.0.1:8100
 ```
 
-Models that take a vendored config file (all but Higgs and ZONOS2) add
+Models that take a vendored config file (all but ZONOS2) add
 `--config app/backends/sglomni/configs/<key>.yaml` to the serve command —
-the managed spec builds this from the catalog automatically.
+the managed spec builds this from the catalog automatically. Higgs's config
+also exists for a reason: the upstream pipeline budgets 98% of the card
+across its colocated stages, which OOMs on a 24 GB GPU once anything else
+uses VRAM, and its engine caps requests at ~27 s of speech.
 
 ## CLI examples
 
diff --git a/app/tests/test_backends_sglomni.py b/app/tests/test_backends_sglomni.py
index 1fd5f5a..ebac500 100644
--- a/app/tests/test_backends_sglomni.py
+++ b/app/tests/test_backends_sglomni.py
@@ -330,10 +330,12 @@ class BuildSpecTests(unittest.TestCase):
         self.assertEqual(port_flag, "--port")
         self.assertIn(port, str(spec.url))
 
-    def test_spec_omits_config_when_none_needed(self):
+    def test_spec_launches_the_higgs_config(self):
         entry = entry_by_key("higgs_audio_v3_tts")
         spec = status.build_spec(entry)
-        self.assertNotIn("--config", spec.argv)
+        self.assertIn("--config", spec.argv)
+        self.assertEqual(Path(spec.argv[spec.argv.index("--config") + 1]),
+                         catalog.config_path(entry))
 
 
 class GpuCapabilityTests(unittest.TestCase):
@@ -377,6 +379,30 @@ class GpuCapabilityTests(unittest.TestCase):
                     self.assertIsNone(status.gpu.compute_capability())
 
 
+class HiggsConfigTests(unittest.TestCase):
+    """The vendored Higgs config: VRAM headroom and a raised frame cap."""
+
+    def test_config_declares_the_repo_budget_and_frame_cap(self):
+        entry = entry_by_key("higgs_audio_v3_tts")
+        path = catalog.config_path(entry)
+        self.assertIsNotNone(path)
+        self.assertTrue(path.is_file(), f"missing {path}")
+        text = path.read_text(encoding="utf-8")
+        self.assertIn(f"model_path: {entry.repo}", text)
+        # The upstream pipeline budgets 0.98 of the card across its
+        # colocated stages; 0.80 leaves transient-allocation headroom on
+        # 24 GB cards (request-time CUDA OOM at the 0.85 default).
+        self.assertRegex(text, r"gpu_memory_fraction:\s*0\.80")
+        # The engine's 2048-frame default (~27 s at 75 fps) silently
+        # truncates a full 250-word sub-chunk; per-request values are
+        # clamped to this factory cap server-side.
+        self.assertRegex(text, r"max_new_tokens:\s*12288")
+
+    def test_entry_sends_the_raised_frame_cap_per_request(self):
+        entry = entry_by_key("higgs_audio_v3_tts")
+        self.assertEqual(entry.max_new_tokens, 12288)
+
+
 class Fp8FallbackTests(unittest.TestCase):
     """FP8-only pipelines fall back to a vendored bf16 config on old GPUs."""
 
diff --git a/app/tests/test_tts_sglomni.py b/app/tests/test_tts_sglomni.py
index 58ca57c..85327e1 100644
--- a/app/tests/test_tts_sglomni.py
+++ b/app/tests/test_tts_sglomni.py
@@ -254,8 +254,15 @@ class PayloadTests(unittest.TestCase):
         payload = client._request_payload("Hello.")
         self.assertEqual(payload["max_new_tokens"], 12288)
 
-    def test_models_without_a_cap_send_no_max_new_tokens(self):
+    def test_higgs_payload_raises_the_generation_cap(self):
+        """Higgs's 2048-frame engine default caps a request at ~27 s
+        (75 fps), below a full 250-word sub-chunk."""
         client = self._make_client("higgs_audio_v3_tts")
+        payload = client._request_payload("Hello.")
+        self.assertEqual(payload["max_new_tokens"], 12288)
+
+    def test_models_without_a_cap_send_no_max_new_tokens(self):
+        client = self._make_client("moss_tts")
         self.assertNotIn("max_new_tokens",
                          client._request_payload("Hello."))