feat: reserve 20% vram when running higgs with sglang-omni

1 files changed, 30 insertions, 0 deletions

diff --git a/app/backends/sglomni/configs/higgs_audio_v3_tts.yaml b/app/backends/sglomni/configs/higgs_audio_v3_tts.yaml
new file mode 100644
index 0000000..74736fa
--- /dev/null
+++ b/app/backends/sglomni/configs/higgs_audio_v3_tts.yaml
@@ -0,0 +1,30 @@
+# Higgs Audio v3 TTS with VRAM headroom and a raised generation cap.
+#
+# The upstream default pipeline (HiggsTtsPipelineConfig) budgets VRAM as
+# gpu_memory_fraction 0.85 (tts_engine) + 0.10 (vocoder) + 0.03
+# (audio_encoder) = 0.98 of the card. The engine's static pool then fills
+# ~85% of a 24 GB GPU by itself (sglang mem_fraction_static = 0.85), and any
+# other VRAM consumer on the card (desktop, browsers) leaves too little room
+# for transient allocations: the first /v1/audio/speech request aborts with
+# "CUDA out of memory. Tried to allocate 14.00 MiB".
+#
+# 0.80 trims the engine's static pool by ~1.2 GB per 24 GB of VRAM while
+# leaving a KV cache pool (~10 GB on a 24 GB card) far larger than any
+# narration request needs. Cards with heavy other-GPU-process usage can go
+# lower (e.g. 0.75).
+#
+# The tts_engine factory also caps every request at max_new_tokens=2048
+# audio frames, and per-request values are clamped to that cap server-side
+# (make_higgs_scheduler_adapters) — the Higgs codec runs 75 frames per
+# second (24 kHz / 320 downsample), so the default is ~27 s of speech, which
+# silently truncates this tool's full 250-word sub-chunks (~100 s). Raising
+# the factory cap is the only way past it; the catalog also sends
+# max_new_tokens=12288 per request (the same value ZONOS2 uses) so a request
+# may use the room: 12288 frames ≈ 164 s.
+config_cls: HiggsTtsPipelineConfig
+model_path: bosonai/higgs-audio-v3-tts-4b
+stages:
+  tts_engine:
+    gpu_memory_fraction: 0.80
+    factory:
+      max_new_tokens: 12288