fix: warn before using a likely too-big chunk size for sglang-omni models

1 files changed, 12 insertions, 7 deletions

diff --git a/app/backends/sglomni/configs/higgs_audio_v3_tts.yaml b/app/backends/sglomni/configs/higgs_audio_v3_tts.yaml
index 74736fa..8a42771 100644
--- a/app/backends/sglomni/configs/higgs_audio_v3_tts.yaml
+++ b/app/backends/sglomni/configs/higgs_audio_v3_tts.yaml
@@ -9,22 +9,27 @@
 # "CUDA out of memory. Tried to allocate 14.00 MiB".
 #
 # 0.80 trims the engine's static pool by ~1.2 GB per 24 GB of VRAM while
-# leaving a KV cache pool (~10 GB on a 24 GB card) far larger than any
-# narration request needs. Cards with heavy other-GPU-process usage can go
-# lower (e.g. 0.75).
+# leaving a KV cache pool far larger than any narration request needs.
+# Cards with heavy other-GPU-process usage can go lower (e.g. 0.75).
 #
 # The tts_engine factory also caps every request at max_new_tokens=2048
 # audio frames, and per-request values are clamped to that cap server-side
 # (make_higgs_scheduler_adapters) — the Higgs codec runs 75 frames per
 # second (24 kHz / 320 downsample), so the default is ~27 s of speech, which
 # silently truncates this tool's full 250-word sub-chunks (~100 s). Raising
-# the factory cap is the only way past it; the catalog also sends
-# max_new_tokens=12288 per request (the same value ZONOS2 uses) so a request
-# may use the room: 12288 frames ≈ 164 s.
+# the factory cap is the only way past it, but the ceiling is hard: upstream
+# pins the thinker engine's context_length at 4096 (HiggsTtsEngineBuilder —
+# not overridable), and the scheduler rejects any request whose prompt
+# tokens (including the reference-audio tokens) plus max_new_tokens exceed
+# that window ("Request requires more tokens than the thinker KV cache can
+# hold", kv_capacity=4095, on every GPU). The cap therefore lands at 3000
+# frames ≈ 40 s — the most the window allows with prompt headroom (an
+# 80-word chunk with a 20.5 s reference measured 684 prompt tokens) — and
+# the catalog caps sub-requests at 80 words to match (chunk_words).
 config_cls: HiggsTtsPipelineConfig
 model_path: bosonai/higgs-audio-v3-tts-4b
 stages:
   tts_engine:
     gpu_memory_fraction: 0.80
     factory:
-      max_new_tokens: 12288
+      max_new_tokens: 3000