diff options
Diffstat (limited to 'app')
| -rw-r--r-- | app/backends/sglomni/configs/moss_tts.yaml | 20 | ||||
| -rw-r--r-- | app/docs/backend-sglomni.md | 14 | ||||
| -rw-r--r-- | app/tests/test_backends_sglomni.py | 40 |
3 files changed, 49 insertions, 25 deletions
diff --git a/app/backends/sglomni/configs/moss_tts.yaml b/app/backends/sglomni/configs/moss_tts.yaml index 41bddd5..1de9a4c 100644 --- a/app/backends/sglomni/configs/moss_tts.yaml +++ b/app/backends/sglomni/configs/moss_tts.yaml @@ -11,14 +11,24 @@ # fix; note moss_tts_local does NOT need this — MossTTSLocalPipelineConfig # already budgets its colocated stages explicitly: 0.15/0.67/0.18). # -# 0.70 pins the static pool at ~70% of the card (~16.5 GB on a 24 GB GPU — -# a KV pool far larger than any narration request needs) and leaves ~7 GB -# for the vocoder, CUDA graphs, transient allocations, and other GPU -# processes. Precedent: dots_tts.yaml pins this same knob. +# Unlike those models, MOSS cannot take the 0.70 pin. The v1.5 AR weights +# alone are ~17.1 GB in bf16, and on a 24 GB card the engine's profiler only +# sees ~20.7 GB free at that point — the colocated preprocessing stage +# (MOSS-Audio-Tokenizer) is already resident. The KV budget is computed as +# free_after_weights minus pre_model_load_memory * (1 - mem_fraction_static), +# so 0.70 charges a negative KV budget and the boot aborts with "Loaded +# weights leave no GPU memory for the KV cache" (floor ~= 0.828 on a clean +# 24 GB card, and higher with any other GPU process on the card). 0.87 +# leaves a ~0.8 GB KV pool — far larger than any narration sub-request +# needs — with ~2.7 GB of slack for the CUDA graphs, the colocated vocoder, +# and transient allocations. If the first requests OOM anyway, drop to +# 0.86; on a guaranteed-headless card 0.88 buys a ~1 GB KV pool. The +# smaller pinned models (qwen3_tts, voxtral, zonos2 bf16 — weights at or +# under ~11.5 GB) keep 0.70. config_cls: MossTTSPipelineConfig model_path: OpenMOSS-Team/MOSS-TTS-v1.5 stages: tts_engine: engine: - mem_fraction_static: 0.70 + mem_fraction_static: 0.87 diff --git a/app/docs/backend-sglomni.md b/app/docs/backend-sglomni.md index 9091ebd..7af6a72 100644 --- a/app/docs/backend-sglomni.md +++ b/app/docs/backend-sglomni.md @@ -180,9 +180,17 @@ auto-sized to nearly all free VRAM at boot. By the time a real request runs, the engine's CUDA graphs (~1.5 GB) and the colocated vocoder are resident too and only tens of MiB are left — the transient allocations generation needs do not fit. The vendored configs for those models pin -the pool at `mem_fraction_static: 0.70` (~7 GB of headroom on a 24 GB -card) exactly for this; Higgs (`gpu_memory_fraction: 0.80`) and ZONOS2's -bf16 fallback (`0.70`) carry the same treatment. If you hit the failure +the pool exactly for this — 0.70 for the small ones (~7 GB of headroom +on a 24 GB card); Higgs (`gpu_memory_fraction: 0.80`) and ZONOS2's +bf16 fallback (`0.70`) carry the same treatment. MOSS-TTS is the +exception: its ~17.1 GB of v1.5 bf16 weights alone exceed the 0.70 +budget on the ~20.7 GB the engine profiler sees at that point (its +colocated audio-tokenizer stage is already resident), so a 0.70 pin +aborts the boot outright with "Loaded weights leave no GPU memory for +the KV cache" (the floor is ~0.828 on a clean 24 GB card) instead of +fixing the request-time OOM — it runs `mem_fraction_static: 0.87`, +keeping a ~0.8 GB KV pool with ~2.7 GB of slack for the CUDA graphs +and the colocated vocoder. If you hit the failure on a server started by hand or an older checkout, pin the budget on the command line and let this tool use the running server as-is: diff --git a/app/tests/test_backends_sglomni.py b/app/tests/test_backends_sglomni.py index e61d51e..54659d6 100644 --- a/app/tests/test_backends_sglomni.py +++ b/app/tests/test_backends_sglomni.py @@ -447,30 +447,36 @@ class EngineMemoryBudgetTests(unittest.TestCase): VRAM when mem_fraction_static is unset (qwen3_tts, moss_tts, voxtral): on a 24 GB card the first /v1/audio/speech request aborts with "CUDA out of memory. Tried to allocate ~100 MiB" and every retry fails - identically. These vendored configs pin the pool at 0.70, leaving ~7 GB - for the colocated vocoder, CUDA graphs, and other GPU processes. + identically. These vendored configs pin the pool below auto-size, + leaving several GB for the colocated vocoder, CUDA graphs, and other + GPU processes — 0.70 for the small models. moss_tts pins 0.87 instead: + its ~17.1 GB of v1.5 bf16 weights alone exceed the 0.70 budget on the + ~20.7 GB the engine profiler sees on a 24 GB card (its colocated + tokenizer stage is already resident), so a 0.70 pin would abort the + boot with "Loaded weights leave no GPU memory for the KV cache" + instead of fixing the request-time OOM. """ - # (catalog key, engine stage name). Voxtral's engine stage is - # tts_generation; every other engine-bearing pipeline names it - # tts_engine. moss_tts_local is intentionally absent — its upstream - # config class already budgets its colocated stages explicitly - # (0.15 preprocessing / 0.67 AR / 0.18 vocoder), and pinning would - # fight that logic (the codec reserve derives from the fractions). + # (catalog key, engine stage name, pinned mem_fraction_static). + # Voxtral's engine stage is tts_generation; every other engine-bearing + # pipeline names it tts_engine. moss_tts_local is intentionally absent — + # its upstream config class already budgets its colocated stages + # explicitly (0.15 preprocessing / 0.67 AR / 0.18 vocoder), and pinning + # would fight that logic (the codec reserve derives from the fractions). # s2-pro's OOM is model size, not budgeting (upstream issue #359). PINNED = ( - ("qwen3_tts_0_6b_base", "tts_engine"), - ("qwen3_tts_0_6b_customvoice", "tts_engine"), - ("qwen3_tts_1_7b_base", "tts_engine"), - ("qwen3_tts_1_7b_voicedesign", "tts_engine"), - ("moss_tts", "tts_engine"), - ("voxtral_tts", "tts_generation"), + ("qwen3_tts_0_6b_base", "tts_engine", 0.70), + ("qwen3_tts_0_6b_customvoice", "tts_engine", 0.70), + ("qwen3_tts_1_7b_base", "tts_engine", 0.70), + ("qwen3_tts_1_7b_voicedesign", "tts_engine", 0.70), + ("moss_tts", "tts_engine", 0.87), + ("voxtral_tts", "tts_generation", 0.70), ) UNPINNED = ("moss_tts_local", "fish_s2_pro") def test_engine_budget_is_pinned_under_the_engine_stage(self): - for key, stage in self.PINNED: - with self.subTest(key=key, stage=stage): + for key, stage, fraction in self.PINNED: + with self.subTest(key=key, stage=stage, fraction=fraction): path = config_path(entry_by_key(key)) self.assertIsNotNone(path) self.assertTrue(path.is_file(), f"missing {path}") @@ -479,7 +485,7 @@ class EngineMemoryBudgetTests(unittest.TestCase): "stages:\n" f" {stage}:\n" " engine:\n" - " mem_fraction_static: 0.70\n", + f" mem_fraction_static: {fraction:.2f}\n", text) def test_already_budgeted_and_oversized_models_stay_unpinned(self): |