fix: increase gpu memory for MOSS with gplang-omni to .87

3 files changed, 49 insertions, 25 deletions

diff --git a/app/backends/sglomni/configs/moss_tts.yaml b/app/backends/sglomni/configs/moss_tts.yaml
index 41bddd5..1de9a4c 100644
--- a/app/backends/sglomni/configs/moss_tts.yaml
+++ b/app/backends/sglomni/configs/moss_tts.yaml
@@ -11,14 +11,24 @@
 # fix; note moss_tts_local does NOT need this — MossTTSLocalPipelineConfig
 # already budgets its colocated stages explicitly: 0.15/0.67/0.18).
 #
-# 0.70 pins the static pool at ~70% of the card (~16.5 GB on a 24 GB GPU —
-# a KV pool far larger than any narration request needs) and leaves ~7 GB
-# for the vocoder, CUDA graphs, transient allocations, and other GPU
-# processes. Precedent: dots_tts.yaml pins this same knob.
+# Unlike those models, MOSS cannot take the 0.70 pin. The v1.5 AR weights
+# alone are ~17.1 GB in bf16, and on a 24 GB card the engine's profiler only
+# sees ~20.7 GB free at that point — the colocated preprocessing stage
+# (MOSS-Audio-Tokenizer) is already resident. The KV budget is computed as
+# free_after_weights minus pre_model_load_memory * (1 - mem_fraction_static),
+# so 0.70 charges a negative KV budget and the boot aborts with "Loaded
+# weights leave no GPU memory for the KV cache" (floor ~= 0.828 on a clean
+# 24 GB card, and higher with any other GPU process on the card). 0.87
+# leaves a ~0.8 GB KV pool — far larger than any narration sub-request
+# needs — with ~2.7 GB of slack for the CUDA graphs, the colocated vocoder,
+# and transient allocations. If the first requests OOM anyway, drop to
+# 0.86; on a guaranteed-headless card 0.88 buys a ~1 GB KV pool. The
+# smaller pinned models (qwen3_tts, voxtral, zonos2 bf16 — weights at or
+# under ~11.5 GB) keep 0.70.
 config_cls: MossTTSPipelineConfig
 model_path: OpenMOSS-Team/MOSS-TTS-v1.5
 
 stages:
   tts_engine:
     engine:
-      mem_fraction_static: 0.70
+      mem_fraction_static: 0.87
diff --git a/app/docs/backend-sglomni.md b/app/docs/backend-sglomni.md
index 9091ebd..7af6a72 100644
--- a/app/docs/backend-sglomni.md
+++ b/app/docs/backend-sglomni.md
@@ -180,9 +180,17 @@ auto-sized to nearly all free VRAM at boot. By the time a real request
 runs, the engine's CUDA graphs (~1.5 GB) and the colocated vocoder are
 resident too and only tens of MiB are left — the transient allocations
 generation needs do not fit. The vendored configs for those models pin
-the pool at `mem_fraction_static: 0.70` (~7 GB of headroom on a 24 GB
-card) exactly for this; Higgs (`gpu_memory_fraction: 0.80`) and ZONOS2's
-bf16 fallback (`0.70`) carry the same treatment. If you hit the failure
+the pool exactly for this — 0.70 for the small ones (~7 GB of headroom
+on a 24 GB card); Higgs (`gpu_memory_fraction: 0.80`) and ZONOS2's
+bf16 fallback (`0.70`) carry the same treatment. MOSS-TTS is the
+exception: its ~17.1 GB of v1.5 bf16 weights alone exceed the 0.70
+budget on the ~20.7 GB the engine profiler sees at that point (its
+colocated audio-tokenizer stage is already resident), so a 0.70 pin
+aborts the boot outright with "Loaded weights leave no GPU memory for
+the KV cache" (the floor is ~0.828 on a clean 24 GB card) instead of
+fixing the request-time OOM — it runs `mem_fraction_static: 0.87`,
+keeping a ~0.8 GB KV pool with ~2.7 GB of slack for the CUDA graphs
+and the colocated vocoder. If you hit the failure
 on a server started by hand or an older checkout, pin the budget on the
 command line and let this tool use the running server as-is:
 
diff --git a/app/tests/test_backends_sglomni.py b/app/tests/test_backends_sglomni.py
index e61d51e..54659d6 100644
--- a/app/tests/test_backends_sglomni.py
+++ b/app/tests/test_backends_sglomni.py
@@ -447,30 +447,36 @@ class EngineMemoryBudgetTests(unittest.TestCase):
     VRAM when mem_fraction_static is unset (qwen3_tts, moss_tts, voxtral):
     on a 24 GB card the first /v1/audio/speech request aborts with
     "CUDA out of memory. Tried to allocate ~100 MiB" and every retry fails
-    identically. These vendored configs pin the pool at 0.70, leaving ~7 GB
-    for the colocated vocoder, CUDA graphs, and other GPU processes.
+    identically. These vendored configs pin the pool below auto-size,
+    leaving several GB for the colocated vocoder, CUDA graphs, and other
+    GPU processes — 0.70 for the small models. moss_tts pins 0.87 instead:
+    its ~17.1 GB of v1.5 bf16 weights alone exceed the 0.70 budget on the
+    ~20.7 GB the engine profiler sees on a 24 GB card (its colocated
+    tokenizer stage is already resident), so a 0.70 pin would abort the
+    boot with "Loaded weights leave no GPU memory for the KV cache"
+    instead of fixing the request-time OOM.
     """
 
-    # (catalog key, engine stage name). Voxtral's engine stage is
-    # tts_generation; every other engine-bearing pipeline names it
-    # tts_engine. moss_tts_local is intentionally absent — its upstream
-    # config class already budgets its colocated stages explicitly
-    # (0.15 preprocessing / 0.67 AR / 0.18 vocoder), and pinning would
-    # fight that logic (the codec reserve derives from the fractions).
+    # (catalog key, engine stage name, pinned mem_fraction_static).
+    # Voxtral's engine stage is tts_generation; every other engine-bearing
+    # pipeline names it tts_engine. moss_tts_local is intentionally absent —
+    # its upstream config class already budgets its colocated stages
+    # explicitly (0.15 preprocessing / 0.67 AR / 0.18 vocoder), and pinning
+    # would fight that logic (the codec reserve derives from the fractions).
     # s2-pro's OOM is model size, not budgeting (upstream issue #359).
     PINNED = (
-        ("qwen3_tts_0_6b_base", "tts_engine"),
-        ("qwen3_tts_0_6b_customvoice", "tts_engine"),
-        ("qwen3_tts_1_7b_base", "tts_engine"),
-        ("qwen3_tts_1_7b_voicedesign", "tts_engine"),
-        ("moss_tts", "tts_engine"),
-        ("voxtral_tts", "tts_generation"),
+        ("qwen3_tts_0_6b_base", "tts_engine", 0.70),
+        ("qwen3_tts_0_6b_customvoice", "tts_engine", 0.70),
+        ("qwen3_tts_1_7b_base", "tts_engine", 0.70),
+        ("qwen3_tts_1_7b_voicedesign", "tts_engine", 0.70),
+        ("moss_tts", "tts_engine", 0.87),
+        ("voxtral_tts", "tts_generation", 0.70),
     )
     UNPINNED = ("moss_tts_local", "fish_s2_pro")
 
     def test_engine_budget_is_pinned_under_the_engine_stage(self):
-        for key, stage in self.PINNED:
-            with self.subTest(key=key, stage=stage):
+        for key, stage, fraction in self.PINNED:
+            with self.subTest(key=key, stage=stage, fraction=fraction):
                 path = config_path(entry_by_key(key))
                 self.assertIsNotNone(path)
                 self.assertTrue(path.is_file(), f"missing {path}")
@@ -479,7 +485,7 @@ class EngineMemoryBudgetTests(unittest.TestCase):
                     "stages:\n"
                     f"  {stage}:\n"
                     "    engine:\n"
-                    "      mem_fraction_static: 0.70\n",
+                    f"      mem_fraction_static: {fraction:.2f}\n",
                     text)
 
     def test_already_budgeted_and_oversized_models_stay_unpinned(self):