blob: 1de9a4cf2671c928edd32961d09a19f1501f7501 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
|
# MOSS-TTS v1.5 with a pinned AR-engine memory budget.
#
# The upstream pipeline (MossTTSPipelineConfig) colocates its three stages
# (preprocessing -> tts_engine -> vocoder) in one process on GPU 0 and leaves
# the engine's sglang mem_fraction_static unset: the static pool (weights +
# KV cache) is auto-sized to nearly all free VRAM at boot. On a 24 GB card
# that leaves only tens of MiB free once the engine's CUDA graphs and the
# colocated vocoder are resident — the first /v1/audio/speech request aborts
# with "CUDA out of memory. Tried to allocate ~100 MiB" and every retry fails
# identically (the same failure the qwen3_tts/higgs/zonos2 vendored configs
# fix; note moss_tts_local does NOT need this — MossTTSLocalPipelineConfig
# already budgets its colocated stages explicitly: 0.15/0.67/0.18).
#
# Unlike those models, MOSS cannot take the 0.70 pin. The v1.5 AR weights
# alone are ~17.1 GB in bf16, and on a 24 GB card the engine's profiler only
# sees ~20.7 GB free at that point — the colocated preprocessing stage
# (MOSS-Audio-Tokenizer) is already resident. The KV budget is computed as
# free_after_weights minus pre_model_load_memory * (1 - mem_fraction_static),
# so 0.70 charges a negative KV budget and the boot aborts with "Loaded
# weights leave no GPU memory for the KV cache" (floor ~= 0.828 on a clean
# 24 GB card, and higher with any other GPU process on the card). 0.87
# leaves a ~0.8 GB KV pool — far larger than any narration sub-request
# needs — with ~2.7 GB of slack for the CUDA graphs, the colocated vocoder,
# and transient allocations. If the first requests OOM anyway, drop to
# 0.86; on a guaranteed-headless card 0.88 buys a ~1 GB KV pool. The
# smaller pinned models (qwen3_tts, voxtral, zonos2 bf16 — weights at or
# under ~11.5 GB) keep 0.70.
config_cls: MossTTSPipelineConfig
model_path: OpenMOSS-Team/MOSS-TTS-v1.5
stages:
tts_engine:
engine:
mem_fraction_static: 0.87
|
