diff options
Diffstat (limited to 'app/backends/sglomni/configs/zonos2_bf16.yaml')
| -rw-r--r-- | app/backends/sglomni/configs/zonos2_bf16.yaml | 28 |
1 files changed, 28 insertions, 0 deletions
diff --git a/app/backends/sglomni/configs/zonos2_bf16.yaml b/app/backends/sglomni/configs/zonos2_bf16.yaml new file mode 100644 index 0000000..79d5d26 --- /dev/null +++ b/app/backends/sglomni/configs/zonos2_bf16.yaml @@ -0,0 +1,28 @@ +# ZONOS2 without the FP8-quantized MoE pipeline. +# +# sglang-omni's default ZONOS2 config (Zonos2PipelineConfig) hardcodes +# `fp8: True` in the tts_engine stage factory: the MoE experts are +# dynamically quantized bf16 -> fp8 at load time, and sglang's fused-MoE +# Triton kernel for fp8e4nv only compiles on compute capability 8.9+ +# (RTX 4090/5090, Hopper). On older GPUs the server dies mid-boot with +# `type fp8e4nv not supported in this architecture`. +# +# This copy turns FP8 off (`factory.fp8: false` overrides the hardcoded +# kwarg — free-form factory keys pass through to the stage factory), so +# the model runs in bf16 and works on e.g. Ampere (RTX 30xx, A100) at +# roughly twice the MoE VRAM. The managed spec selects it automatically +# on GPUs the FP8 path cannot run (backends.sglomni.status.build_spec). +# +# The bf16 weights (~11.5 GB) also need a bigger static-pool budget than +# the builder's default 0.5: on a 24 GB card 0.5 leaves no room for the +# KV cache inside 12 GB, and the server aborts with "Loaded weights +# leave no GPU memory for the KV cache" (the profiler asks for >=0.64). +# 0.70 gives ~16.8 GB static -> ~1.4 GB KV, and leaves ~7 GB of the card +# for the colocated speaker-encode/vocoder stages. +config_cls: Zonos2PipelineConfig +model_path: Zyphra/zonos2 +stages: + tts_engine: + factory: + fp8: false + mem_fraction_static: 0.70 |