From 8579517a35ef1865fc9b428899d73d52dcb27a14 Mon Sep 17 00:00:00 2001 From: historia Date: Wed, 2 Sep 2026 01:26:09 -0400 Subject: feat: sglang backend support --- app/backends/sglomni/configs/zonos2_bf16.yaml | 28 +++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 app/backends/sglomni/configs/zonos2_bf16.yaml (limited to 'app/backends/sglomni/configs/zonos2_bf16.yaml') diff --git a/app/backends/sglomni/configs/zonos2_bf16.yaml b/app/backends/sglomni/configs/zonos2_bf16.yaml new file mode 100644 index 0000000..79d5d26 --- /dev/null +++ b/app/backends/sglomni/configs/zonos2_bf16.yaml @@ -0,0 +1,28 @@ +# ZONOS2 without the FP8-quantized MoE pipeline. +# +# sglang-omni's default ZONOS2 config (Zonos2PipelineConfig) hardcodes +# `fp8: True` in the tts_engine stage factory: the MoE experts are +# dynamically quantized bf16 -> fp8 at load time, and sglang's fused-MoE +# Triton kernel for fp8e4nv only compiles on compute capability 8.9+ +# (RTX 4090/5090, Hopper). On older GPUs the server dies mid-boot with +# `type fp8e4nv not supported in this architecture`. +# +# This copy turns FP8 off (`factory.fp8: false` overrides the hardcoded +# kwarg — free-form factory keys pass through to the stage factory), so +# the model runs in bf16 and works on e.g. Ampere (RTX 30xx, A100) at +# roughly twice the MoE VRAM. The managed spec selects it automatically +# on GPUs the FP8 path cannot run (backends.sglomni.status.build_spec). +# +# The bf16 weights (~11.5 GB) also need a bigger static-pool budget than +# the builder's default 0.5: on a 24 GB card 0.5 leaves no room for the +# KV cache inside 12 GB, and the server aborts with "Loaded weights +# leave no GPU memory for the KV cache" (the profiler asks for >=0.64). +# 0.70 gives ~16.8 GB static -> ~1.4 GB KV, and leaves ~7 GB of the card +# for the colocated speaker-encode/vocoder stages. +config_cls: Zonos2PipelineConfig +model_path: Zyphra/zonos2 +stages: + tts_engine: + factory: + fp8: false + mem_fraction_static: 0.70 -- cgit v1.2.3