feat: sglang backend support

1 files changed, 28 insertions, 0 deletions

diff --git a/app/backends/sglomni/configs/zonos2_bf16.yaml b/app/backends/sglomni/configs/zonos2_bf16.yaml
new file mode 100644
index 0000000..79d5d26
--- /dev/null
+++ b/app/backends/sglomni/configs/zonos2_bf16.yaml
@@ -0,0 +1,28 @@
+# ZONOS2 without the FP8-quantized MoE pipeline.
+#
+# sglang-omni's default ZONOS2 config (Zonos2PipelineConfig) hardcodes
+# `fp8: True` in the tts_engine stage factory: the MoE experts are
+# dynamically quantized bf16 -> fp8 at load time, and sglang's fused-MoE
+# Triton kernel for fp8e4nv only compiles on compute capability 8.9+
+# (RTX 4090/5090, Hopper). On older GPUs the server dies mid-boot with
+# `type fp8e4nv not supported in this architecture`.
+#
+# This copy turns FP8 off (`factory.fp8: false` overrides the hardcoded
+# kwarg — free-form factory keys pass through to the stage factory), so
+# the model runs in bf16 and works on e.g. Ampere (RTX 30xx, A100) at
+# roughly twice the MoE VRAM. The managed spec selects it automatically
+# on GPUs the FP8 path cannot run (backends.sglomni.status.build_spec).
+#
+# The bf16 weights (~11.5 GB) also need a bigger static-pool budget than
+# the builder's default 0.5: on a 24 GB card 0.5 leaves no room for the
+# KV cache inside 12 GB, and the server aborts with "Loaded weights
+# leave no GPU memory for the KV cache" (the profiler asks for >=0.64).
+# 0.70 gives ~16.8 GB static -> ~1.4 GB KV, and leaves ~7 GB of the card
+# for the colocated speaker-encode/vocoder stages.
+config_cls: Zonos2PipelineConfig
+model_path: Zyphra/zonos2
+stages:
+  tts_engine:
+    factory:
+      fp8: false
+      mem_fraction_static: 0.70