blob: 79d5d261938721608794646cf32af3898409a8b1 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
|
# ZONOS2 without the FP8-quantized MoE pipeline.
#
# sglang-omni's default ZONOS2 config (Zonos2PipelineConfig) hardcodes
# `fp8: True` in the tts_engine stage factory: the MoE experts are
# dynamically quantized bf16 -> fp8 at load time, and sglang's fused-MoE
# Triton kernel for fp8e4nv only compiles on compute capability 8.9+
# (RTX 4090/5090, Hopper). On older GPUs the server dies mid-boot with
# `type fp8e4nv not supported in this architecture`.
#
# This copy turns FP8 off (`factory.fp8: false` overrides the hardcoded
# kwarg — free-form factory keys pass through to the stage factory), so
# the model runs in bf16 and works on e.g. Ampere (RTX 30xx, A100) at
# roughly twice the MoE VRAM. The managed spec selects it automatically
# on GPUs the FP8 path cannot run (backends.sglomni.status.build_spec).
#
# The bf16 weights (~11.5 GB) also need a bigger static-pool budget than
# the builder's default 0.5: on a 24 GB card 0.5 leaves no room for the
# KV cache inside 12 GB, and the server aborts with "Loaded weights
# leave no GPU memory for the KV cache" (the profiler asks for >=0.64).
# 0.70 gives ~16.8 GB static -> ~1.4 GB KV, and leaves ~7 GB of the card
# for the colocated speaker-encode/vocoder stages.
config_cls: Zonos2PipelineConfig
model_path: Zyphra/zonos2
stages:
tts_engine:
factory:
fp8: false
mem_fraction_static: 0.70
|
