diff options
Diffstat (limited to 'app/backends/sglomni')
| -rw-r--r-- | app/backends/sglomni/catalog.py | 15 | ||||
| -rw-r--r-- | app/backends/sglomni/configs/moss_tts.yaml | 22 | ||||
| -rw-r--r-- | app/backends/sglomni/configs/qwen3_tts_0_6b.yaml | 21 | ||||
| -rw-r--r-- | app/backends/sglomni/configs/qwen3_tts_0_6b_customvoice.yaml | 21 | ||||
| -rw-r--r-- | app/backends/sglomni/configs/qwen3_tts_1_7b.yaml | 21 | ||||
| -rw-r--r-- | app/backends/sglomni/configs/qwen3_tts_1_7b_voicedesign.yaml | 21 | ||||
| -rw-r--r-- | app/backends/sglomni/configs/voxtral_tts.yaml | 23 | ||||
| -rw-r--r-- | app/backends/sglomni/wizard.py | 2 |
8 files changed, 143 insertions, 3 deletions
diff --git a/app/backends/sglomni/catalog.py b/app/backends/sglomni/catalog.py index 160eb8b..f20509f 100644 --- a/app/backends/sglomni/catalog.py +++ b/app/backends/sglomni/catalog.py @@ -104,10 +104,16 @@ _QWEN_EXTRAS: Tuple[Extra, ...] = ( _SOX_HINT = ("install the sox system package (e.g. sudo pacman -S sox, " "sudo apt install sox, brew install sox)") # The Fish Audio and ZONOS2 pipelines use the Descript DAC codec, which -# upstream installs WITH dependencies (nothing conflicts). +# upstream installs WITH dependencies — but descript-audiotools carries a +# vestigial 2021-era pin, protobuf<3.20 (its code never imports protobuf), +# that downgrades the protobuf 6.x the sglang-omni stack itself needs +# (smg-grpc-proto, grpcio health/reflection, cutlass-dsl, onnxruntime, +# s3prl all demand >=4). The final extra re-pins protobuf AFTER +# descript-audiotools so the downgrade never survives the install. _DAC_EXTRAS: Tuple[Extra, ...] = ( ("descript-audiotools==0.7.2", False), - ("descript-audio-codec==1.0.0", False)) + ("descript-audio-codec==1.0.0", False), + ("protobuf==6.33.6", False)) # Companion distributions whose top-level import name differs from the pip # name's plain dash-to-underscore normalization (verified against their @@ -115,6 +121,11 @@ _DAC_EXTRAS: Tuple[Extra, ...] = ( _EXTRA_IMPORT_OVERRIDES = { "descript-audiotools": "audiotools", "descript-audio-codec": "dac", + # Not a naming quirk: a version marker. `google.protobuf` imports just + # as happily at the downgraded 3.19.6 as at the required 6.33.6, but + # this submodule only exists from protobuf 5.27 — so the venv probe + # fails (and the extra re-installs) whenever the audiotools pin won. + "protobuf": "google.protobuf.runtime_version", } diff --git a/app/backends/sglomni/configs/moss_tts.yaml b/app/backends/sglomni/configs/moss_tts.yaml index ef8d8fa..41bddd5 100644 --- a/app/backends/sglomni/configs/moss_tts.yaml +++ b/app/backends/sglomni/configs/moss_tts.yaml @@ -1,2 +1,24 @@ +# MOSS-TTS v1.5 with a pinned AR-engine memory budget. +# +# The upstream pipeline (MossTTSPipelineConfig) colocates its three stages +# (preprocessing -> tts_engine -> vocoder) in one process on GPU 0 and leaves +# the engine's sglang mem_fraction_static unset: the static pool (weights + +# KV cache) is auto-sized to nearly all free VRAM at boot. On a 24 GB card +# that leaves only tens of MiB free once the engine's CUDA graphs and the +# colocated vocoder are resident — the first /v1/audio/speech request aborts +# with "CUDA out of memory. Tried to allocate ~100 MiB" and every retry fails +# identically (the same failure the qwen3_tts/higgs/zonos2 vendored configs +# fix; note moss_tts_local does NOT need this — MossTTSLocalPipelineConfig +# already budgets its colocated stages explicitly: 0.15/0.67/0.18). +# +# 0.70 pins the static pool at ~70% of the card (~16.5 GB on a 24 GB GPU — +# a KV pool far larger than any narration request needs) and leaves ~7 GB +# for the vocoder, CUDA graphs, transient allocations, and other GPU +# processes. Precedent: dots_tts.yaml pins this same knob. config_cls: MossTTSPipelineConfig model_path: OpenMOSS-Team/MOSS-TTS-v1.5 + +stages: + tts_engine: + engine: + mem_fraction_static: 0.70 diff --git a/app/backends/sglomni/configs/qwen3_tts_0_6b.yaml b/app/backends/sglomni/configs/qwen3_tts_0_6b.yaml index a712ef9..bbb88bb 100644 --- a/app/backends/sglomni/configs/qwen3_tts_0_6b.yaml +++ b/app/backends/sglomni/configs/qwen3_tts_0_6b.yaml @@ -1,2 +1,23 @@ +# Qwen3-TTS 0.6B Base with a pinned AR-engine memory budget. +# +# The upstream pipeline (Qwen3TTSPipelineConfig) colocates its three stages +# (preprocessing -> tts_engine -> vocoder) in one process on GPU 0 and leaves +# the engine's sglang mem_fraction_static unset: the static pool (weights + +# KV cache) is auto-sized to nearly all free VRAM at boot. On a 24 GB card +# that leaves only tens of MiB free once the engine's CUDA graphs (~1.5 GB) +# and the colocated vocoder are resident — the first /v1/audio/speech request +# aborts with "CUDA out of memory. Tried to allocate ~100 MiB" and every +# retry fails identically (the weights themselves are tiny: 0.6B bf16). +# +# 0.70 pins the static pool at ~70% of the card (~16.5 GB on a 24 GB GPU — +# a KV pool far larger than any narration request needs) and leaves ~7 GB +# for the vocoder, CUDA graphs, transient allocations, and other GPU +# processes. Precedent: dots_tts.yaml pins this same knob; zonos2_bf16.yaml +# and higgs_audio_v3_tts.yaml fix the same 24 GB OOM class their way. config_cls: Qwen3TTSPipelineConfig model_path: Qwen/Qwen3-TTS-12Hz-0.6B-Base + +stages: + tts_engine: + engine: + mem_fraction_static: 0.70 diff --git a/app/backends/sglomni/configs/qwen3_tts_0_6b_customvoice.yaml b/app/backends/sglomni/configs/qwen3_tts_0_6b_customvoice.yaml index 6b284da..7481058 100644 --- a/app/backends/sglomni/configs/qwen3_tts_0_6b_customvoice.yaml +++ b/app/backends/sglomni/configs/qwen3_tts_0_6b_customvoice.yaml @@ -1,2 +1,23 @@ +# Qwen3-TTS 0.6B CustomVoice with a pinned AR-engine memory budget. +# +# The upstream pipeline (Qwen3TTSPipelineConfig) colocates its three stages +# (preprocessing -> tts_engine -> vocoder) in one process on GPU 0 and leaves +# the engine's sglang mem_fraction_static unset: the static pool (weights + +# KV cache) is auto-sized to nearly all free VRAM at boot. On a 24 GB card +# that leaves only tens of MiB free once the engine's CUDA graphs (~1.5 GB) +# and the colocated vocoder are resident — the first /v1/audio/speech request +# aborts with "CUDA out of memory. Tried to allocate ~100 MiB" and every +# retry fails identically (the weights themselves are tiny: 0.6B bf16). +# +# 0.70 pins the static pool at ~70% of the card (~16.5 GB on a 24 GB GPU — +# a KV pool far larger than any narration request needs) and leaves ~7 GB +# for the vocoder, CUDA graphs, transient allocations, and other GPU +# processes. Precedent: dots_tts.yaml pins this same knob; zonos2_bf16.yaml +# and higgs_audio_v3_tts.yaml fix the same 24 GB OOM class their way. config_cls: Qwen3TTSPipelineConfig model_path: Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice + +stages: + tts_engine: + engine: + mem_fraction_static: 0.70 diff --git a/app/backends/sglomni/configs/qwen3_tts_1_7b.yaml b/app/backends/sglomni/configs/qwen3_tts_1_7b.yaml index 4f7706d..b9cb5ec 100644 --- a/app/backends/sglomni/configs/qwen3_tts_1_7b.yaml +++ b/app/backends/sglomni/configs/qwen3_tts_1_7b.yaml @@ -1,2 +1,23 @@ +# Qwen3-TTS 1.7B Base with a pinned AR-engine memory budget. +# +# The upstream pipeline (Qwen3TTSPipelineConfig) colocates its three stages +# (preprocessing -> tts_engine -> vocoder) in one process on GPU 0 and leaves +# the engine's sglang mem_fraction_static unset: the static pool (weights + +# KV cache) is auto-sized to nearly all free VRAM at boot. On a 24 GB card +# that leaves only tens of MiB free once the engine's CUDA graphs (~1.5 GB) +# and the colocated vocoder are resident — the first /v1/audio/speech request +# aborts with "CUDA out of memory. Tried to allocate ~100 MiB" and every +# retry fails identically (the weights themselves are small: 1.7B bf16). +# +# 0.70 pins the static pool at ~70% of the card (~16.5 GB on a 24 GB GPU — +# a KV pool far larger than any narration request needs) and leaves ~7 GB +# for the vocoder, CUDA graphs, transient allocations, and other GPU +# processes. Precedent: dots_tts.yaml pins this same knob; zonos2_bf16.yaml +# and higgs_audio_v3_tts.yaml fix the same 24 GB OOM class their way. config_cls: Qwen3TTSPipelineConfig model_path: Qwen/Qwen3-TTS-12Hz-1.7B-Base + +stages: + tts_engine: + engine: + mem_fraction_static: 0.70 diff --git a/app/backends/sglomni/configs/qwen3_tts_1_7b_voicedesign.yaml b/app/backends/sglomni/configs/qwen3_tts_1_7b_voicedesign.yaml index 20ae0b8..ade6a24 100644 --- a/app/backends/sglomni/configs/qwen3_tts_1_7b_voicedesign.yaml +++ b/app/backends/sglomni/configs/qwen3_tts_1_7b_voicedesign.yaml @@ -1,2 +1,23 @@ +# Qwen3-TTS 1.7B VoiceDesign with a pinned AR-engine memory budget. +# +# The upstream pipeline (Qwen3TTSPipelineConfig) colocates its three stages +# (preprocessing -> tts_engine -> vocoder) in one process on GPU 0 and leaves +# the engine's sglang mem_fraction_static unset: the static pool (weights + +# KV cache) is auto-sized to nearly all free VRAM at boot. On a 24 GB card +# that leaves only tens of MiB free once the engine's CUDA graphs (~1.5 GB) +# and the colocated vocoder are resident — the first /v1/audio/speech request +# aborts with "CUDA out of memory. Tried to allocate ~100 MiB" and every +# retry fails identically (the weights themselves are small: 1.7B bf16). +# +# 0.70 pins the static pool at ~70% of the card (~16.5 GB on a 24 GB GPU — +# a KV pool far larger than any narration request needs) and leaves ~7 GB +# for the vocoder, CUDA graphs, transient allocations, and other GPU +# processes. Precedent: dots_tts.yaml pins this same knob; zonos2_bf16.yaml +# and higgs_audio_v3_tts.yaml fix the same 24 GB OOM class their way. config_cls: Qwen3TTSPipelineConfig model_path: Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign + +stages: + tts_engine: + engine: + mem_fraction_static: 0.70 diff --git a/app/backends/sglomni/configs/voxtral_tts.yaml b/app/backends/sglomni/configs/voxtral_tts.yaml index 450cbce..b53f83a 100644 --- a/app/backends/sglomni/configs/voxtral_tts.yaml +++ b/app/backends/sglomni/configs/voxtral_tts.yaml @@ -1,2 +1,25 @@ +# Voxtral TTS 4B with a pinned AR-engine memory budget. +# +# The upstream pipeline (VoxtralTTSPipelineConfig) colocates its three stages +# (preprocessing -> tts_generation -> vocoder) in one process on GPU 0 and +# leaves the engine's sglang mem_fraction_static unset: the static pool +# (weights + KV cache) is auto-sized to nearly all free VRAM at boot. On a +# 24 GB card that leaves only tens of MiB free once the engine's CUDA graphs +# and the colocated vocoder are resident — the first /v1/audio/speech request +# aborts with "CUDA out of memory. Tried to allocate ~100 MiB" and every +# retry fails identically (the same failure the qwen3_tts/higgs/zonos2 +# vendored configs fix). +# +# 0.70 pins the static pool at ~70% of the card (~16.5 GB on a 24 GB GPU — +# ~8 GB of weights leaves a KV pool far larger than any narration request +# needs) and leaves ~7 GB for the vocoder, CUDA graphs, transient +# allocations, and other GPU processes. Precedent: dots_tts.yaml pins this +# same knob. NOTE the stage name: Voxtral's engine stage is tts_generation, +# not tts_engine. config_cls: VoxtralTTSPipelineConfig model_path: mistralai/Voxtral-4B-TTS-2603 + +stages: + tts_generation: + engine: + mem_fraction_static: 0.70 diff --git a/app/backends/sglomni/wizard.py b/app/backends/sglomni/wizard.py index 2083ec8..96679d6 100644 --- a/app/backends/sglomni/wizard.py +++ b/app/backends/sglomni/wizard.py @@ -101,7 +101,7 @@ def _wizard(stdscr, args: argparse.Namespace) -> Optional[dict]: installed = set(sg_models.installed_keys()) picked = tui.checkbox_tree( stdscr, "Select SGLang-Omni Models to Install", families, - expand_all=True, back_value=_GO_BACK, + back_value=_GO_BACK, checked={(index, option["key"]) for index, family in enumerate(families) for option in family["options"] |