From 477ac3e827e3bdc9f14583fc3aa8db1fa2d27c52 Mon Sep 17 00:00:00 2001 From: historia Date: Wed, 26 Aug 2026 21:22:40 -0400 Subject: feat: design model support for qwen-tts backend. remove unnecessary port split for qwen models --- app/converter/clients/__init__.py | 4 +-- app/converter/clients/base.py | 10 +++++--- app/converter/clients/qwen.py | 54 ++++++++++++++++++++++++++++++--------- app/converter/config.py | 34 +++++++++++++++--------- app/converter/converter.py | 35 ++++++++++++++++++++----- 5 files changed, 100 insertions(+), 37 deletions(-) (limited to 'app/converter') diff --git a/app/converter/clients/__init__.py b/app/converter/clients/__init__.py index e216011..6ae8850 100644 --- a/app/converter/clients/__init__.py +++ b/app/converter/clients/__init__.py @@ -15,7 +15,7 @@ BACKEND_AUDIOCPP = "audiocpp" BACKENDS = (BACKEND_AUDIOCPP, BACKEND_QWEN, BACKEND_FASTER) from .base import BaseTTSClient, ConversionCancelled, VOICE_MODE_CLONE, \ - VOICE_MODE_CUSTOM, VOICE_MODES, resolve_request_seed + VOICE_MODE_CUSTOM, VOICE_MODE_DESIGN, VOICE_MODES, resolve_request_seed from .languages import LANGUAGE_ISO_CODES, TTS_LANGUAGES, \ TTS_LANGUAGE_ALIASES, normalize_language from .speakers import QWEN3_TTS_SPEAKERS, SPEAKER_DISPLAY_NAMES, \ @@ -44,7 +44,7 @@ from .audiocpp import ( __all__ = [ # vocabulary "BACKEND_QWEN", "BACKEND_FASTER", "BACKEND_AUDIOCPP", "BACKENDS", - "VOICE_MODE_CUSTOM", "VOICE_MODE_CLONE", "VOICE_MODES", + "VOICE_MODE_CUSTOM", "VOICE_MODE_CLONE", "VOICE_MODE_DESIGN", "VOICE_MODES", # clients "BaseTTSClient", "ConversionCancelled", "resolve_request_seed", "QwenTTSClient", "FasterTTSClient", "AudioCppTTSClient", diff --git a/app/converter/clients/base.py b/app/converter/clients/base.py index a0f28cf..d0d57fb 100644 --- a/app/converter/clients/base.py +++ b/app/converter/clients/base.py @@ -17,12 +17,14 @@ class ConversionCancelled(Exception): """Raised when the run's cancel event is set (between requests).""" -# How a run supplies its voice: a built-in CustomVoice speaker, or by -# cloning a reference audio clip (the faster and audiocpp backends always -# clone server-side; only the Qwen client branches on this at request time). +# How a run supplies its voice: a built-in CustomVoice speaker, by cloning +# a reference audio clip (the faster and audiocpp backends always clone +# server-side; only the Qwen client branches on this at request time), or +# designed from an instruction (Qwen's VoiceDesign model). VOICE_MODE_CUSTOM = "custom_voice" VOICE_MODE_CLONE = "voice_clone" -VOICE_MODES = (VOICE_MODE_CUSTOM, VOICE_MODE_CLONE) +VOICE_MODE_DESIGN = "voice_design" +VOICE_MODES = (VOICE_MODE_CUSTOM, VOICE_MODE_CLONE, VOICE_MODE_DESIGN) def resolve_request_seed() -> int: diff --git a/app/converter/clients/qwen.py b/app/converter/clients/qwen.py index 354ee04..ed3149b 100644 --- a/app/converter/clients/qwen.py +++ b/app/converter/clients/qwen.py @@ -1,4 +1,4 @@ -"""Client for the qwen-tts Gradio demo servers (CustomVoice + Base).""" +"""Client for the qwen-tts Gradio demo servers (CustomVoice / Base / VoiceDesign).""" import io import logging @@ -12,14 +12,15 @@ from .. import config from ..audio import concat_audio_files from ..chunking import split_into_chunks from .base import (BaseTTSClient, ConversionCancelled, resolve_request_seed, - VOICE_MODE_CLONE, VOICE_MODE_CUSTOM, VOICE_MODES) + VOICE_MODE_CLONE, VOICE_MODE_CUSTOM, VOICE_MODE_DESIGN, + VOICE_MODES) from .languages import normalize_language from .speakers import speaker_display_name logger = logging.getLogger(__name__) -# Fixed model facts: both demos run the 1.7B model (the CustomVoice demo -# takes its full HuggingFace id), and the 12Hz codec outputs 24 kHz audio. +# Fixed model facts: the demos run the 1.7B model (each takes its full +# HuggingFace id), and the 12Hz codec outputs 24 kHz audio. MODEL_SIZE = "1.7B" CUSTOM_VOICE_MODEL_ID = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice" @@ -31,7 +32,7 @@ class QwenTTSClient(BaseTTSClient): voice_mode: str = "custom_voice", voice_clone_ref_audio: Optional[str] = None, voice_clone_ref_text: Optional[str] = None, skip_transcription: bool = False, language: Optional[str] = None, api_url: Optional[str] = None, - quiet: bool = False): + instructions: Optional[str] = None, quiet: bool = False): super().__init__(chunks_dir, quiet=quiet) if voice_mode not in VOICE_MODES: raise ValueError( @@ -41,6 +42,11 @@ class QwenTTSClient(BaseTTSClient): self.voice_clone_ref_audio = voice_clone_ref_audio self.voice_clone_ref_text = (voice_clone_ref_text or "").strip() self.skip_transcription = skip_transcription + # Voice design / style instruction (VoiceDesign mode): describes the + # voice to design. Defaults to the configured CustomVoice INSTRUCT so + # a run never sends an empty design prompt. + self.instructions = (instructions if instructions is not None + else config.INSTRUCT).strip() # api_url overrides the configured endpoint for the active voice mode # (used by the hub's "[remote]" backend entries and --api-url). self.api_url = (api_url or "").strip() or None @@ -66,25 +72,28 @@ class QwenTTSClient(BaseTTSClient): # ------------------------------------------------------------------ def _connect(self) -> None: - api_url = self.api_url or ( - config.CLONE_API_URL if self.voice_mode == VOICE_MODE_CLONE - else config.QWEN_API_URL) + # One demo server runs at a time on the configured port, hosting + # whichever model this run selected (CustomVoice / Base / VoiceDesign). + api_url = self.api_url or config.QWEN_API_URL try: if self.voice_mode == VOICE_MODE_CLONE: - # Voice clone uses the Base-model demo, which is a separate server - # from the CustomVoice demo (that one only exposes /run_instruct). + # Voice clone talks to the Base-model demo. self._init_client(api_url, clone=True) self._report(f"[OK] Connected to Voice Clone API at {api_url}") self._resolve_reference_text() else: self._init_client(api_url, clone=False) - self._report("[OK] Connected to Qwen API") + if self.voice_mode == VOICE_MODE_DESIGN: + self._report(f"[OK] Connected to Voice Design API at {api_url}") + else: + self._report("[OK] Connected to Qwen API") except Exception as exc: raise RuntimeError( f"Qwen API initialization failed at {api_url}: {exc}. " "Make sure the Qwen demo server is running and reachable, and that your " "installed Qwen3-TTS version matches this converter's API expectations " - "(voice clone requires the Base-model demo: Qwen/Qwen3-TTS-12Hz-1.7B-Base)." + "(voice clone requires the Base-model demo: Qwen/Qwen3-TTS-12Hz-1.7B-Base; " + "voice design requires: Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign)." ) from exc def _resolve_reference_text(self) -> None: @@ -226,6 +235,8 @@ class QwenTTSClient(BaseTTSClient): result = self._generate_custom_voice(text) elif self.voice_mode == VOICE_MODE_CLONE: result = self._generate_voice_clone(text) + elif self.voice_mode == VOICE_MODE_DESIGN: + result = self._generate_voice_design(text) else: raise ValueError(f"Unknown voice mode: {self.voice_mode}") @@ -276,6 +287,25 @@ class QwenTTSClient(BaseTTSClient): return self.client.predict(**payload, api_name=custom_api) + def _generate_voice_design(self, text: str) -> Tuple: + """Generate audio using VoiceDesign mode (described-voice model).""" + design_api = self._resolve_api_name("/run_voice_design") + # The demo's field is named "design"; older builds may call it + # "instruct" instead. + design_field = ("instruct" + if self._endpoint_accepts_param(design_api, "instruct") + and not self._endpoint_accepts_param(design_api, "design") + else "design") + payload = dict( + text=text, + lang_disp=self.language, + **{design_field: self.instructions}, + ) + if self._endpoint_accepts_param(design_api, "seed"): + payload["seed"] = self._seed + + return self.client.predict(**payload, api_name=design_api) + def _ref_audio_payload(self) -> Dict[str, Any]: """Gradio file payload for the reference audio (built once, reused).""" if self._ref_audio_filedata is None: diff --git a/app/converter/config.py b/app/converter/config.py index ae7189f..4694e80 100644 --- a/app/converter/config.py +++ b/app/converter/config.py @@ -25,21 +25,31 @@ BACKEND = "audiocpp" # BACKEND 1: qwen-tts-demo (qwen) options # ############################################################################### -# There are different API URLs for CustomVoice and Base models so you can run both at once -QWEN_API_URL = "http://127.0.0.1:7860" # CustomVoice model -CLONE_API_URL = "http://127.0.0.1:7861" # Base model - -# Remote (externally-run) server URLs. The hub probes these and offers a -# "[remote]" backend entry when one answers with the expected backend, so an -# externally-started server can be used alongside a locally-managed one. -# Leave empty to disable remote probing for that backend. The defaults match -# the local ports so an external server squatting the local port is found -# without any configuration. -QWEN_REMOTE_URL = "http://127.0.0.1:7860" # CustomVoice model -CLONE_REMOTE_URL = "http://127.0.0.1:7861" # Base model +# The qwen backend runs ONE demo server at a time, on this port. Which model +# the server hosts is chosen per run on the Generate audiobooks screen and +# persisted below (see QWEN_MODEL); switching models restarts the server. +QWEN_API_URL = "http://127.0.0.1:7860" # single qwen-tts-demo server + +# Remote (externally-run) server URL. The hub probes it and offers a +# "[remote]" backend entry when it answers with a known qwen-tts demo (any +# of the three models), so an externally-started server can be used alongside +# a locally-managed one. Leave empty to disable remote probing. The default +# matches the local port so an external server squatting the local port is +# found without any configuration. +QWEN_REMOTE_URL = "http://127.0.0.1:7860" + +# Which model the managed demo server runs (one server hosts one model): +# CustomVoice - built-in speakers (see SPEAKER) +# Base - voice cloning from a reference .wav +# VoiceDesign - voice described by an instruction +# Chosen per run in the Generate-audiobooks form; edited here only as the +# default for the next run. +QWEN_MODEL = "CustomVoice" # Custom voice options SPEAKER = "Vivian" #Vivian, Serena, Uncle_Fu, Dylan, Eric, Ryan, Aiden, Ono_Anna, Sohee +# Style/delivery instruction for CustomVoice runs; also the default design +# instruction when a VoiceDesign run does not override it. INSTRUCT = "Speak naturally and clearly, as if reading a dramatic book to an adult audience." # Don't clone with transcription, only use x-vector-only cloning. Generally "worse" diff --git a/app/converter/converter.py b/app/converter/converter.py index 48d4987..b356448 100644 --- a/app/converter/converter.py +++ b/app/converter/converter.py @@ -24,6 +24,7 @@ from .clients import ( MODEL_SIZE, VOICE_MODE_CLONE, VOICE_MODE_CUSTOM, + VOICE_MODE_DESIGN, VOICE_MODES, AudioCppTTSClient, FasterTTSClient, @@ -100,19 +101,23 @@ def setup_directories() -> None: def voice_mode_for(backend: str, voice: Optional[str] = None, - clone: Optional[str] = None) -> str: + clone: Optional[str] = None, + instructions: Optional[str] = None) -> str: """The voice mode a run with these options would use. Mirrors the choice ``audiobook.convert`` makes from the same inputs (faster always clones; audiocpp clones through a server-side voice; - qwen clones only with a reference .wav), so the hub can run the - pre-flight overwrite checks against exactly the output names the - conversion will produce. + qwen designs with instructions, clones only with a reference .wav, and + uses a built-in speaker otherwise), so the hub can run the pre-flight + overwrite checks against exactly the output names the conversion will + produce. """ if backend == BACKEND_FASTER: return VOICE_MODE_CLONE if backend == BACKEND_AUDIOCPP: return VOICE_MODE_CLONE if voice else VOICE_MODE_CUSTOM + if (instructions or "").strip(): + return VOICE_MODE_DESIGN return VOICE_MODE_CLONE if clone else VOICE_MODE_CUSTOM @@ -245,6 +250,9 @@ class AudiobookConverter: request_options=self.request_options, api_url=api_url, quiet=quiet) else: + # Qwen: the voice mode picks the request shape (built-in + # speaker, clone from a reference .wav, or a designed voice); + # instructions describe the voice in design mode. self.tts = QwenTTSClient( chunks_dir=CHUNKS_FOLDER, voice_mode=voice_mode, @@ -252,6 +260,7 @@ class AudiobookConverter: voice_clone_ref_text=voice_clone_ref_text, skip_transcription=skip_transcription, language=self.language, + instructions=self.instructions, api_url=api_url, quiet=quiet, ) @@ -281,6 +290,12 @@ class AudiobookConverter: f"Unknown voice mode: {self.voice_mode!r} " f"(expected one of {VOICE_MODES})" ) + if self.backend == BACKEND_QWEN and self.voice_mode == VOICE_MODE_DESIGN \ + and not (self.instructions or "").strip(): + raise ValueError( + "Voice Design mode requires a voice description. " + "Use --instructions \"...\" to describe the voice to synthesize with." + ) if self.voice_mode == VOICE_MODE_CLONE and self.backend == BACKEND_QWEN: if not self.voice_clone_ref_audio: raise ValueError( @@ -336,6 +351,10 @@ class AudiobookConverter: narrator = "designed" else: narrator = speaker_display_name() + elif voice_mode == VOICE_MODE_DESIGN: + # Qwen's VoiceDesign model: the voice is described by an + # instruction and has no speaker name. + narrator = "designed" elif voice_mode == VOICE_MODE_CLONE: narrator = Path(voice_clone_ref_audio).stem else: @@ -722,9 +741,7 @@ class AudiobookConverter: else: tts_client = getattr(self, "tts", None) api_url = (getattr(tts_client, "api_url", None) - or (config.CLONE_API_URL - if self.voice_mode == VOICE_MODE_CLONE - else config.QWEN_API_URL)) + or config.QWEN_API_URL) self._say(f"Qwen API endpoint: {api_url}") self._say(f"Voice mode: {self.voice_mode}") self._say(f"Model size: {MODEL_SIZE} (always)") @@ -734,6 +751,10 @@ class AudiobookConverter: elif self.voice_mode == VOICE_MODE_CLONE: self._say(f"Reference audio: {Path(self.voice_clone_ref_audio).name}") self._say(f"Language: {self.language}") + elif self.voice_mode == VOICE_MODE_DESIGN: + self._say("Backend: qwen-tts (voice from --instructions description)") + self._say(f"Instruction: {self.instructions}") + self._say(f"Language: {self.language}") self._say(f"Output format: {self.output_format}") if self.single_file and self.output_format != "m4b": self._say("Chapter mode: single file (--single-file)") -- cgit v1.2.3