feat: design model support for qwen-tts backend. remove unnecessary port split for qwen models

5 files changed, 100 insertions, 37 deletions

diff --git a/app/converter/clients/__init__.py b/app/converter/clients/__init__.py
index e216011..6ae8850 100644
--- a/app/converter/clients/__init__.py
+++ b/app/converter/clients/__init__.py
@@ -15,7 +15,7 @@ BACKEND_AUDIOCPP = "audiocpp"
 BACKENDS = (BACKEND_AUDIOCPP, BACKEND_QWEN, BACKEND_FASTER)
 
 from .base import BaseTTSClient, ConversionCancelled, VOICE_MODE_CLONE, \
-    VOICE_MODE_CUSTOM, VOICE_MODES, resolve_request_seed
+    VOICE_MODE_CUSTOM, VOICE_MODE_DESIGN, VOICE_MODES, resolve_request_seed
 from .languages import LANGUAGE_ISO_CODES, TTS_LANGUAGES, \
     TTS_LANGUAGE_ALIASES, normalize_language
 from .speakers import QWEN3_TTS_SPEAKERS, SPEAKER_DISPLAY_NAMES, \
@@ -44,7 +44,7 @@ from .audiocpp import (
 __all__ = [
     # vocabulary
     "BACKEND_QWEN", "BACKEND_FASTER", "BACKEND_AUDIOCPP", "BACKENDS",
-    "VOICE_MODE_CUSTOM", "VOICE_MODE_CLONE", "VOICE_MODES",
+    "VOICE_MODE_CUSTOM", "VOICE_MODE_CLONE", "VOICE_MODE_DESIGN", "VOICE_MODES",
     # clients
     "BaseTTSClient", "ConversionCancelled", "resolve_request_seed",
     "QwenTTSClient", "FasterTTSClient", "AudioCppTTSClient",
diff --git a/app/converter/clients/base.py b/app/converter/clients/base.py
index a0f28cf..d0d57fb 100644
--- a/app/converter/clients/base.py
+++ b/app/converter/clients/base.py
@@ -17,12 +17,14 @@ class ConversionCancelled(Exception):
     """Raised when the run's cancel event is set (between requests)."""
 
 
-# How a run supplies its voice: a built-in CustomVoice speaker, or by
-# cloning a reference audio clip (the faster and audiocpp backends always
-# clone server-side; only the Qwen client branches on this at request time).
+# How a run supplies its voice: a built-in CustomVoice speaker, by cloning
+# a reference audio clip (the faster and audiocpp backends always clone
+# server-side; only the Qwen client branches on this at request time), or
+# designed from an instruction (Qwen's VoiceDesign model).
 VOICE_MODE_CUSTOM = "custom_voice"
 VOICE_MODE_CLONE = "voice_clone"
-VOICE_MODES = (VOICE_MODE_CUSTOM, VOICE_MODE_CLONE)
+VOICE_MODE_DESIGN = "voice_design"
+VOICE_MODES = (VOICE_MODE_CUSTOM, VOICE_MODE_CLONE, VOICE_MODE_DESIGN)
 
 
 def resolve_request_seed() -> int:
diff --git a/app/converter/clients/qwen.py b/app/converter/clients/qwen.py
index 354ee04..ed3149b 100644
--- a/app/converter/clients/qwen.py
+++ b/app/converter/clients/qwen.py
@@ -1,4 +1,4 @@
-"""Client for the qwen-tts Gradio demo servers (CustomVoice + Base)."""
+"""Client for the qwen-tts Gradio demo servers (CustomVoice / Base / VoiceDesign)."""
 
 import io
 import logging
@@ -12,14 +12,15 @@ from .. import config
 from ..audio import concat_audio_files
 from ..chunking import split_into_chunks
 from .base import (BaseTTSClient, ConversionCancelled, resolve_request_seed,
-                   VOICE_MODE_CLONE, VOICE_MODE_CUSTOM, VOICE_MODES)
+                   VOICE_MODE_CLONE, VOICE_MODE_CUSTOM, VOICE_MODE_DESIGN,
+                   VOICE_MODES)
 from .languages import normalize_language
 from .speakers import speaker_display_name
 
 logger = logging.getLogger(__name__)
 
-# Fixed model facts: both demos run the 1.7B model (the CustomVoice demo
-# takes its full HuggingFace id), and the 12Hz codec outputs 24 kHz audio.
+# Fixed model facts: the demos run the 1.7B model (each takes its full
+# HuggingFace id), and the 12Hz codec outputs 24 kHz audio.
 MODEL_SIZE = "1.7B"
 CUSTOM_VOICE_MODEL_ID = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
 
@@ -31,7 +32,7 @@ class QwenTTSClient(BaseTTSClient):
                  voice_mode: str = "custom_voice", voice_clone_ref_audio: Optional[str] = None,
                  voice_clone_ref_text: Optional[str] = None, skip_transcription: bool = False,
                  language: Optional[str] = None, api_url: Optional[str] = None,
-                 quiet: bool = False):
+                 instructions: Optional[str] = None, quiet: bool = False):
         super().__init__(chunks_dir, quiet=quiet)
         if voice_mode not in VOICE_MODES:
             raise ValueError(
@@ -41,6 +42,11 @@ class QwenTTSClient(BaseTTSClient):
         self.voice_clone_ref_audio = voice_clone_ref_audio
         self.voice_clone_ref_text = (voice_clone_ref_text or "").strip()
         self.skip_transcription = skip_transcription
+        # Voice design / style instruction (VoiceDesign mode): describes the
+        # voice to design. Defaults to the configured CustomVoice INSTRUCT so
+        # a run never sends an empty design prompt.
+        self.instructions = (instructions if instructions is not None
+                             else config.INSTRUCT).strip()
         # api_url overrides the configured endpoint for the active voice mode
         # (used by the hub's "[remote]" backend entries and --api-url).
         self.api_url = (api_url or "").strip() or None
@@ -66,25 +72,28 @@ class QwenTTSClient(BaseTTSClient):
     # ------------------------------------------------------------------
 
     def _connect(self) -> None:
-        api_url = self.api_url or (
-            config.CLONE_API_URL if self.voice_mode == VOICE_MODE_CLONE
-            else config.QWEN_API_URL)
+        # One demo server runs at a time on the configured port, hosting
+        # whichever model this run selected (CustomVoice / Base / VoiceDesign).
+        api_url = self.api_url or config.QWEN_API_URL
         try:
             if self.voice_mode == VOICE_MODE_CLONE:
-                # Voice clone uses the Base-model demo, which is a separate server
-                # from the CustomVoice demo (that one only exposes /run_instruct).
+                # Voice clone talks to the Base-model demo.
                 self._init_client(api_url, clone=True)
                 self._report(f"[OK] Connected to Voice Clone API at {api_url}")
                 self._resolve_reference_text()
             else:
                 self._init_client(api_url, clone=False)
-                self._report("[OK] Connected to Qwen API")
+                if self.voice_mode == VOICE_MODE_DESIGN:
+                    self._report(f"[OK] Connected to Voice Design API at {api_url}")
+                else:
+                    self._report("[OK] Connected to Qwen API")
         except Exception as exc:
             raise RuntimeError(
                 f"Qwen API initialization failed at {api_url}: {exc}. "
                 "Make sure the Qwen demo server is running and reachable, and that your "
                 "installed Qwen3-TTS version matches this converter's API expectations "
-                "(voice clone requires the Base-model demo: Qwen/Qwen3-TTS-12Hz-1.7B-Base)."
+                "(voice clone requires the Base-model demo: Qwen/Qwen3-TTS-12Hz-1.7B-Base; "
+                "voice design requires: Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign)."
             ) from exc
 
     def _resolve_reference_text(self) -> None:
@@ -226,6 +235,8 @@ class QwenTTSClient(BaseTTSClient):
             result = self._generate_custom_voice(text)
         elif self.voice_mode == VOICE_MODE_CLONE:
             result = self._generate_voice_clone(text)
+        elif self.voice_mode == VOICE_MODE_DESIGN:
+            result = self._generate_voice_design(text)
         else:
             raise ValueError(f"Unknown voice mode: {self.voice_mode}")
 
@@ -276,6 +287,25 @@ class QwenTTSClient(BaseTTSClient):
 
         return self.client.predict(**payload, api_name=custom_api)
 
+    def _generate_voice_design(self, text: str) -> Tuple:
+        """Generate audio using VoiceDesign mode (described-voice model)."""
+        design_api = self._resolve_api_name("/run_voice_design")
+        # The demo's field is named "design"; older builds may call it
+        # "instruct" instead.
+        design_field = ("instruct"
+                        if self._endpoint_accepts_param(design_api, "instruct")
+                        and not self._endpoint_accepts_param(design_api, "design")
+                        else "design")
+        payload = dict(
+            text=text,
+            lang_disp=self.language,
+            **{design_field: self.instructions},
+        )
+        if self._endpoint_accepts_param(design_api, "seed"):
+            payload["seed"] = self._seed
+
+        return self.client.predict(**payload, api_name=design_api)
+
     def _ref_audio_payload(self) -> Dict[str, Any]:
         """Gradio file payload for the reference audio (built once, reused)."""
         if self._ref_audio_filedata is None:
diff --git a/app/converter/config.py b/app/converter/config.py
index ae7189f..4694e80 100644
--- a/app/converter/config.py
+++ b/app/converter/config.py
@@ -25,21 +25,31 @@ BACKEND = "audiocpp"
 # BACKEND 1: qwen-tts-demo (qwen) options                                       #
 ###############################################################################
 
-# There are different API URLs for CustomVoice and Base models so you can run both at once
-QWEN_API_URL     = "http://127.0.0.1:7860"  # CustomVoice model
-CLONE_API_URL    = "http://127.0.0.1:7861"  # Base model
-
-# Remote (externally-run) server URLs. The hub probes these and offers a
-# "[remote]" backend entry when one answers with the expected backend, so an
-# externally-started server can be used alongside a locally-managed one.
-# Leave empty to disable remote probing for that backend. The defaults match
-# the local ports so an external server squatting the local port is found
-# without any configuration.
-QWEN_REMOTE_URL  = "http://127.0.0.1:7860"  # CustomVoice model
-CLONE_REMOTE_URL = "http://127.0.0.1:7861"  # Base model
+# The qwen backend runs ONE demo server at a time, on this port. Which model
+# the server hosts is chosen per run on the Generate audiobooks screen and
+# persisted below (see QWEN_MODEL); switching models restarts the server.
+QWEN_API_URL     = "http://127.0.0.1:7860"  # single qwen-tts-demo server
+
+# Remote (externally-run) server URL. The hub probes it and offers a
+# "[remote]" backend entry when it answers with a known qwen-tts demo (any
+# of the three models), so an externally-started server can be used alongside
+# a locally-managed one. Leave empty to disable remote probing. The default
+# matches the local port so an external server squatting the local port is
+# found without any configuration.
+QWEN_REMOTE_URL  = "http://127.0.0.1:7860"
+
+# Which model the managed demo server runs (one server hosts one model):
+#   CustomVoice - built-in speakers (see SPEAKER)
+#   Base        - voice cloning from a reference .wav
+#   VoiceDesign - voice described by an instruction
+# Chosen per run in the Generate-audiobooks form; edited here only as the
+# default for the next run.
+QWEN_MODEL = "CustomVoice"
 
 # Custom voice options
 SPEAKER = "Vivian"  #Vivian, Serena, Uncle_Fu, Dylan, Eric, Ryan, Aiden, Ono_Anna, Sohee
+# Style/delivery instruction for CustomVoice runs; also the default design
+# instruction when a VoiceDesign run does not override it.
 INSTRUCT = "Speak naturally and clearly, as if reading a dramatic book to an adult audience."
 
 # Don't clone with transcription, only use x-vector-only cloning. Generally "worse"
diff --git a/app/converter/converter.py b/app/converter/converter.py
index 48d4987..b356448 100644
--- a/app/converter/converter.py
+++ b/app/converter/converter.py
@@ -24,6 +24,7 @@ from .clients import (
     MODEL_SIZE,
     VOICE_MODE_CLONE,
     VOICE_MODE_CUSTOM,
+    VOICE_MODE_DESIGN,
     VOICE_MODES,
     AudioCppTTSClient,
     FasterTTSClient,
@@ -100,19 +101,23 @@ def setup_directories() -> None:
 
 
 def voice_mode_for(backend: str, voice: Optional[str] = None,
-                   clone: Optional[str] = None) -> str:
+                   clone: Optional[str] = None,
+                   instructions: Optional[str] = None) -> str:
     """The voice mode a run with these options would use.
 
     Mirrors the choice ``audiobook.convert`` makes from the same inputs
     (faster always clones; audiocpp clones through a server-side voice;
-    qwen clones only with a reference .wav), so the hub can run the
-    pre-flight overwrite checks against exactly the output names the
-    conversion will produce.
+    qwen designs with instructions, clones only with a reference .wav, and
+    uses a built-in speaker otherwise), so the hub can run the pre-flight
+    overwrite checks against exactly the output names the conversion will
+    produce.
     """
     if backend == BACKEND_FASTER:
         return VOICE_MODE_CLONE
     if backend == BACKEND_AUDIOCPP:
         return VOICE_MODE_CLONE if voice else VOICE_MODE_CUSTOM
+    if (instructions or "").strip():
+        return VOICE_MODE_DESIGN
     return VOICE_MODE_CLONE if clone else VOICE_MODE_CUSTOM
 
 
@@ -245,6 +250,9 @@ class AudiobookConverter:
                                          request_options=self.request_options,
                                          api_url=api_url, quiet=quiet)
         else:
+            # Qwen: the voice mode picks the request shape (built-in
+            # speaker, clone from a reference .wav, or a designed voice);
+            # instructions describe the voice in design mode.
             self.tts = QwenTTSClient(
                 chunks_dir=CHUNKS_FOLDER,
                 voice_mode=voice_mode,
@@ -252,6 +260,7 @@ class AudiobookConverter:
                 voice_clone_ref_text=voice_clone_ref_text,
                 skip_transcription=skip_transcription,
                 language=self.language,
+                instructions=self.instructions,
                 api_url=api_url,
                 quiet=quiet,
             )
@@ -281,6 +290,12 @@ class AudiobookConverter:
                 f"Unknown voice mode: {self.voice_mode!r} "
                 f"(expected one of {VOICE_MODES})"
             )
+        if self.backend == BACKEND_QWEN and self.voice_mode == VOICE_MODE_DESIGN \
+                and not (self.instructions or "").strip():
+            raise ValueError(
+                "Voice Design mode requires a voice description. "
+                "Use --instructions \"...\" to describe the voice to synthesize with."
+            )
         if self.voice_mode == VOICE_MODE_CLONE and self.backend == BACKEND_QWEN:
             if not self.voice_clone_ref_audio:
                 raise ValueError(
@@ -336,6 +351,10 @@ class AudiobookConverter:
                 narrator = "designed"
             else:
                 narrator = speaker_display_name()
+        elif voice_mode == VOICE_MODE_DESIGN:
+            # Qwen's VoiceDesign model: the voice is described by an
+            # instruction and has no speaker name.
+            narrator = "designed"
         elif voice_mode == VOICE_MODE_CLONE:
             narrator = Path(voice_clone_ref_audio).stem
         else:
@@ -722,9 +741,7 @@ class AudiobookConverter:
         else:
             tts_client = getattr(self, "tts", None)
             api_url = (getattr(tts_client, "api_url", None)
-                       or (config.CLONE_API_URL
-                           if self.voice_mode == VOICE_MODE_CLONE
-                           else config.QWEN_API_URL))
+                       or config.QWEN_API_URL)
             self._say(f"Qwen API endpoint: {api_url}")
             self._say(f"Voice mode: {self.voice_mode}")
             self._say(f"Model size: {MODEL_SIZE} (always)")
@@ -734,6 +751,10 @@ class AudiobookConverter:
             elif self.voice_mode == VOICE_MODE_CLONE:
                 self._say(f"Reference audio: {Path(self.voice_clone_ref_audio).name}")
                 self._say(f"Language: {self.language}")
+            elif self.voice_mode == VOICE_MODE_DESIGN:
+                self._say("Backend: qwen-tts (voice from --instructions description)")
+                self._say(f"Instruction: {self.instructions}")
+                self._say(f"Language: {self.language}")
         self._say(f"Output format: {self.output_format}")
         if self.single_file and self.output_format != "m4b":
             self._say("Chapter mode: single file (--single-file)")