diff options
Diffstat (limited to 'converter')
| -rw-r--r-- | converter/config.py | 7 | ||||
| -rw-r--r-- | converter/converter.py | 54 | ||||
| -rw-r--r-- | converter/tts.py | 150 |
3 files changed, 181 insertions, 30 deletions
diff --git a/converter/config.py b/converter/config.py index d15efa5..4686764 100644 --- a/converter/config.py +++ b/converter/config.py @@ -68,3 +68,10 @@ AUDIOCPP_API_URL = "http://127.0.0.1:8080" # audio.cpp audiocpp_server # run with the --model CLI flag. AUDIOCPP_MODEL_ID = "qwen" AUDIOCPP_CLONE_MODEL_ID = "qwen-clone" + +# Voice design / style instruction sent with every audio.cpp request when +# the --instructions CLI flag is not given. Required for server entries +# hosted with task "vdes" (voice design models such as Qwen3-TTS +# VoiceDesign); on other families it acts as a style/delivery instruction +# when the model supports one and is ignored otherwise. Empty by default. +AUDIOCPP_INSTRUCTIONS = "" diff --git a/converter/converter.py b/converter/converter.py index 3915fe4..b24ac09 100644 --- a/converter/converter.py +++ b/converter/converter.py @@ -138,7 +138,9 @@ class AudiobookConverter: speed: float = 1.0, single_file: bool = False, output_format: str = config.AUDIO_FORMAT, language: Optional[str] = None, backend: str = config.BACKEND, voice: Optional[str] = None, debug: bool = False, - chunk: bool = False, model_id: Optional[str] = None): + chunk: bool = False, model_id: Optional[str] = None, + instructions: Optional[str] = None, + request_options: Optional[Dict[str, str]] = None): if speed <= 0: raise ValueError(f"Speed must be a positive number, got {speed}") if output_format not in AUDIO_FORMATS: @@ -164,6 +166,11 @@ class AudiobookConverter: # defaults to one request per chapter; --chunk forces client-side # chunking on top (possible needless double-chunking). self.client_chunks = bool(chunk) or backend != BACKEND_AUDIOCPP + # Voice design / style instruction and free-form request options + # (audio.cpp only): forwarded to AudioCppTTSClient, which validates + # them against the server-hosted model at connect time. + self.instructions = instructions + self.request_options = dict(request_options or {}) self._validate_configuration() if backend == BACKEND_FASTER: # The faster backend always voice-clones using a reference voice @@ -172,10 +179,14 @@ class AudiobookConverter: elif backend == BACKEND_AUDIOCPP: # Speaker mode (no voice) uses a built-in CustomVoice speaker; # an explicit voice selects a server-side preset (cloning). - # model_id overrides AUDIOCPP_MODEL_ID for multi-model servers. + # model_id overrides AUDIOCPP_MODEL_ID for multi-model servers; + # instructions describe or style the voice, request_options pass + # per-model controls through to the server. self.tts = AudioCppTTSClient(voice=voice, language=self.language, chunk_text=self.client_chunks, - model_id=model_id) + model_id=model_id, + instructions=instructions, + request_options=self.request_options) else: self.tts = QwenTTSClient( voice_mode=voice_mode, @@ -214,18 +225,22 @@ class AudiobookConverter: def _narrator_tag(self) -> str: """Narrator name used in output file names (see compute_narrator_tag).""" return self.compute_narrator_tag( - self.backend, self.voice, self.voice_mode, self.voice_clone_ref_audio) + self.backend, self.voice, self.voice_mode, + self.voice_clone_ref_audio, self.instructions) @staticmethod def compute_narrator_tag(backend: str, voice: Optional[str], voice_mode: str, - voice_clone_ref_audio: Optional[str]) -> str: + voice_clone_ref_audio: Optional[str], + instructions: Optional[str] = None) -> str: """Narrator name used in output file names, without a server connection. Custom voice mode uses the built-in speaker's display name; voice clone mode uses the reference audio file's stem; the faster and audiocpp backends use the server-side voice name (falling back to - the built-in speaker for the audiocpp backend's speaker mode). + the built-in speaker for the audiocpp backend's speaker mode). An + instruction without a voice (voice design, or instruction-defined + voices on families without built-in speakers) uses "designed". Spaces become underscores (e.g. "Uncle Fu" -> "Uncle_Fu"). Pure (no I/O, no server) so the pre-flight overwrite check can @@ -235,7 +250,13 @@ class AudiobookConverter: if backend == BACKEND_FASTER: narrator = voice or config.FASTER_VOICE elif backend == BACKEND_AUDIOCPP: - narrator = voice or speaker_display_name() + if voice: + narrator = voice + elif instructions: + # The voice comes from the instruction, not a speaker name. + narrator = "designed" + else: + narrator = speaker_display_name() elif voice_mode == VOICE_MODE_CLONE: narrator = Path(voice_clone_ref_audio).stem else: @@ -590,9 +611,14 @@ class AudiobookConverter: if self.voice: print("Backend: audio.cpp (voice cloning, reference configured on server)") print(f"Voice: {self.voice}") + elif self.instructions: + print("Backend: audio.cpp (voice from --instructions description)") + print(f"Instruction: {self.instructions}") else: print("Backend: audio.cpp (custom voice, built-in speaker)") print(f"Speaker: {config.SPEAKER}") + if self.request_options: + print(f"Request options: {self.request_options}") if self.client_chunks: print("Chunking: client-side (--chunk; the server also chunks " "long text itself, so this may double-chunk)") @@ -629,8 +655,9 @@ class AudiobookConverter: def preflight_overwrites(backend: str, voice: Optional[str], voice_mode: str, voice_clone_ref_audio: Optional[str], - output_format: str) -> Tuple[List[Path], - List[Tuple[Path, str]]]: + output_format: str, + instructions: Optional[str] = None + ) -> Tuple[List[Path], List[Tuple[Path, str]]]: """Discover books and ask every overwrite question up front. Pure of the TTS server: it scans the books folder, computes the @@ -639,8 +666,8 @@ class AudiobookConverter: existing output files. Returns ``(book_files, planned)`` where ``planned`` is the subset the user agreed to (re)convert. - Asking before connecting means a user who declines a prompt (or - has nothing to convert) never waits on a slow server handshake. + Asking before connecting means a user who declines a prompt (or has + nothing to convert) never waits on a slow server handshake. """ book_files = sorted( f for f in BOOKS_FOLDER.iterdir() @@ -658,7 +685,7 @@ class AudiobookConverter: # starts, so the rest of the run is unattended. planned: List[Tuple[Path, str]] = [] narrator_tag = AudiobookConverter.compute_narrator_tag( - backend, voice, voice_mode, voice_clone_ref_audio) + backend, voice, voice_mode, voice_clone_ref_audio, instructions) for book_file in book_files: output_name = book_file.stem if stem_counts[book_file.stem] > 1: @@ -689,7 +716,8 @@ class AudiobookConverter: else: book_files, planned = AudiobookConverter.preflight_overwrites( self.backend, self.voice, self.voice_mode, - self.voice_clone_ref_audio, self.output_format) + self.voice_clone_ref_audio, self.output_format, + self.instructions) if not book_files: print(f"[INFO] No supported files found in {BOOKS_FOLDER}") diff --git a/converter/tts.py b/converter/tts.py index 9b54cf4..f5d54e2 100644 --- a/converter/tts.py +++ b/converter/tts.py @@ -115,6 +115,17 @@ AUDIOCPP_LANG_OMIT = "omit" # no language field; the model detects it # voice comes from a server-side preset requested with --voice. AUDIOCPP_FAMILY_QWEN3_TTS = "qwen3_tts" +# Server model entry tasks this client can synthesize audiobooks with, +# taken from GET /v1/models (the "task" field of each entry; servers that +# predate the field reported TTS models only, so a missing task is treated +# as "tts"). "vdes" entries are voice design models: the voice is described +# with --instructions instead of coming from a speaker or a reference clip. +# Entries with any other task (asr, vc, diar, ...) are rejected at connect +# time with a hint to pick a synthesis entry. +AUDIOCPP_TASK_TTS = "tts" +AUDIOCPP_TASK_VDES = "vdes" +AUDIOCPP_SYNTHESIS_TASKS = (AUDIOCPP_TASK_TTS, "clon", AUDIOCPP_TASK_VDES) + class AudioCppFamilyProfile: """Request conventions of one audio.cpp model family.""" @@ -748,17 +759,17 @@ class AudioCppTTSClient(_BaseTTSClient): Talks to the OpenAI-style HTTP API of audiocpp_server, which hosts TTS model families through a native ggml runtime (GGUF weights, no Python - serving stack). The server API is family-agnostic; the family of the - configured model entry is read from GET /v1/models at startup and - adapts the request payload (language field style, style instructions) - through AUDIOCPP_FAMILY_PROFILES. Two voice modes, both resolved - server-side from the request's "voice" field: + serving stack). The server API is family-agnostic; the family and task + of the configured model entry are read from GET /v1/models at startup + and adapt the request payload (language field style, style instructions) + through AUDIOCPP_FAMILY_PROFILES. Three voice modes, all resolved + server-side from the request's "voice"/"instructions" fields: - Speaker mode (no ``voice``): Qwen3-TTS only. A built-in CustomVoice speaker name (e.g. "Vivian") is passed through, plus the INSTRUCT style prompt. The server must be configured with the CustomVoice model for this. Families without built-in speakers reject this mode - with a hint to pass --voice. + with a hint to pass --voice (or --instructions, see below). - Preset mode (``voice=NAME``): a voice configured on the server (``voice_presets`` or ``voice_dir`` in its config, e.g. a cloning reference). The name is validated against GET /v1/audio/voices at @@ -769,6 +780,21 @@ class AudioCppTTSClient(_BaseTTSClient): to it. Only the entry actually used needs to exist on the server: a clone-only (Base) server works for --voice runs, while speaker mode on such a server fails with a hint to pass --voice. + - Voice design (task "vdes" entries, e.g. Qwen3-TTS VoiceDesign): the + voice is described in natural language through ``instructions``, + which is required and sent with every request (no ``voice`` field). + A constant per-run seed keeps the designed voice consistent across + chunk boundaries. + + ``instructions`` also works on non-design entries, where it acts as a + generic style/delivery instruction (voice control): families that read + it (OmniVoice, Qwen3-TTS CustomVoice, ...) shape the voice or delivery + accordingly, and others ignore it. On instruction-conditioned families + without built-in speakers it may replace --voice entirely (the + instruction defines the voice). Extra request options (``--option + KEY=VALUE``, e.g. emotion, voice_id, speed) are forwarded verbatim in + the request's "options" object, which is the server's generic + pass-through for per-model controls. Chunking: the server does its own long-form text chunking for every family (its ``text_chunk_size`` option, with a per-family default), so @@ -784,7 +810,9 @@ class AudioCppTTSClient(_BaseTTSClient): def __init__(self, voice: Optional[str] = None, language: Optional[str] = None, api_url: Optional[str] = None, chunk_text: bool = False, - model_id: Optional[str] = None): + model_id: Optional[str] = None, + instructions: Optional[str] = None, + request_options: Optional[Dict[str, str]] = None): self.api_url = (api_url or config.AUDIOCPP_API_URL).rstrip("/") # Per-run model selection: the --model CLI flag overrides config; an # empty value is resolved at connect time when the server hosts exactly @@ -802,13 +830,28 @@ class AudioCppTTSClient(_BaseTTSClient): self._seed = _resolve_request_seed() self.preset_mode = bool(voice) self.voice = voice or speaker_display_name() + # Style/voice-design instruction sent with every request (the CLI + # --instructions flag overrides AUDIOCPP_INSTRUCTIONS in config.py). + # For task "vdes" entries it describes the voice to design; for other + # families it is a generic style instruction when the model reads one. + self.instructions = (instructions if instructions is not None + else config.AUDIOCPP_INSTRUCTIONS or "").strip() + # Free-form per-request options (--option KEY=VALUE) forwarded in the + # request's "options" object; models ignore keys they don't know. + self.request_options: Dict[str, str] = dict(request_options or {}) + # Both set during _connect once the entry's task is known: design_mode + # for "vdes" entries, instruction_voice when a family without built-in + # speakers gets its voice from the instruction alone (no voice field). + self.design_mode = False + self.instruction_voice = False # When False (default), each chapter is sent as one request and the # server does its own long-form chunking (text_chunk_size); when True, # text is split client-side into CHUNK_SIZE-word sub-requests first. self.chunk_text = bool(chunk_text) - # Family of the selected model entry and its request profile; both - # are resolved from GET /v1/models during _connect. + # Family and task of the selected model entry and the family's request + # profile; all are resolved from GET /v1/models during _connect. self.family = "" + self.task = AUDIOCPP_TASK_TTS self.profile = AUDIOCPP_DEFAULT_FAMILY_PROFILE self._connect() @@ -817,12 +860,14 @@ class AudioCppTTSClient(_BaseTTSClient): # ------------------------------------------------------------------ def _connect(self) -> None: - """Health-check the server and resolve the model, family, and voice. + """Health-check the server and resolve the model, family, task, and voice. Speaker mode is only offered to families with built-in speakers (Qwen3-TTS); every other family must select a server-side voice - with --voice, so it fails fast with a hint instead of silently - synthesizing with a random default voice. + with --voice or describe one with --instructions, so it fails fast + with a hint instead of silently synthesizing with a random default + voice. Voice design entries (task "vdes") require --instructions + and reject --voice. """ self._check_health() models = self._list_models() @@ -831,7 +876,33 @@ class AudioCppTTSClient(_BaseTTSClient): self._select_model(models) self._require_model_id(models) self._resolve_family(models) - if self.preset_mode: + self._resolve_task(models) + if self.task not in AUDIOCPP_SYNTHESIS_TASKS: + available = ", ".join(model["id"] for model in models) or "none" + raise RuntimeError( + f"The audio.cpp model '{self.model_id}' has task " + f"'{self.task}'; audiobook.py can only synthesize with TTS " + f"model entries (tasks {', '.join(AUDIOCPP_SYNTHESIS_TASKS)}). " + f"Pick a synthesis entry with --model (available: {available})." + ) + if self.design_mode: + if self.preset_mode: + raise RuntimeError( + f"--voice cannot be used with the voice design model " + f"'{self.model_id}': the voice is described by the " + "--instructions text instead (see README).") + if not self.instructions: + raise RuntimeError( + f"The audio.cpp model '{self.model_id}' (family " + f"'{self.family}') is a voice design model: pass a " + "description of the voice to synthesize with, e.g. " + '--instructions "A warm adult female narrator with a ' + 'British accent" (see README).') + print(f"[OK] Connected to audio.cpp server at {self.api_url} " + f"(model '{self.model_id}', family '{self.family}', " + "voice design)") + print(f"[INFO] Designing the voice from: {self.instructions}") + elif self.preset_mode: self._check_voice() print(f"[OK] Connected to audio.cpp server at {self.api_url} " f"(model '{self.model_id}', family '{self.family}', " @@ -843,13 +914,26 @@ class AudioCppTTSClient(_BaseTTSClient): print("[INFO] Speaker mode expects the server to be configured with the " "CustomVoice model; with the Base model the speaker name is ignored " "and a random default voice is used (see README).") + elif self.instructions: + # Families without built-in speakers can still get their voice + # from the instruction alone (e.g. OmniVoice voice design). + self.instruction_voice = True + print(f"[OK] Connected to audio.cpp server at {self.api_url} " + f"(model '{self.model_id}', family '{self.family}', " + "instruction voice)") + print(f"[INFO] Designing the voice from: {self.instructions}") else: raise RuntimeError( f"The audio.cpp model '{self.model_id}' (family " f"'{self.family}') has no built-in speakers, so its voice " "must come from the server: rerun with --voice NAME " "matching a voice_preset or voice_dir entry in the server " - "config (see README).") + "config, or describe a voice with --instructions for " + "families that support it (see README).") + if self.instructions and not self.design_mode and not self.instruction_voice: + print(f"[INFO] Sending instruction with every request: {self.instructions}") + print("[INFO] Its effect (style, emotion, delivery) depends on the " + "model family; models without instruction support ignore it.") def _get_json(self, path: str, timeout: int = 10) -> Dict[str, Any]: """GET a JSON document from the server.""" @@ -884,7 +968,7 @@ class AudioCppTTSClient(_BaseTTSClient): f"{payload.get('status')!r} instead of 'ok'") def _list_models(self) -> List[Dict[str, str]]: - """Fetch the (id, family) pairs reported by the server.""" + """Fetch the (id, family, task) triples reported by the server.""" try: payload = self._get_json("/v1/models") except Exception as exc: @@ -898,6 +982,7 @@ class AudioCppTTSClient(_BaseTTSClient): models.append({ "id": entry["id"], "family": entry.get("family") or "", + "task": entry.get("task") or "", }) return models @@ -1034,6 +1119,25 @@ class AudioCppTTSClient(_BaseTTSClient): "generic profile (voice cloning via --voice, model-detected " "language)", family) + def _resolve_task(self, models: List[Dict[str, str]]) -> None: + """Resolve the selected model's task (tts, clon, vdes, ...) and set + design mode for voice design entries. + + The task comes from GET /v1/models and is fixed per server entry by + its server.json config (a VoiceDesign model must be hosted with + "task": "vdes"). Servers that predate the task field hosted plain + TTS models, so a missing task is treated as tts. + """ + entry = next( + (model for model in models if model["id"] == self.model_id), None) + task = (entry["task"] if entry is not None else "") or "" + if not task: + task = AUDIOCPP_TASK_TTS + logger.debug("Model '%s' reported no task; assuming tts", + self.model_id) + self.task = task + self.design_mode = task == AUDIOCPP_TASK_VDES + def _check_voice(self) -> None: """Verify the requested voice is available on the server. @@ -1076,8 +1180,12 @@ class AudioCppTTSClient(_BaseTTSClient): payload: Dict[str, Any] = { "model": self.model_id, "input": text, - "voice": self.voice, } + # Design models take no voice field (the voice comes from the + # instruction); instruction-voice runs on families without built-in + # speakers omit it too, since no speaker or preset was requested. + if not self.design_mode and not self.instruction_voice: + payload["voice"] = self.voice if self.profile.language_style == AUDIOCPP_LANG_DISPLAY: payload["language"] = self.language elif self.profile.language_style == AUDIOCPP_LANG_ISO: @@ -1092,11 +1200,19 @@ class AudioCppTTSClient(_BaseTTSClient): # audio.cpp has no negative "randomize" seed; a negative seed # means "let the server randomize", so the field is omitted. payload["seed"] = self._seed - if not self.preset_mode and config.INSTRUCT \ + if self.instructions: + # Explicit voice-design or style instruction (required for task + # "vdes" entries; a Ctrl/style control on families that read it). + payload["instructions"] = self.instructions + elif not self.preset_mode and config.INSTRUCT \ and self.profile.sends_instructions: # Style instruction for the Qwen3-TTS CustomVoice speakers; # ignored by the Base (cloning) model and other families. payload["instructions"] = config.INSTRUCT + if self.request_options: + # Generic per-model controls (--option KEY=VALUE): forwarded + # verbatim; the model ignores keys it does not know. + payload["options"] = dict(self.request_options) request = urllib.request.Request( url, data=json.dumps(payload).encode("utf-8"), headers={"Content-Type": "application/json"}, method="POST") |