From 4d3530f63730b47870d25629802c0c41f0c9ffae Mon Sep 17 00:00:00 2001 From: historia Date: Thu, 20 Aug 2026 16:17:57 -0400 Subject: feat: audio.cpp backend support --- README.md | 195 ++++++++++++++-------- audiobook.py | 90 +++++++--- converter/config.py | 86 +++++----- converter/converter.py | 60 +++++-- converter/tts.py | 281 ++++++++++++++++++++++++++++++- tests/test_converter.py | 14 +- tests/test_tts.py | 431 ++++++++++++++++++++++++++++++++++++++++++++++-- 7 files changed, 987 insertions(+), 170 deletions(-) diff --git a/README.md b/README.md index ef44cce..dd3aece 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Convert TXT, PDF, and EPUB files into audiobooks using the Qwen3-TTS voice model. -This builds upon [WhiskeyCoder/Qwen3-Audiobook-Converter](https://github.com/WhiskeyCoder/Qwen3-Audiobook-Converter) adding more output files, support for a faster backend, metadata, generated cover art, transcription/speed/language options, better text cleanup, and clearer instructions. It also expects the qwen-tts server to be on different ports per model, so two server processes can run at once. +This builds upon [WhiskeyCoder/Qwen3-Audiobook-Converter](https://github.com/WhiskeyCoder/Qwen3-Audiobook-Converter) adding more output files, support for faster backends, metadata, generated cover art, transcription/speed/language options, better text cleanup, and clearer instructions. The converter supports three TTS backends, selected with `--backend`: the original Qwen3-TTS Gradio demos (`gradio`, default), [faster-qwen3-tts](https://github.com/andimarafioti/faster-qwen3-tts) (`faster`), and [audio.cpp](https://github.com/0xShug0/audio.cpp) (`audiocpp`) — all serving the same Qwen3-TTS 1.7B model. ## Overview @@ -10,11 +10,10 @@ The converter sends text extracted from your books to a locally running Qwen3-TT - Input: `.txt`, `.pdf`, or `.epub` - Output: `.m4b`, `.mp3`, `.ogg`, or `.flac` +- Supports [qwen-tts](https://pypi.org/project/qwen-tts/), [faster-qwen-tts](https://github.com/andimarafioti/faster-qwen3-tts), and [audio.cpp](https://github.com/0xShug0/audio.cpp) backend servers - Output a single file or one per chapter - Automatic metadata (title/artist/album tags, chapter track numbers) and a generated cover -- Two voice modes: - - Custom voice: pre-built speakers - - Voice clone: clone a voice from a `.wav` reference audio file +- Clone voices from .wav reference files or use the built-in speaker in the CustomVoice model. ## Prerequisites @@ -24,40 +23,56 @@ The converter sends text extracted from your books to a locally running Qwen3-TT ## Installation -Install ffmpeg and your python environment of choice, e.g. conda. - -```bash -sudo pacman -S conda ffmpeg #Arch Linux -sudo apt-get install ffmpeg #Debian, conda must be installed separately -``` - -### Install Qwen3-TTS (Server) - ```bash conda create -n qwen3-tts python=3.12 -y conda activate qwen3-tts -pip install -U qwen-tts -``` - -### Install the conversion script - -```bash git clone https://git.historia.vg/git/qwen3-audiobook-converter cd qwen3-audiobook-converter pip install -r requirements.txt ``` -## Running the Qwen Gradio server and audiobook script +Put your book files (epub, etc.) in the `input/` directory. The output goes to `output/`. -The audiobook script talks to a Qwen3-TTS Gradio server that is run using `qwen-tts-demo`. Add `--no-flash-attn` if FlashAttention isn't installed (see below). The script expects the custom voice model and base model to be on different ports depending on which you're using. The Qwen model(s) will automatically download. +You will also need to install one of the following backends (see below for installation/usage) -Put your book files (epub, etc.) in the `input/` folder. Then run the script. The output goes to `output/`. +| Backend | Description | +| -------------------------------------------------------------------- | ------------------------------------------------- | +| [Qwen-TTS](https://pypi.org/project/qwen-tts/) | Gradio server released by Qwen | +| [Faster-Qwen-TTS](https://github.com/andimarafioti/faster-qwen3-tts) | Server with 2-8x faster inference for NVidia GPUs | +| [audio.cpp](https://github.com/0xShug0/audio.cpp) | Newer C++ TTS backend that supports Qwen-TTS | -### Voice clone +## Options + +| Flag | Description | +| ----------------------------- | ------------------------------------------------------------------------------------------------ | +| `--format {mp3,m4b,ogg,flac}` | Output format (default `m4b`). `m4b` uses AAC audio and has built-in chapters. | +| `--clone ` | Reference audio (`wav`) for voice cloning. | +| `--transcription "..."` | Override whisper auto-transcription with manual audio transcript. | +| `--no-transcription` | Skip auto-transcription of the reference audio. | +| `--speed ` | Playback speed, pitch-preserving (`1.0` = normal). A normal-speed copy is also output. | +| `--single-file` | Merge all chapters into a single file (default: one file per chapter). `m4b` is always one file. | +| `--language ` | Output language for the synthesized speech. Can add an accent even if the text is English. | +| `--backend {gradio,faster,audiocpp}` | TTS server to use (default `gradio`). `faster` and `audiocpp` require their server running first — see the backend sections above. | +| `--voice ` | Voice to request from a server-side voice configuration (`--backend faster` or `audiocpp` only). | +| `--debug` | Troubleshooting: dump each chunk's raw audio and sent text to `debug/` and log every request. | + +Other options including backend server URLs/ports are configured in `converter/config.py` + +## Backend Option 1: Qwen3-TTS + +Install qwen-tts with pip: ```bash conda activate qwen3-tts -qwen-tts-demo Qwen/Qwen3-TTS-12Hz-1.7B-Base --ip 127.0.0.1 --port 7861 --no-flash-attn +pip install -U qwen-tts +``` + +Run the backend with `qwen-tts-demo`. Add `--no-flash-attn` if FlashAttention isn't installed (see below). Note that the Base model and CustomVoice model run on different ports. + +### Voice clone + +```bash +qwen-tts-demo Qwen/Qwen3-TTS-12Hz-1.7B-Base --ip 127.0.0.1 --port 7861 [--no-flash-attn] ``` Then in another terminal: @@ -75,7 +90,7 @@ Whisper (`faster_whisper` or `whisper`) is used automatically to transcribe the ```bash conda activate qwen3-tts -qwen-tts-demo Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --ip 127.0.0.1 --port 7860 --no-flash-attn +qwen-tts-demo Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --ip 127.0.0.1 --port 7860 [--no-flash-attn] ``` ```bash @@ -85,22 +100,22 @@ python audiobook.py Change the voice settings in `converter/config.py`. -## Optional: Using the `--faster` backend - -Instead of the Qwen Gradio demos, `--faster` talks to the OpenAI-compatible server from [faster-qwen3-tts](https://github.com/andimarafioti/faster-qwen3-tts), which uses CUDA graph capture for roughly 5-10x faster inference with the same models. **It requires an NVIDIA GPU**. +## Backend Option 2: faster-qwen-tts -**The `--faster` backend always uses voice cloning**. The reference voice and language are configured on the **server**, not through the converter. The server does not transcribe reference audio itself, so transcripts must come from you — either by hand, or with the `tools/make_voices.py` helper (see below). +`--backend faster` talks to the OpenAI-compatible server from [faster-qwen3-tts](https://github.com/andimarafioti/faster-qwen3-tts), which uses CUDA graph capture for roughly 5-10x faster inference with the same models. **It requires an NVIDIA GPU**. -### Install -Install into the **same `qwen3-tts` conda environment** used for the Gradio server. +Install into the **same `qwen3-tts` conda environment** used for qwen-tts. ```bash conda activate qwen3-tts +pip install -U qwen-tts pip install "faster-qwen3-tts[demo]" ``` -### Run the server +### Voice Clone + +**This backend always uses voice cloning**. The reference voice and language are configured on the **server**, not through the converter. The server does not transcribe reference audio itself, so do it manually or use the `tools/make_faster_voices_json.py` helper (see below). The pip package does not include the server script, so clone the repository: @@ -109,17 +124,7 @@ git clone https://github.com/andimarafioti/faster-qwen3-tts cd faster-qwen3-tts ``` -Single voice (the voice is named `default`): - -```bash -python examples/openai_server.py \ - --model Qwen/Qwen3-TTS-12Hz-1.7B-Base \ - --ref-audio /absolute/path/to/reference.wav \ - --ref-text "Transcript of the reference audio." \ - --language English --port 8000 -``` - -Multiple voices — create a `voices.json` mapping names to reference configurations: +Create a `voices.json` mapping names to reference configurations (.wav to clone, transcript, language). Optionally run `python ./tools/make_faster_voices_json.py path/to/clone/wavs` to automatically create a `voices.json` using whisper to automatically transcribe the test audio. ```json { @@ -128,44 +133,93 @@ Multiple voices — create a `voices.json` mapping names to reference configurat } ``` +Run the server + ```bash python examples/openai_server.py --voices voices.json --port 8000 ``` -### Generating voices.json (optional) +Then from another terminal, run audiobook.py with `--backend faster` + +```bash +python audiobook.py --backend faster [--voice NAME] +``` + +## Backend Option 3: audio.cpp -The `tools/make_voices.py` helper builds a `voices.json` for the server: it transcribes every `.wav` in a directory with using whisper (which is in the qwen3-tts environment). By default it puts voices.json into the input directory. Check the help with `-h` for more options. +Build `audiocpp_server` for your platform and backend `(cuda, vulkan, hip, cpu)`. Check [audio.cpp's readme](https://github.com/0xShug0/audio.cpp) for details. I'm using one of the helper scripts: ```bash -python tools/make_voices.py path/to/wavs +git clone https://github.com/0xShug0/audio.cpp +cd audio.cpp +scripts/build_linux.sh --backend cuda --target audiocpp_server ``` -### Run the converter +Download the Qwen3-TTS GGUF packages (Base for cloning, CustomVoice for built-in speakers) with the python model manager script. This will download these to `./models` ```bash -python audiobook.py --faster [--faster-voice NAME] +python3 tools/model_manager_v2.py install qwen3_tts_1_7b_base_q8_0 +python3 tools/model_manager_v2.py install qwen3_tts_1_7b_customvoice_q8_0 ``` -## Options +Create a `server.json` file. One server can host multiple models. Note that the `id:` field(s) must match `AUDIOCPP_MODEL_ID` and `AUDIOCPP_CLONE_MODEL_ID` in qwen3_ebook_converter's .`converter/config.py`. Optionally run `tools/make_audiocpp_server_json.py path/to/clone/wavs` to make `server.json` for you with automatic whisper transcription. -| Flag | Description | -| ----------------------------- | ------------------------------------------------------------------------------------------------ | -| `--format {mp3,m4b,ogg,flac}` | Output format (default `m4b`). `m4b` uses AAC audio and has built-in chapters. | -| `--clone ` | Reference audio (`wav`) for voice cloning. | -| `--transcription "..."` | Override whisper auto-transcription with manual audio transcript. | -| `--no-transcription` | Skip auto-transcription of the reference audio. | -| `--speed ` | Playback speed, pitch-preserving (`1.0` = normal). A normal-speed copy is also output. | -| `--single-file` | Merge all chapters into a single file (default: one file per chapter). `m4b` is always one file. | -| `--language ` | Output language for the synthesized speech. Can add an accent even if the text is English. | -| `--faster` | Use a faster-qwen3-tts OpenAI-compatible server (up to 5x faster in certain cases). | -| `--faster-voice ` | Chooses a voice from voices.json when using `--faster` with multiple voices. | -| `--debug` | Troubleshooting: dump each chunk's raw audio and sent text to `debug/` and log every request. | +```json +{ + "host": "127.0.0.1", + "port": 8080, + "backend": "cuda", + "lazy_load": false, + "models": [ + { + "id": "qwen", + "family": "qwen3_tts", + "path": "models/Qwen3-TTS-12Hz-1.7B-CustomVoice-GGUF", + "task": "tts", + "mode": "offline" + }, + { + "id": "qwen3-clone", + "family": "qwen3_tts", + "path": "models/Qwen3-TTS-12Hz-1.7B-Base-GGUF", + "task": "tts", + "mode": "offline", + "voice_presets": { + "narrator": { + "voice_ref": "/path/to/reference.wav", + "reference_text": "Transcript of the reference audio." + }, + "obama": { + "voice_ref": "/path/to/reference2.wav", + "reference_text": "Transcript of reference audio." + } + } + } + ] +} +``` + +Run the server. The `audiocpp_server` path will be slightly different depending on your platform and build options: -Other options and defaults are configured in `converter/config.py` +```bash +./build/linux-cuda-release/bin/audiocpp_server --config server.json +``` + +Then in a different terminal, run `audiobook.py` + +```bash +# Built-in speaker +python audiobook.py --backend audiocpp + +# Voice cloning +python audiobook.py --backend audiocpp --voice narrator +``` ## Optional: FlashAttention for qwen-tts-demo server -This is **not** used with the `--faster` backend. The Gradio server tries to use FlashAttention 2 by default, but requires `--no-flash-attn` without it. On supported GPUs FlashAttention can give a modest speedup. +FlashAttention provides a small speed boost on the `qwen-tts-demo` backend. It is **not** relevant with the `faster` or `audiocpp` backends, and switching to either of those will provide a bigger speed boost. + +`qwen-tts-demo` server tries to use FlashAttention 2 by default and requires `--no-flash-attn` without it. You have two options to install FlashAttention in your python environment: 1. Build from source (takes absolutely forever). If you run out of memory, lower MAX_JOBS until you don't. @@ -175,23 +229,23 @@ pip install ninja packaging psutil MAX_JOBS=4 pip install --no-build-isolation flash-attn ``` -2. Or pip install a prebuilt wheel matching your torch / CUDA / Python / CXX11-ABI combination: +2. pip install a prebuilt wheel matching your torch / CUDA / Python / CXX11-ABI combination: ```bash +conda activate qwen3-tts python -c "import torch; print(torch.__version__, torch.version.cuda, torch._C._GLIBCXX_USE_CXX11_ABI)" ``` -Official wheels: https://github.com/Dao-AILab/flash-attention/releases (pick `cp312` + matching `cuX` + `torchX.Y` + `cxx11abiTRUE/FALSE`). - -Third-party wheels: https://mjunya.com/flash-attention-prebuild-wheels/ (hosted at https://github.com/mjun0812/flash-attention-prebuild-wheels). +- [Official wheels](https://github.com/Dao-AILab/flash-attention/releases) - Pick `cp312` + matching `cuX` + `torchX.Y` + `cxx11abiTRUE/FALSE` +- [Third-party wheels](https://mjunya.com/flash-attention-prebuild-wheels/) ## Tips -Transcription affects the output a lot. Whisper is okay, but does not give perfect transcription. A manual transcription passed via `--transcription` is better. +Transcription affects the output a lot. Whisper does not always give perfect transcription. Manual transcription is better. If you're cloning one language and outputting another language, `--no-transcription` will remove the accent. Alternatively, setting the "wrong" output `--language` can add an accent. -Even tiny amounts of pause between phrases in the sample audio can have a big impact. Try increasing or decreasing them. +Even tiny amounts of pause between phrases in the sample audio can have a big impact. Try increasing or decreasing them or find a sample with different cadence. ## License @@ -199,5 +253,6 @@ MIT ## Credits -- [Qwen3-Audiobook-Converter](https://github.com/WhiskeyCoder/Qwen3-Audiobook-Converter) by WhiskeyCoder. +- [Qwen3-Audiobook-Converter](https://github.com/WhiskeyCoder/Qwen3-Audiobook-Converter) by WhiskeyCoder. This project was originally built upon this. - [Qwen3-TTS](https://github.com/QwenLM/Qwen3-TTS) voice model. +- [audio.cpp](https://github.com/0xShug0/audio.cpp) inference engine for the `audiocpp` backend. diff --git a/audiobook.py b/audiobook.py index b8c445e..c2639ce 100755 --- a/audiobook.py +++ b/audiobook.py @@ -27,7 +27,14 @@ from converter.converter import ( setup_directories, setup_logging, ) -from converter.tts import VOICE_MODE_CLONE, VOICE_MODE_CUSTOM, normalize_language +from converter.tts import ( + BACKEND_AUDIOCPP, + BACKEND_FASTER, + BACKEND_GRADIO, + VOICE_MODE_CLONE, + VOICE_MODE_CUSTOM, + normalize_language, +) def main() -> None: @@ -37,14 +44,17 @@ def main() -> None: formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Examples: - # Use custom voice (default - Vivian speaker) + # Use the Qwen Gradio demo with a custom voice (default - Vivian speaker) python audiobook.py - # Use voice cloning with reference audio + # Use the Qwen Gradio demo with voice cloning from reference audio python audiobook.py --clone path/to/reference.wav # Use the faster-qwen3-tts server (voice cloning, configured server-side) - python audiobook.py --faster + python audiobook.py --backend faster [--voice NAME] + + # Use an audio.cpp audiocpp_server (speaker mode, or a server-side voice) + python audiobook.py --backend audiocpp [--voice NAME] """ ) @@ -105,23 +115,26 @@ Examples: ) parser.add_argument( - "--faster", - action="store_true", - help=("Use a faster-qwen3-tts OpenAI-compatible server instead of the Qwen " - "Gradio demos (5-10x faster inference via CUDA graphs). Always voice " - "cloning: the reference audio and transcript are configured on the " - "server itself (see the 'Faster backend' section of the README).") + "--backend", + choices=[BACKEND_GRADIO, BACKEND_FASTER, BACKEND_AUDIOCPP], + default=config.BACKEND, + help=("TTS server to talk to: the Qwen3-TTS Gradio demos (gradio), the " + "faster-qwen3-tts OpenAI-compatible server (faster), or an " + "audio.cpp audiocpp_server (audiocpp). Defaults to the BACKEND " + "setting in converter/config.py (gradio).") ) parser.add_argument( - "--faster-voice", + "--voice", type=str, default=None, metavar="NAME", - help=("Voice entry to request from the faster server's voice config " - "(default: the FASTER_VOICE setting in converter/config.py, " - "typically 'default'). Must match a key in the server's voices.json, " - "or 'default' when the server was started with --ref-audio.") + help=("Voice to request from a server-side voice configuration. faster: " + "a key in the server's voices.json ('default' when it was started " + "with --ref-audio). audiocpp: a voice_preset or voice_dir entry " + "(cloning); without this flag the audiocpp backend uses a built-in " + "CustomVoice speaker instead. Not used by the gradio backend " + "(use converter/config.py SPEAKER or --clone there).") ) parser.add_argument( @@ -137,23 +150,44 @@ Examples: if args.speed <= 0: parser.error(f"--speed must be a positive number (got {args.speed:g})") - if args.faster: + if args.backend == BACKEND_FASTER: if args.clone: - print("[WARNING] --clone is ignored with --faster: the faster backend " + print("[WARNING] --clone is ignored with --backend faster: that backend " "always uses voice cloning, and the reference voice is configured " - "on the faster server (see README)") + "on the server (see README)") args.clone = None if args.transcription or args.no_transcription: print("[WARNING] --transcription/--no-transcription are ignored with " - "--faster: the reference transcript is configured on the faster " + "--backend faster: the reference transcript is configured on the " "server (--ref-text or voices.json, see README)") args.transcription = None args.no_transcription = False if args.language is not None: - print("[WARNING] --language is ignored with --faster: language is " - "configured on the faster server (see README)") + print("[WARNING] --language is ignored with --backend faster: language " + "is configured on the server (see README)") args.language = None + elif args.backend == BACKEND_AUDIOCPP: + if args.clone: + print("[WARNING] --clone is ignored with --backend audiocpp: cloning " + "uses a voice configured on the server (voice_presets or " + "voice_dir in its config); select it with --voice (see README)") + args.clone = None + if args.transcription or args.no_transcription: + print("[WARNING] --transcription/--no-transcription are ignored with " + "--backend audiocpp: the reference transcript is configured on " + "the server (see README)") + args.transcription = None + args.no_transcription = False + if args.language is not None: + try: + args.language = normalize_language(args.language) + except ValueError as exc: + parser.error(str(exc)) else: + if args.voice is not None: + parser.error("--voice requires --backend faster or audiocpp; the " + "gradio backend uses built-in speakers " + "(converter/config.py SPEAKER) or --clone") if args.language is not None: try: args.language = normalize_language(args.language) @@ -167,10 +201,16 @@ Examples: setup_logging(debug=args.debug) setup_directories() + if args.backend == BACKEND_FASTER: + voice_mode = VOICE_MODE_CLONE + elif args.backend == BACKEND_AUDIOCPP: + voice_mode = VOICE_MODE_CLONE if args.voice else VOICE_MODE_CUSTOM + else: + voice_mode = VOICE_MODE_CLONE if args.clone else VOICE_MODE_CUSTOM + try: converter = AudiobookConverter( - voice_mode=VOICE_MODE_CLONE if (args.clone or args.faster) - else VOICE_MODE_CUSTOM, + voice_mode=voice_mode, voice_clone_ref_audio=args.clone, voice_clone_ref_text=args.transcription, skip_transcription=args.no_transcription, @@ -178,8 +218,8 @@ Examples: single_file=args.single_file, output_format=args.format, language=args.language, - faster=args.faster, - faster_voice=args.faster_voice, + backend=args.backend, + voice=args.voice, debug=args.debug, ) ok = converter.run() diff --git a/converter/config.py b/converter/config.py index 9f4dd95..4cb5ab8 100644 --- a/converter/config.py +++ b/converter/config.py @@ -1,50 +1,58 @@ -"""Configuration for the audiobook converter. - -Edit these values to change voices and processing behavior. Everything -else (voice mode names, languages, speaker names, model ids, folders, -file formats) is fixed in the code where it is used. -""" - -# Server endpoints. Voice clone needs the Base-model demo, which is a -# separate server from the CustomVoice demo (that one only exposes -# /run_instruct); the faster backend is the OpenAI-compatible server from -# the faster-qwen3-tts repository. -QWEN_API_URL = "http://127.0.0.1:7860" # CustomVoice demo -CLONE_API_URL = "http://127.0.0.1:7861" # Base-model demo -FASTER_API_URL = "http://127.0.0.1:8000" # faster-qwen3-tts server - -# Words per TTS generation request. Each request is ONE model generation: -# the voice is re-sampled per request (every chunk boundary can drift -# slightly), while over-long generations lose prosody and can turn garbled. -# ~250 words is ~1.5-2 minutes of speech: few voice boundaries while -# staying inside both servers' generation caps. +# Default output options +AUDIO_FORMAT = "m4b" +AUDIO_BITRATE = "128k" +LANGUAGE = "English" + +API_TIMEOUT = 600 # Timeout per chunk request in seconds +MAX_RETRIES = 3 # Attempts per chunk request +HEARTBEAT_INTERVAL_SECONDS = 30 # Print "still working" in console logs every N seconds + +# Words per TTS generation request. +# Note that qwen-tts-demo does no chunking at all, but faster-qwen-tts and +# audio.cpp may do chunking as well, so you may be needlessly double-chunking. CHUNK_SIZE = 250 -API_TIMEOUT = 600 # Seconds before an API call times out (a ~250-word request can take minutes on the Gradio demo) -MAX_RETRIES = 3 # Attempts per chunk request -HEARTBEAT_INTERVAL_SECONDS = 30 # Print "still working" this often during a chunk +# Default TTS backend. +# gradio: qwen-tts-demo +# faster: faster-qwen-tts +# audiocpp: audiocpp_server +# The --backend CLI flag overrides this +BACKEND = "gradio" -LANGUAGE = "English" +############################################################################### +# BACKEND 1: qwen-tts-demo (gradio) options # +############################################################################### -# Seed sent to the TTS servers (only the endpoints that accept one; the -# primary /run_instruct and /run_voice_clone endpoints and the faster -# backend never receive a seed). -1 means "randomize per generation". -# With CONSTANT_SEED = True and SEED = -1, one random seed is drawn at -# startup and reused for every request of the run, keeping the voice -# consistent across chunk boundaries; set SEED to a fixed number to also -# reproduce the same voice across runs. -SEED = -1 -CONSTANT_SEED = True +# There are different API URLs for CustomVoice and Base models so you can run both at once +QWEN_API_URL = "http://127.0.0.1:7860" # CustomVoice model +CLONE_API_URL = "http://127.0.0.1:7861" # Base model -SPEAKER = "Vivian" +# Custom voice options +SPEAKER = "Vivian" #Vivian, Serena, Uncle_Fu, Dylan, Eric, Ryan, Aiden, Ono_Anna, Sohee INSTRUCT = "Speak naturally and clearly, as if reading a dramatic book to an adult audience." +# Don't clone with transcription, only use x-vector-only cloning. Generally "worse" XVECTOR_ONLY = False -# Must match a key in the faster server's voices.json ("default" when the -# server was launched with --ref-audio). The server silently falls back to -# its first voice for unknown names. +# Randomization seed. -1 means randomize with every generation +# With SEED = -1 and CONSTANT_SEED = True, one random seed will be used for the entire audiobook. +# This may keep the voice slightly more consistent across chunk boundaries +SEED = -1 +CONSTANT_SEED = False + +############################################################################### +# BACKEND 2: faster-qwen-tts options # +############################################################################### +FASTER_API_URL = "http://127.0.0.1:8000" # faster-qwen3-tts server (Base model only) + +# Default voice if no --voice is passed FASTER_VOICE = "default" -AUDIO_FORMAT = "m4b" # Default output container -AUDIO_BITRATE = "128k" +############################################################################### +# BACKEND 3: audio.cpp options # +############################################################################### +AUDIOCPP_API_URL = "http://127.0.0.1:8080" # audio.cpp audiocpp_server + +# Model ids in the audio.cpp server.json config. +AUDIOCPP_MODEL_ID = "qwen"https://github.com/0xShug0/audio.cpp +AUDIOCPP_CLONE_MODEL_ID = "qwen-clone" diff --git a/converter/converter.py b/converter/converter.py index 4422316..eb3e80e 100644 --- a/converter/converter.py +++ b/converter/converter.py @@ -15,10 +15,15 @@ from typing import Dict, List, Optional, Tuple from . import audio, chunking, config, cover, extractors from .audio import TrackMeta from .tts import ( + BACKENDS, + BACKEND_AUDIOCPP, + BACKEND_FASTER, + BACKEND_GRADIO, MODEL_SIZE, VOICE_MODE_CLONE, VOICE_MODE_CUSTOM, VOICE_MODES, + AudioCppTTSClient, FasterTTSClient, QwenTTSClient, normalize_language, @@ -128,12 +133,16 @@ class AudiobookConverter: def __init__(self, voice_mode: str = VOICE_MODE_CUSTOM, voice_clone_ref_audio: Optional[str] = None, voice_clone_ref_text: Optional[str] = None, skip_transcription: bool = False, speed: float = 1.0, single_file: bool = False, output_format: str = config.AUDIO_FORMAT, - language: Optional[str] = None, faster: bool = False, - faster_voice: Optional[str] = None, debug: bool = False): + language: Optional[str] = None, backend: str = BACKEND_GRADIO, + voice: Optional[str] = None, debug: bool = False): if speed <= 0: raise ValueError(f"Speed must be a positive number, got {speed}") if output_format not in AUDIO_FORMATS: raise ValueError(f"Unsupported output format: {output_format}") + if backend not in BACKENDS: + raise ValueError( + f"Unknown backend: {backend!r} (expected one of {BACKENDS})" + ) if language is None: language = config.LANGUAGE self.language = normalize_language(language) @@ -142,14 +151,18 @@ class AudiobookConverter: self.speed = speed self.single_file = single_file self.output_format = output_format - self.faster = faster - self.faster_voice = faster_voice + self.backend = backend + self.voice = voice self.debug = bool(debug) self._validate_configuration() - if faster: + if backend == BACKEND_FASTER: # The faster backend always voice-clones using a reference voice # configured on the server, so no local reference audio is needed. - self.tts = FasterTTSClient(voice=faster_voice) + self.tts = FasterTTSClient(voice=voice) + elif backend == BACKEND_AUDIOCPP: + # Speaker mode (no voice) uses a built-in CustomVoice speaker; + # an explicit voice selects a server-side preset (cloning). + self.tts = AudioCppTTSClient(voice=voice, language=self.language) else: self.tts = QwenTTSClient( voice_mode=voice_mode, @@ -166,7 +179,7 @@ class AudiobookConverter: f"Unknown voice mode: {self.voice_mode!r} " f"(expected one of {VOICE_MODES})" ) - if self.voice_mode == VOICE_MODE_CLONE and not self.faster: + if self.voice_mode == VOICE_MODE_CLONE and self.backend == BACKEND_GRADIO: if not self.voice_clone_ref_audio: raise ValueError( "Voice Clone mode requires a reference audio file. " @@ -189,12 +202,15 @@ class AudiobookConverter: """Narrator name used in output file names. Custom voice mode uses the built-in speaker's display name; voice - clone mode uses the reference audio file's stem; the faster backend - uses the server-side voice name. Spaces become underscores - (e.g. "Uncle Fu" -> "Uncle_Fu"). + clone mode uses the reference audio file's stem; the faster and + audiocpp backends use the server-side voice name (falling back to + the built-in speaker for the audiocpp backend's speaker mode). + Spaces become underscores (e.g. "Uncle Fu" -> "Uncle_Fu"). """ - if self.faster: - narrator = self.faster_voice or config.FASTER_VOICE + if self.backend == BACKEND_FASTER: + narrator = self.voice or config.FASTER_VOICE + elif self.backend == BACKEND_AUDIOCPP: + narrator = self.voice or speaker_display_name() elif self.voice_mode == VOICE_MODE_CLONE: narrator = Path(self.voice_clone_ref_audio).stem else: @@ -445,7 +461,11 @@ class AudiobookConverter: chunk_sizes = [len(chunk.split()) for chunk in chunks] avg_chunk_size = sum(chunk_sizes) / len(chunk_sizes) logger.info("Split into %d chunks (avg %.0f words per chunk)", total_chunks, avg_chunk_size) - backend = "faster TTS API" if self.faster else "Qwen API" + backend_labels = { + BACKEND_FASTER: "faster TTS API", + BACKEND_AUDIOCPP: "audio.cpp server", + } + backend = backend_labels.get(self.backend, "Qwen API") print(f"[INFO] Processing {total_chunks} chunks via {backend}...") results = self._synthesize_chunks(chunks, debug_dir=debug_dir) @@ -493,10 +513,20 @@ class AudiobookConverter: print("=" * 70) print(f"Books folder: {BOOKS_FOLDER}") print(f"Output folder: {AUDIOBOOKS_FOLDER}") - if self.faster: + if self.backend == BACKEND_FASTER: print(f"Faster TTS endpoint: {config.FASTER_API_URL}") print("Backend: faster (voice cloning, reference configured on server)") - print(f"Voice: {self.faster_voice or config.FASTER_VOICE}") + print(f"Voice: {self.voice or config.FASTER_VOICE}") + elif self.backend == BACKEND_AUDIOCPP: + print(f"audio.cpp endpoint: {config.AUDIOCPP_API_URL}") + print(f"Model id: {config.AUDIOCPP_MODEL_ID}") + if self.voice: + print("Backend: audio.cpp (voice cloning, reference configured on server)") + print(f"Voice: {self.voice}") + else: + print("Backend: audio.cpp (custom voice, built-in speaker)") + print(f"Speaker: {config.SPEAKER}") + print(f"Language: {self.language}") else: api_url = (config.CLONE_API_URL if self.voice_mode == VOICE_MODE_CLONE else config.QWEN_API_URL) diff --git a/converter/tts.py b/converter/tts.py index ff9da2b..741d9d3 100644 --- a/converter/tts.py +++ b/converter/tts.py @@ -4,6 +4,9 @@ QwenTTSClient talks to the Qwen3-TTS Gradio demos (custom voice / voice clone). FasterTTSClient talks to the OpenAI-compatible server from the faster-qwen3-tts repository (voice cloning only; the reference voice is configured server-side — see the "Faster backend" section of the README). +AudioCppTTSClient talks to the audiocpp_server from the audio.cpp +repository, which serves the same Qwen3-TTS models through an +OpenAI-style API (see the "audio.cpp backend" section of the README). """ import contextlib @@ -17,6 +20,7 @@ import tempfile import threading import time import urllib.error +import urllib.parse import urllib.request import wave from pathlib import Path @@ -33,6 +37,12 @@ VOICE_MODE_CUSTOM = "custom_voice" VOICE_MODE_CLONE = "voice_clone" VOICE_MODES = (VOICE_MODE_CUSTOM, VOICE_MODE_CLONE) +# TTS backends (re-exported for the CLI and the converter orchestrator). +BACKEND_GRADIO = "gradio" +BACKEND_FASTER = "faster" +BACKEND_AUDIOCPP = "audiocpp" +BACKENDS = (BACKEND_GRADIO, BACKEND_FASTER, BACKEND_AUDIOCPP) + # Languages understood by the Qwen3-TTS API. Display names must match the # demo dropdown exactly (the demo silently falls back to "Auto" for # unrecognized values, so languages are validated client-side first). @@ -92,6 +102,21 @@ SAMPLE_RATE = 24000 CHUNKS_FOLDER = Path(__file__).resolve().parent.parent / "chunks" +def _resolve_request_seed() -> int: + """Resolve the seed sent with every request. + + Returns config.SEED as-is, or (with CONSTANT_SEED and SEED < 0) one + random value drawn per run, meant to be reused for every request so + the voice stays consistent across chunk boundaries. Without + CONSTANT_SEED, -1 is returned so the server re-samples the voice on + every generation. + """ + seed = config.SEED + if config.CONSTANT_SEED and seed < 0: + seed = random.randrange(2 ** 31) + return seed + + def speaker_display_name() -> str: """Return the Gradio display name for the configured custom speaker.""" return SPEAKER_DISPLAY_NAMES.get( @@ -289,9 +314,7 @@ class QwenTTSClient(_BaseTTSClient): # reused for every request so the voice stays consistent across # chunk boundaries. Without CONSTANT_SEED, -1 is forwarded so the # server re-samples the voice on every generation. - self._seed = config.SEED - if config.CONSTANT_SEED and self._seed < 0: - self._seed = random.randrange(2 ** 31) + self._seed = _resolve_request_seed() if language is None: language = config.LANGUAGE # Validate before connecting so bad values fail fast without a server. @@ -677,3 +700,255 @@ class FasterTTSClient(_BaseTTSClient): except Exception as exc: logger.error("Faster chunk processing failed for chunk %d: %s", chunk_num, exc) return None + + +class AudioCppTTSClient(_BaseTTSClient): + """Generates audio chunks through an audio.cpp audiocpp_server. + + Talks to the OpenAI-style HTTP API of audiocpp_server, which serves + the same Qwen3-TTS models as the Gradio demos through a native + ggml runtime (GGUF weights, no Python serving stack). Two voice + modes, both resolved server-side from the request's "voice" field: + + - Speaker mode (no ``voice``): a built-in CustomVoice speaker name + (e.g. "Vivian") is passed through, plus the INSTRUCT style prompt. + The server must be configured with the CustomVoice model for this. + - Preset mode (``voice=NAME``): a voice configured on the server + (``voice_presets`` or ``voice_dir`` in its config, e.g. a cloning + reference). The name is validated against GET /v1/audio/voices at + startup because an unresolvable name would silently fall back to + plain TTS on the Base model instead of failing. When + AUDIOCPP_CLONE_MODEL_ID names a second server entry (typically the + Base model), preset requests are routed to it. + + Each response is a complete WAV file, so sub-request audio is + concatenated with the same lossless path used for the Gradio client. + """ + + def __init__(self, voice: Optional[str] = None, language: Optional[str] = None, + api_url: Optional[str] = None): + self.api_url = (api_url or config.AUDIOCPP_API_URL).rstrip("/") + self.model_id = config.AUDIOCPP_MODEL_ID + # Validate before connecting so bad values fail fast without a server. + self.language = normalize_language( + language if language is not None else config.LANGUAGE) + # Same seed convention as the Gradio client: one value per run, + # reused for every request (see _resolve_request_seed). + self._seed = _resolve_request_seed() + self.preset_mode = bool(voice) + self.voice = voice or speaker_display_name() + self._check_health() + model_ids = self._check_model() + if self.preset_mode: + self._select_model(model_ids) + self._check_voice() + print(f"[OK] Connected to audio.cpp server at {self.api_url} " + f"(model '{self.model_id}', voice '{self.voice}')") + else: + print(f"[OK] Connected to audio.cpp server at {self.api_url} " + f"(model '{self.model_id}', speaker '{self.voice}')") + print("[INFO] Speaker mode expects the server to be configured with the " + "CustomVoice model; with the Base model the speaker name is ignored " + "and a random default voice is used (see README).") + + # ------------------------------------------------------------------ + # Connection + # ------------------------------------------------------------------ + + def _get_json(self, path: str, timeout: int = 10) -> Dict[str, Any]: + """GET a JSON document from the server.""" + url = f"{self.api_url}{path}" + try: + with urllib.request.urlopen(url, timeout=timeout) as response: + return json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + detail = "" + try: + detail = exc.read().decode("utf-8", errors="replace")[:200] + except Exception: + pass + raise RuntimeError( + f"audio.cpp server returned HTTP {exc.code} for {path}: {detail}") from exc + except urllib.error.URLError as exc: + raise RuntimeError(f"audio.cpp request failed for {path}: {exc.reason}") from exc + + def _check_health(self) -> None: + """Verify the server is reachable and reports healthy.""" + try: + payload = self._get_json("/health") + except Exception as exc: + raise RuntimeError( + f"audio.cpp server not reachable at {self.api_url}: {exc}. " + "Start audiocpp_server first (see the 'audio.cpp backend' " + "section of the README)." + ) from exc + if payload.get("status") != "ok": + raise RuntimeError( + f"The audio.cpp server at {self.api_url} reports status " + f"{payload.get('status')!r} instead of 'ok'") + + def _check_model(self) -> List[str]: + """Verify the configured model id exists; return all server model ids.""" + try: + payload = self._get_json("/v1/models") + except Exception as exc: + raise RuntimeError( + f"The audio.cpp server at {self.api_url} did not answer " + f"/v1/models: {exc}") from exc + entries = payload.get("data") or [] + model_ids = [entry.get("id") for entry in entries if isinstance(entry, dict)] + if self.model_id not in model_ids: + configured = ", ".join(str(mid) for mid in model_ids if mid) or "none" + raise RuntimeError( + f"The audio.cpp server at {self.api_url} has no model id " + f"'{self.model_id}' (configured: {configured}). Add a qwen3_tts " + "model entry to the server config and match AUDIOCPP_MODEL_ID " + "in converter/config.py to its id (see README)." + ) + return [mid for mid in model_ids if mid] + + def _select_model(self, model_ids: List[str]) -> None: + """Pick the model for preset (cloning) requests. + + Defaults to the primary model id. When AUDIOCPP_CLONE_MODEL_ID is + configured (typically a Base-model entry, since only that variant + consumes reference audio) and present on the server, preset + requests are routed to it instead, so one server can host the + CustomVoice model for speaker mode and the Base model for + cloning. + """ + clone_model_id = config.AUDIOCPP_CLONE_MODEL_ID + if not clone_model_id or clone_model_id == self.model_id: + return + if clone_model_id in model_ids: + self.model_id = clone_model_id + else: + logger.warning( + "AUDIOCPP_CLONE_MODEL_ID %r is not configured on the audio.cpp " + "server; preset requests use '%s' instead", + clone_model_id, self.model_id) + + def _check_voice(self) -> None: + """Verify the requested voice is available on the server. + + A voice name that matches no server preset or voice-library wav + would be passed through to the model as a cached voice id; on the + Base (cloning) model that is silently ignored and plain TTS audio + comes back, so preset names are validated up front. When the + voices endpoint cannot be queried, validation is skipped with a + warning rather than blocking the run. + """ + query = urllib.parse.urlencode({"model": self.model_id}) + try: + payload = self._get_json(f"/v1/audio/voices?{query}") + except Exception as exc: + logger.warning("Could not list server voices; skipping voice " + "validation: %s", exc) + return + voices = payload.get("voices") or [] + if self.voice not in voices: + available = ", ".join(str(v) for v in voices) or "none" + raise RuntimeError( + f"Voice '{self.voice}' is not available on the audio.cpp server " + f"(available: {available}). Configure it as a voice_preset or " + "voice_dir entry in the server config, or pass a listed name " + "with --voice (see README)." + ) + + # ------------------------------------------------------------------ + # HTTP requests + # ------------------------------------------------------------------ + + def _request_wav(self, text: str) -> bytes: + """POST one sub-chunk and return the raw WAV bytes.""" + url = f"{self.api_url}/v1/audio/speech" + payload: Dict[str, Any] = { + "model": self.model_id, + "input": text, + "voice": self.voice, + "language": self.language, + "seed": self._seed, + } + if not self.preset_mode and config.INSTRUCT: + # Style instruction for the CustomVoice speakers; ignored by + # the Base (cloning) model. + payload["instructions"] = config.INSTRUCT + request = urllib.request.Request( + url, data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, method="POST") + try: + with urllib.request.urlopen(request, timeout=config.API_TIMEOUT) as response: + wav = response.read() + except urllib.error.HTTPError as exc: + detail = "" + try: + detail = exc.read().decode("utf-8", errors="replace")[:200] + except Exception: + pass + raise RuntimeError(f"audio.cpp server returned HTTP {exc.code}: {detail}") from exc + except urllib.error.URLError as exc: + raise RuntimeError(f"audio.cpp request failed: {exc.reason}") from exc + if len(wav) < 12 or wav[:4] != b"RIFF" or wav[8:12] != b"WAVE": + raise RuntimeError("audio.cpp server returned audio that is not a WAV file") + return wav + + def _request_wav_with_retry(self, text: str, chunk_num: int, sub_num: int, + sub_total: int) -> bytes: + """Request one sub-chunk, retrying transient failures.""" + for attempt in range(config.MAX_RETRIES): + try: + return self._request_wav(text) + except Exception as exc: + logger.warning("Chunk %d sub-chunk %d/%d attempt %d failed: %s", + chunk_num, sub_num, sub_total, attempt + 1, exc) + if attempt < config.MAX_RETRIES - 1: + time.sleep(2 + 2 * attempt) + raise RuntimeError(f"Sub-chunk {sub_num}/{sub_total} failed after " + f"{config.MAX_RETRIES} attempts") + + # ------------------------------------------------------------------ + # Chunk generation + # ------------------------------------------------------------------ + + def generate_chunk(self, text: str, chunk_num: int) -> Optional[str]: + """Generate one audio chunk; returns its path in the chunks folder. + + The text is split into sub-requests of at most + ``MAX_REQUEST_WORDS`` words each (defense in depth against + pathological input, matching the Gradio client), each sub-request + returns a complete WAV file, and the parts are concatenated into + one chunk file. + """ + try: + sub_texts = split_into_chunks(text, max_words=MAX_REQUEST_WORDS) + if not sub_texts: + raise RuntimeError("No text to synthesize") + + output_path: Optional[Path] = None + with tempfile.TemporaryDirectory(prefix="tts_parts_") as parts_dir, \ + self._chunk_heartbeat(chunk_num): + part_paths = [] + for sub_num, sub_text in enumerate(sub_texts, 1): + wav = self._request_wav_with_retry( + sub_text, chunk_num, sub_num, len(sub_texts)) + destination = Path(parts_dir) / f"part_{sub_num:02d}.wav" + destination.write_bytes(wav) + check_for_truncation( + sub_text, _audio_duration_seconds(destination), + f"Chunk {chunk_num} sub-request {sub_num}/{len(sub_texts)}") + part_paths.append(destination) + if len(part_paths) == 1: + output_path = self._chunk_path(chunk_num, ".wav") + shutil.copy2(part_paths[0], output_path) + else: + output_path = self._chunk_path(chunk_num, ".wav") + concat_audio_files(part_paths, output_path) + + logger.debug("Chunk %d generated successfully (%d sub-request(s))", + chunk_num, len(sub_texts)) + return str(output_path) + + except Exception as exc: + logger.error("audio.cpp chunk processing failed for chunk %d: %s", + chunk_num, exc) + return None diff --git a/tests/test_converter.py b/tests/test_converter.py index 20ebabc..5ef9785 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -45,6 +45,12 @@ class ConfigurationValidationTests(unittest.TestCase): with self.assertRaises(ValueError): AudiobookConverter(language="klingon") + def test_unknown_backend_rejected(self): + with self.assertRaises(ValueError) as ctx: + AudiobookConverter(backend="piper") + self.assertIn("piper", str(ctx.exception)) + self.assertIn("audiocpp", str(ctx.exception)) + def test_language_defaults_to_config(self): with patch("converter.converter.QwenTTSClient") as mock_tts: AudiobookConverter() @@ -120,8 +126,8 @@ class NarratorTagTests(unittest.TestCase): converter = AudiobookConverter.__new__(AudiobookConverter) converter.voice_mode = voice_mode converter.voice_clone_ref_audio = ref_audio - converter.faster = False - converter.faster_voice = None + converter.backend = tts.BACKEND_GRADIO + converter.voice = None return converter def test_custom_voice_uses_speaker_display_name(self): @@ -383,8 +389,8 @@ class RunOverwritePromptTests(unittest.TestCase): self.converter = AudiobookConverter.__new__(AudiobookConverter) self.converter.voice_mode = tts.VOICE_MODE_CUSTOM self.converter.voice_clone_ref_audio = None - self.converter.faster = False - self.converter.faster_voice = None + self.converter.backend = tts.BACKEND_GRADIO + self.converter.voice = None self.converter.speed = 1.0 self.converter.single_file = False self.converter.output_format = "mp3" diff --git a/tests/test_tts.py b/tests/test_tts.py index 64b4846..813332d 100644 --- a/tests/test_tts.py +++ b/tests/test_tts.py @@ -1,5 +1,6 @@ """Tests for the TTS client wrappers (language handling and payloads).""" +import io import json import tempfile import unittest @@ -9,7 +10,12 @@ from unittest.mock import MagicMock, patch from converter import config, tts from converter.converter import AudiobookConverter -from converter.tts import FasterTTSClient, QwenTTSClient, normalize_language +from converter.tts import ( + AudioCppTTSClient, + FasterTTSClient, + QwenTTSClient, + normalize_language, +) class NormalizeLanguageTests(unittest.TestCase): @@ -518,47 +524,444 @@ class QwenTTSClientGenerateTests(unittest.TestCase): mock_generate.assert_not_called() -class FasterModeWiringTests(unittest.TestCase): - """AudiobookConverter wiring for the --faster backend.""" +class AudioCppTTSClientHealthTests(unittest.TestCase): + """Connection behavior of the audio.cpp client.""" - def test_faster_mode_uses_faster_client_without_reference(self): + @staticmethod + def _json_response(payload): + response = MagicMock() + response.__enter__.return_value = response + response.read.return_value = json.dumps(payload).encode("utf-8") + return response + + def _get_responses(self, health=None, models=None, voices=None): + """Side effect dispatching GET responses by URL.""" + def _dispatch(request, **_kwargs): + url = request if isinstance(request, str) else request.full_url + if url.endswith("/health"): + return self._json_response(health if health is not None + else {"status": "ok"}) + if url.endswith("/v1/models"): + return self._json_response(models if models is not None else + {"data": [{"id": config.AUDIOCPP_MODEL_ID}]}) + if "/v1/audio/voices" in url: + if voices is Exception: + raise Exception("voices endpoint down") + return self._json_response(voices if voices is not None + else {"voices": ["narrator"]}) + raise AssertionError(f"unexpected URL: {url}") + return _dispatch + + def _client(self, voice=None, language=None, **kwargs): + with patch("converter.tts.urllib.request.urlopen", + side_effect=self._get_responses(**kwargs)): + return AudioCppTTSClient(voice=voice, language=language) + + def test_unreachable_server_raises_with_readme_pointer(self): + import urllib.error + with patch("converter.tts.urllib.request.urlopen", + side_effect=urllib.error.URLError("Connection refused")): + with self.assertRaises(RuntimeError) as ctx: + AudioCppTTSClient() + message = str(ctx.exception) + self.assertIn("not reachable", message) + self.assertIn("README", message) + + def test_unhealthy_status_raises(self): + with self.assertRaises(RuntimeError) as ctx: + self._client(health={"status": "starting"}) + self.assertIn("starting", str(ctx.exception)) + + def test_unknown_model_id_raises_with_configured_ids(self): + with self.assertRaises(RuntimeError) as ctx: + self._client(models={"data": [{"id": "pocket-tts"}, {"id": "other"}]}) + message = str(ctx.exception) + self.assertIn(config.AUDIOCPP_MODEL_ID, message) + self.assertIn("pocket-tts", message) + self.assertIn("other", message) + + def test_healthy_server_speaker_mode_defaults(self): + client = self._client() + self.assertEqual(client.api_url, config.AUDIOCPP_API_URL.rstrip("/")) + self.assertEqual(client.model_id, config.AUDIOCPP_MODEL_ID) + self.assertEqual(client.language, config.LANGUAGE) + self.assertEqual(client.voice, "Vivian") + self.assertFalse(client.preset_mode) + + def test_speaker_mode_uses_configured_speaker(self): + with patch.object(config, "SPEAKER", "uncle_fu"): + client = self._client() + self.assertEqual(client.voice, "Uncle Fu") + + def test_preset_mode_uses_requested_voice(self): + client = self._client(voice="narrator") + self.assertEqual(client.voice, "narrator") + self.assertTrue(client.preset_mode) + + def test_preset_mode_validates_voice_against_server_list(self): + with self.assertRaises(RuntimeError) as ctx: + self._client(voice="ghost", voices={"voices": ["narrator", "obama"]}) + message = str(ctx.exception) + self.assertIn("ghost", message) + self.assertIn("narrator", message) + self.assertIn("obama", message) + + def test_preset_mode_skips_validation_when_voices_endpoint_fails(self): + client = self._client(voice="narrator", voices=Exception) + self.assertEqual(client.voice, "narrator") + + def test_invalid_language_fails_before_connect(self): + with patch("converter.tts.urllib.request.urlopen") as mock_urlopen: + with self.assertRaises(ValueError): + AudioCppTTSClient(language="klingon") + mock_urlopen.assert_not_called() + + def test_explicit_language_normalized(self): + client = self._client(language="ja") + self.assertEqual(client.language, "Japanese") + + def test_seed_resolved_once_per_run(self): + with patch.object(config, "CONSTANT_SEED", True), \ + patch.object(config, "SEED", -1): + client = self._client() + self.assertGreaterEqual(client._seed, 0) + + def test_preset_mode_routes_to_clone_model_when_configured(self): + with patch.object(config, "AUDIOCPP_CLONE_MODEL_ID", "qwen3-tts-clone"): + client = self._client( + voice="narrator", + models={"data": [{"id": "qwen3-tts"}, {"id": "qwen3-tts-clone"}]}) + self.assertEqual(client.model_id, "qwen3-tts-clone") + + def test_preset_mode_falls_back_when_clone_model_not_on_server(self): + with patch.object(config, "AUDIOCPP_CLONE_MODEL_ID", "qwen3-tts-clone"), \ + self.assertLogs("converter.tts", level="WARNING") as logs: + client = self._client( + voice="narrator", + models={"data": [{"id": "qwen3-tts"}, {"id": "pocket-tts"}]}) + self.assertEqual(client.model_id, config.AUDIOCPP_MODEL_ID) + self.assertTrue(any("qwen3-tts-clone" in line for line in logs.output)) + + def test_clone_model_id_ignored_for_speaker_mode(self): + with patch.object(config, "AUDIOCPP_CLONE_MODEL_ID", "qwen3-tts-clone"): + client = self._client( + models={"data": [{"id": "qwen3-tts"}, {"id": "qwen3-tts-clone"}]}) + self.assertEqual(client.model_id, config.AUDIOCPP_MODEL_ID) + + def test_clone_model_id_equal_to_primary_is_noop(self): + with patch.object(config, "AUDIOCPP_CLONE_MODEL_ID", + config.AUDIOCPP_MODEL_ID): + client = self._client(voice="narrator") + self.assertEqual(client.model_id, config.AUDIOCPP_MODEL_ID) + + +class AudioCppTTSClientRequestTests(unittest.TestCase): + """The /v1/audio/speech payload and response validation.""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self._chunks = patch.object(tts, "CHUNKS_FOLDER", Path(self._tmp.name)) + self._chunks.start() + self._sleep = patch("converter.tts.time.sleep") + self._sleep.start() + + def tearDown(self): + self._sleep.stop() + self._chunks.stop() + self._tmp.cleanup() + + @staticmethod + def _make_client(preset_mode=False, voice="Vivian", language="English", seed=-1): + client = AudioCppTTSClient.__new__(AudioCppTTSClient) + client.api_url = "http://127.0.0.1:8080" + client.model_id = config.AUDIOCPP_MODEL_ID + client.preset_mode = preset_mode + client.voice = voice + client.language = language + client._seed = seed + return client + + @staticmethod + def _wav_bytes(frames=b"\x01\x00" * 10, rate=tts.SAMPLE_RATE): + buffer = io.BytesIO() + with wave.open(buffer, "wb") as wav_file: + wav_file.setnchannels(1) + wav_file.setsampwidth(2) + wav_file.setframerate(rate) + wav_file.writeframes(frames) + return buffer.getvalue() + + def _post_response(self, body): + response = MagicMock() + response.__enter__.return_value = response + response.read.return_value = body + return response + + def test_payload_includes_model_input_voice_language_and_seed(self): + client = self._make_client(preset_mode=True, voice="narrator", + language="Japanese", seed=1234) + with patch("converter.tts.urllib.request.urlopen", + return_value=self._post_response(self._wav_bytes())) as mock_urlopen: + client._request_wav("Hello world.") + request = mock_urlopen.call_args[0][0] + self.assertEqual(request.full_url, + "http://127.0.0.1:8080/v1/audio/speech") + payload = json.loads(request.data.decode("utf-8")) + self.assertEqual(payload["model"], config.AUDIOCPP_MODEL_ID) + self.assertEqual(payload["input"], "Hello world.") + self.assertEqual(payload["voice"], "narrator") + self.assertEqual(payload["language"], "Japanese") + self.assertEqual(payload["seed"], 1234) + self.assertNotIn("instructions", payload) + + def test_speaker_mode_sends_instruct(self): + client = self._make_client(preset_mode=False) + with patch("converter.tts.urllib.request.urlopen", + return_value=self._post_response(self._wav_bytes())) as mock_urlopen: + client._request_wav("Hello.") + payload = json.loads(mock_urlopen.call_args[0][0].data.decode("utf-8")) + self.assertEqual(payload["instructions"], config.INSTRUCT) + + def test_non_wav_response_rejected(self): + client = self._make_client() + for body in (b"", b"RIFFxxxx", b"MP3DATA-MP3DATA", b"RIFF\x00\x00\x00\x00mpeg"): + with patch("converter.tts.urllib.request.urlopen", + return_value=self._post_response(body)): + with self.assertRaises(RuntimeError): + client._request_wav("Hello.") + + def test_http_error_body_surfaced(self): + import urllib.error + client = self._make_client() + error = urllib.error.HTTPError( + "http://127.0.0.1:8080/v1/audio/speech", 500, + "Server Error", {}, io.BytesIO(b'{"error":"bad voice"}')) + with patch("converter.tts.urllib.request.urlopen", side_effect=error): + with self.assertRaises(RuntimeError) as ctx: + client._request_wav("Hello.") + self.assertIn("500", str(ctx.exception)) + self.assertIn("bad voice", str(ctx.exception)) + + def test_transient_failure_is_retried(self): + client = self._make_client() + wav = self._wav_bytes() + with patch.object(client, "_request_wav", + side_effect=[RuntimeError("boom"), wav]) as mock_request: + result = client.generate_chunk("Hello.", 1) + self.assertIsNotNone(result) + self.assertEqual(mock_request.call_count, 2) + + def test_exhausted_retries_fail_the_chunk(self): + client = self._make_client() + with patch.object(client, "_request_wav", + side_effect=RuntimeError("down")) as mock_request: + result = client.generate_chunk("Hello.", 1) + self.assertIsNone(result) + self.assertEqual(mock_request.call_count, config.MAX_RETRIES) + + def test_empty_text_fails_the_chunk(self): + client = self._make_client() + with patch.object(client, "_request_wav") as mock_request: + result = client.generate_chunk(" ", 1) + self.assertIsNone(result) + mock_request.assert_not_called() + + def test_generate_chunk_writes_valid_wav(self): + client = self._make_client() + frames = b"\x01\x00" * 100 + with patch.object(client, "_request_wav", return_value=self._wav_bytes(frames)): + result = client.generate_chunk("Hello world.", 1) + self.assertIsNotNone(result) + path = Path(result) + self.assertEqual(path.name, "chunk_0001.wav") + with wave.open(str(path), "rb") as wav_file: + self.assertEqual(wav_file.getnchannels(), 1) + self.assertEqual(wav_file.getsampwidth(), 2) + self.assertEqual(wav_file.getframerate(), tts.SAMPLE_RATE) + self.assertEqual(wav_file.readframes(wav_file.getnframes()), frames) + + def test_long_text_is_subchunked_and_concatenated_in_order(self): + client = self._make_client() + sentences = [" ".join(f"word{i}" for i in range(6)) + "." for _ in range(3)] + text = " ".join(sentences) + parts = [self._wav_bytes(b"\x01\x00" * 10), + self._wav_bytes(b"\x02\x00" * 20), + self._wav_bytes(b"\x03\x00" * 30)] + with patch.object(tts, "MAX_REQUEST_WORDS", 10), \ + patch.object(client, "_request_wav", side_effect=parts) as mock_request: + result = client.generate_chunk(text, 1) + self.assertEqual(mock_request.call_count, 3) + with wave.open(str(Path(result)), "rb") as wav_file: + self.assertEqual(wav_file.readframes(wav_file.getnframes()), + b"\x01\x00" * 10 + b"\x02\x00" * 20 + b"\x03\x00" * 30) + + def test_stale_chunk_files_are_removed(self): + stale = Path(self._tmp.name) / "chunk_0001.mp3" + stale.write_bytes(b"old") + client = self._make_client() + with patch.object(client, "_request_wav", return_value=self._wav_bytes()): + client.generate_chunk("Hello.", 1) + remaining = sorted(path.name for path in Path(self._tmp.name).glob("chunk_0001.*")) + self.assertEqual(remaining, ["chunk_0001.wav"]) + + +class AudioCppTTSClientTruncationTests(unittest.TestCase): + """Audio far shorter than its text implies fails the request.""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self._chunks = patch.object(tts, "CHUNKS_FOLDER", Path(self._tmp.name)) + self._chunks.start() + + def tearDown(self): + self._chunks.stop() + self._tmp.cleanup() + + def _make_client(self): + client = AudioCppTTSClient.__new__(AudioCppTTSClient) + client.api_url = "http://127.0.0.1:8080" + client.model_id = config.AUDIOCPP_MODEL_ID + client.preset_mode = True + client.voice = "narrator" + client.language = "English" + client._seed = -1 + return client + + @staticmethod + def _wav_bytes(frames): + buffer = io.BytesIO() + with wave.open(buffer, "wb") as wav_file: + wav_file.setnchannels(1) + wav_file.setsampwidth(2) + wav_file.setframerate(tts.SAMPLE_RATE) + wav_file.writeframes(frames) + return buffer.getvalue() + + def test_truncated_wav_fails_the_chunk(self): + client = self._make_client() + text = " ".join(f"word{i}" for i in range(12)) + wav = self._wav_bytes(b"\x01\x00" * 24) # 0.001s for ~4.8s of speech + with patch.object(client, "_request_wav", return_value=wav), \ + self.assertLogs("converter.tts", level="ERROR") as logs: + result = client.generate_chunk(text, 1) + self.assertIsNone(result) + self.assertTrue(any("truncated" in line for line in logs.output)) + + def test_full_length_wav_passes(self): + client = self._make_client() + text = " ".join(f"word{i}" for i in range(12)) + # 12 words -> expected 4.8s, half is 2.4s -> 2.5s of audio passes. + wav = self._wav_bytes(b"\x01\x00" * int(2.5 * tts.SAMPLE_RATE)) + with patch.object(client, "_request_wav", return_value=wav): + result = client.generate_chunk(text, 1) + self.assertIsNotNone(result) + + +class BackendWiringTests(unittest.TestCase): + """AudiobookConverter wiring for the --backend selector.""" + + def test_faster_backend_uses_faster_client_without_reference(self): with patch("converter.converter.FasterTTSClient") as mock_faster, \ - patch("converter.converter.QwenTTSClient") as mock_qwen: + patch("converter.converter.QwenTTSClient") as mock_qwen, \ + patch("converter.converter.AudioCppTTSClient") as mock_audiocpp: AudiobookConverter(voice_mode=tts.VOICE_MODE_CLONE, - faster=True, faster_voice="narrator") + backend=tts.BACKEND_FASTER, voice="narrator") mock_faster.assert_called_once_with(voice="narrator") mock_qwen.assert_not_called() + mock_audiocpp.assert_not_called() + + def test_audiocpp_backend_with_voice_uses_audiocpp_client(self): + with patch("converter.converter.FasterTTSClient") as mock_faster, \ + patch("converter.converter.QwenTTSClient") as mock_qwen, \ + patch("converter.converter.AudioCppTTSClient") as mock_audiocpp: + AudiobookConverter(voice_mode=tts.VOICE_MODE_CLONE, + backend=tts.BACKEND_AUDIOCPP, voice="narrator", + language="ja") + mock_audiocpp.assert_called_once_with(voice="narrator", language="Japanese") + mock_faster.assert_not_called() + mock_qwen.assert_not_called() + + def test_audiocpp_backend_without_voice_uses_audiocpp_client(self): + with patch("converter.converter.AudioCppTTSClient") as mock_audiocpp: + AudiobookConverter(voice_mode=tts.VOICE_MODE_CUSTOM, + backend=tts.BACKEND_AUDIOCPP) + mock_audiocpp.assert_called_once_with(voice=None, language=config.LANGUAGE) - def test_non_faster_clone_mode_still_requires_reference(self): + def test_gradio_backend_uses_qwen_client(self): + with patch("converter.converter.FasterTTSClient") as mock_faster, \ + patch("converter.converter.QwenTTSClient") as mock_qwen, \ + patch("converter.converter.AudioCppTTSClient") as mock_audiocpp: + AudiobookConverter(voice_mode=tts.VOICE_MODE_CUSTOM) + mock_qwen.assert_called_once() + mock_faster.assert_not_called() + mock_audiocpp.assert_not_called() + + def test_gradio_clone_mode_still_requires_reference(self): with patch("converter.converter.QwenTTSClient"): with self.assertRaises(ValueError): AudiobookConverter(voice_mode=tts.VOICE_MODE_CLONE) - def test_faster_mode_still_validates_other_settings(self): + def test_audiocpp_clone_mode_does_not_require_reference(self): + # Cloning is server-side for the audiocpp backend, so the + # clone-mode voice can be selected without local reference audio. + with patch("converter.converter.AudioCppTTSClient"): + converter = AudiobookConverter(voice_mode=tts.VOICE_MODE_CLONE, + backend=tts.BACKEND_AUDIOCPP, + voice="narrator") + self.assertIsNone(converter.voice_clone_ref_audio) + + def test_faster_backend_still_validates_other_settings(self): with patch("converter.converter.FasterTTSClient"): with self.assertRaises(ValueError): - AudiobookConverter(faster=True, speed=0) + AudiobookConverter(backend=tts.BACKEND_FASTER, speed=0) + with self.assertRaises(ValueError): + AudiobookConverter(backend=tts.BACKEND_FASTER, language="klingon") + + def test_audiocpp_backend_still_validates_other_settings(self): + with patch("converter.converter.AudioCppTTSClient"): + with self.assertRaises(ValueError): + AudiobookConverter(backend=tts.BACKEND_AUDIOCPP, speed=0) with self.assertRaises(ValueError): - AudiobookConverter(faster=True, language="klingon") + AudiobookConverter(backend=tts.BACKEND_AUDIOCPP, language="klingon") - def _faster_converter(self, faster_voice=None): + def _faster_converter(self, voice=None): with patch("converter.converter.FasterTTSClient"): return AudiobookConverter(voice_mode=tts.VOICE_MODE_CLONE, - faster=True, faster_voice=faster_voice) + backend=tts.BACKEND_FASTER, voice=voice) + + def _audiocpp_converter(self, voice=None): + with patch("converter.converter.AudioCppTTSClient"): + return AudiobookConverter( + voice_mode=tts.VOICE_MODE_CLONE if voice else tts.VOICE_MODE_CUSTOM, + backend=tts.BACKEND_AUDIOCPP, voice=voice) def test_narrator_tag_uses_faster_voice_name(self): - converter = self._faster_converter(faster_voice="male_richard_poe") + converter = self._faster_converter(voice="male_richard_poe") self.assertEqual(converter._narrator_tag(), "male_richard_poe") def test_narrator_tag_falls_back_to_config_voice(self): converter = self._faster_converter() self.assertEqual(converter._narrator_tag(), config.FASTER_VOICE) + def test_narrator_tag_audiocpp_uses_voice_name(self): + converter = self._audiocpp_converter(voice="female_narrator") + self.assertEqual(converter._narrator_tag(), "female_narrator") + + def test_narrator_tag_audiocpp_falls_back_to_speaker(self): + converter = self._audiocpp_converter() + self.assertEqual(converter._narrator_tag(), "Vivian") + def test_banner_and_narrator_work_without_reference_audio(self): - converter = self._faster_converter(faster_voice="male_richard_poe") + converter = self._faster_converter(voice="male_richard_poe") converter._print_banner() # must not raise (regression: Path(None)) self.assertIsNone(converter.voice_clone_ref_audio) + def test_audiocpp_banner_prints_without_reference_audio(self): + converter = self._audiocpp_converter(voice="narrator") + converter._print_banner() # must not raise + converter = self._audiocpp_converter() + converter._print_banner() + def test_non_faster_narrator_tag_unchanged(self): with tempfile.TemporaryDirectory() as tmp: ref = Path(tmp) / "ref.wav" -- cgit v1.2.3