diff options
Diffstat (limited to 'app/backends')
| -rwxr-xr-x | app/backends/audiocpp.py | 295 |
1 files changed, 114 insertions, 181 deletions
diff --git a/app/backends/audiocpp.py b/app/backends/audiocpp.py index f6121ea..4bcf8fa 100755 --- a/app/backends/audiocpp.py +++ b/app/backends/audiocpp.py @@ -17,7 +17,7 @@ catalog works without further changes. Usage: python app/backends/audiocpp.py [--wavs WAV_DIR] [--output PATH] - [--audiocpp-dir PATH] [--clone] [--families FAM1,FAM2] + [--clone] [--families FAM1,FAM2] [--all-packages] [--host HOST] [--port PORT] [--build-backend {cuda,vulkan,hip,cpu}] [--backend {cuda,vulkan,hip,cpu}] [--whisper-model NAME] [--force] @@ -36,6 +36,8 @@ longer selected. """ import argparse +import contextlib +import io import json import os import re @@ -68,9 +70,9 @@ from backends.common import ( VOICES_DIR, detect_wav_dir, find_wav_files, - normalize_dir_arg, read_prompt_text, resolve_wav_dir_arg, + url_with_port, write_prompt_text, ) from backends.common import ( @@ -91,7 +93,7 @@ BACKENDS = ("cuda", "vulkan", "hip", "cpu") TASK_TTS = "tts" TASK_VDES = "vdes" -# audio.cpp is cloned into a sibling directory of the audiobook generator. +# audio.cpp is cloned into the app directory of the audiobook generator. AUDIOCPP_DIR_NAME = "audio.cpp" AUDIOCPP_GIT_URL = "https://github.com/0xShug0/audio.cpp" @@ -144,21 +146,6 @@ def _interactive() -> bool: return False -def _resolve_audiocpp_root(directory: Path) -> Optional[Path]: - """Return the audio.cpp checkout root for DIRECTORY, or None. - - Accepts either the checkout root itself (it must contain a - ``model_specs`` directory) or the ``model_specs`` directory inside - it (the parent is used), so the file browser cannot pick the wrong - one of the two. - """ - if (directory / "model_specs").is_dir(): - return directory - if directory.name == "model_specs" and directory.is_dir(): - return directory.parent - return None - - # Backend display order, with short descriptions. The backend name is padded # so the descriptions' dashes line up in the menu. _BACKEND_DESCRIPTIONS = ( @@ -199,36 +186,26 @@ def config_port() -> int: return FALLBACK_PORT -def _url_with_port(url: str, port: int) -> str: - parts = urllib.parse.urlsplit(url) - host = parts.hostname or "127.0.0.1" - return urllib.parse.urlunsplit( - (parts.scheme or "http", f"{host}:{port}", parts.path, "", "")) - - def update_config_api_url_port(port: int, config_path: Optional[Path] = None) -> bool: """Rewrite the port inside AUDIOCPP_API_URL in app/converter/config.py. - Only the quoted URL literal is replaced; surrounding lines and the - trailing comment are preserved. Returns True when the file was changed. + Reads the configured URL from the file (not from the imported module, + which a long hub session can leave behind), swaps its port for PORT, + and writes it back through ``common.update_config_value`` so the + imported module mirrors the change immediately. Returns True when the + file now holds the new URL. """ path = Path(config_path) if config_path is not None else CONFIG_PATH try: text = path.read_text(encoding="utf-8") except OSError: return False - match = re.search(r'(?m)^(\s*AUDIOCPP_API_URL\s*=\s*")([^"]*)(")', text) + match = re.search(r'(?m)^\s*AUDIOCPP_API_URL\s*=\s*"([^"]*)"', text) if not match: return False - new_url = _url_with_port(match.group(2), port) - if new_url == match.group(2): - return False - text = text[:match.start(2)] + new_url + text[match.end(2):] - try: - path.write_text(text, encoding="utf-8") - except OSError: - return False - return True + return common.update_config_value("AUDIOCPP_API_URL", + url_with_port(match.group(1), port), + config_path=path) def update_server_config_port(port: int) -> bool: @@ -269,57 +246,18 @@ def update_config_model_ids(model_id: str, config_path: Optional[Path] = None) -> bool: """Rewrite AUDIOCPP_MODEL_ID (and AUDIOCPP_CLONE_MODEL_ID when given). - Only the quoted id literals are replaced; surrounding lines and - comments are preserved. Returns True when the file was changed. + Goes through ``common.update_config_value`` so the imported config + module mirrors the change immediately. Returns True when every named + key now holds its value in the file. """ path = Path(config_path) if config_path is not None else CONFIG_PATH - try: - text = path.read_text(encoding="utf-8") - except OSError: - return False - updates: List[Tuple[str, str]] = [("AUDIOCPP_MODEL_ID", model_id)] + ok = common.update_config_value("AUDIOCPP_MODEL_ID", model_id, + config_path=path) if clone_model_id is not None: - updates.append(("AUDIOCPP_CLONE_MODEL_ID", clone_model_id)) - changed = False - for name, value in updates: - match = re.search(r'(?m)^(\s*' + name + r'\s*=\s*")([^"]*)(")', text) - if match and match.group(2) != value: - text = text[:match.start(2)] + value + text[match.end(2):] - changed = True - if not changed: - return False - try: - path.write_text(text, encoding="utf-8") - except OSError: - return False - return True - - -def detect_audiocpp_dir() -> Optional[Path]: - """Best-effort location of a local audio.cpp checkout with model_specs. - - Checks the AUDIOCPP_DIR environment variable, then ``app/audio.cpp`` in - the tts-audiobook-generator root, then an ``audio.cpp`` directory in or - above the current working directory. Returns the path only when it - contains a ``model_specs`` directory. - """ - candidates: List[Path] = [] - env_dir = os.environ.get("AUDIOCPP_DIR") - if env_dir: - candidates.append(Path(os.path.expanduser(env_dir))) - candidates.append(APP_DIR / AUDIOCPP_DIR_NAME) - cwd = Path.cwd() - candidates.append(cwd / "audio.cpp") - candidates.append(cwd.parent / "audio.cpp") - candidates.append(cwd.parent.parent / "audio.cpp") - for candidate in candidates: - try: - resolved = candidate.resolve() - except OSError: - continue - if (resolved / "model_specs").is_dir(): - return resolved - return None + ok = common.update_config_value("AUDIOCPP_CLONE_MODEL_ID", + clone_model_id, + config_path=path) and ok + return ok # audio.cpp build directories are named ``<platform>-<backend>-<type>`` (e.g. @@ -398,8 +336,8 @@ def load_model_catalog(audiocpp_dir: Path) -> List[dict]: specs_dir = audiocpp_dir / "model_specs" if not specs_dir.is_dir(): raise NotADirectoryError( - f"{audiocpp_dir} has no model_specs/ directory; point " - "--audiocpp-dir at an audio.cpp checkout") + f"{audiocpp_dir} has no model_specs/ directory; re-run setup " + "to refresh the audio.cpp checkout") entries: List[dict] = [] for spec_path in sorted(specs_dir.glob("*.json")): try: @@ -604,21 +542,17 @@ def _decide_transcription(wav_files: list, existing: Dict[str, str], return {"mode": mode, "missing": missing, "existing": existing} -def _transcribe(args: argparse.Namespace, include_clone: bool, - plan: dict, cancel=None) -> Tuple[Dict[str, str], bool]: +def _transcribe(args: argparse.Namespace, plan: Optional[dict], + cancel=None) -> Tuple[Dict[str, str], bool]: """Transcribe the wav directory into a stem -> transcript mapping. Returns the mapping and a flag indicating whether it should be written to prompt_text (False when an existing, complete prompt_text is kept as-is). PLAN is always pre-collected — by the TUI (via _decide_transcription and its confirm callbacks) or by _flag_plan for a non-interactive run — so no - questions are asked here. CANCEL is checked between files. + questions are asked here; a None PLAN defaults to "transcribe everything". + CANCEL is checked between files. """ - if not include_clone: - print(f"[WARNING] Ignoring {args.input_dir}: no clone-capable family " - "selected, so voice presets are not used") - return {}, False - wav_files = find_wav_files(args.input_dir) if not wav_files: print(f"[WARNING] No .wav files found in {args.input_dir}; writing the " @@ -626,9 +560,10 @@ def _transcribe(args: argparse.Namespace, include_clone: bool, return {}, False prompt_path = args.input_dir / PROMPT_TEXT_FILENAME - existing = plan.get("existing") or {} if plan else {} + existing = dict((plan or {}).get("existing") or {}) + mode = plan["mode"] if plan else "all" - if plan["mode"] == "keep": + if mode == "keep": print(f"[INFO] Kept existing {prompt_path}; all voices were " "already transcribed, nothing new to transcribe") return existing, False @@ -778,7 +713,7 @@ def _write_and_advise(audiocpp_dir: Path, wav_dir: Optional[Path], def _install_models(audiocpp_dir: Path, install_guidance: List[Tuple[str, str]], - download: bool, emit=None, cancel=None) -> None: + download: bool, emit=None, cancel=None) -> int: """Print and optionally run the model install commands. One ``python <manager> install <id>`` command per hosted model (de-duped @@ -792,6 +727,9 @@ def _install_models(audiocpp_dir: Path, to EMIT and — when the checkout's ``model_manager_v2.py`` supports it — runs with ``--progress --cancel-file`` so the view can show a real byte progress bar and cancel gracefully. CANCEL aborts a running download. + + Returns 0 when every command succeeded (or nothing needed running), + 130 when cancelled, 1 when any download failed. """ manager = audiocpp_dir / "tools" / "model_manager_v2.py" seen: Set[str] = set() @@ -808,6 +746,7 @@ def _install_models(audiocpp_dir: Path, "instead of running them") download = False + failed = False for install_id in install_ids: command = f"python {manager} install {install_id}" if not download: @@ -838,10 +777,14 @@ def _install_models(audiocpp_dir: Path, cancel_file.unlink() except OSError: pass + if rc == 130 or (cancel is not None and cancel.is_set()): + return 130 if rc != 0: + failed = True print(f"[WARNING] install {install_id} exited with code " f"{rc}; the model may need to be downloaded " "by hand") + return 1 if failed else 0 def _manager_supports_progress(manager: Path) -> bool: @@ -932,17 +875,8 @@ def _wizard(stdscr, args: argparse.Namespace, parser: argparse.ArgumentParser return result def resolve_checkout(audiocpp_dir: Path) -> None: - """Validate AUDIOCPP_DIR and populate the wizard state ``s``.""" + """Validate the audio.cpp checkout and populate the wizard state ``s``.""" audiocpp_dir = Path(audiocpp_dir).resolve() - if not audiocpp_dir.is_dir(): - raise _TuiError(f"audio.cpp checkout not found: " - f"{audiocpp_dir}") - root = _resolve_audiocpp_root(audiocpp_dir) - if root is None: - raise _TuiError( - f"{audiocpp_dir} has no model_specs/ directory; " - "select the root of your audio.cpp checkout") - audiocpp_dir = root try: catalog = load_model_catalog(audiocpp_dir) except NotADirectoryError as exc: @@ -1336,9 +1270,7 @@ def _wizard(stdscr, args: argparse.Namespace, parser: argparse.ArgumentParser # checkout exists, clone it into ./app/audio.cpp (streaming inside the # TUI task view, not by dropping to the console) without asking, then # continue the same way. - audiocpp_dir = args.audiocpp_dir - if audiocpp_dir is None: - audiocpp_dir = find_local_checkout() + audiocpp_dir = find_local_checkout() if audiocpp_dir is None: target = APP_DIR / AUDIOCPP_DIR_NAME rc = taskview.run_steps(stdscr, "Clone audio.cpp", [ @@ -1438,7 +1370,7 @@ def _model_path_present(path: Path) -> bool: def _all_models_present(audiocpp_dir: Path, model_entries: List[dict]) -> bool: """True when every selected model entry's path already holds files on disk. - Paths resolve against AUDIOCPP_DIR (where model_manager_v2.py installs + Paths resolve against the checkout (where model_manager_v2.py installs them), honoring absolute paths. Used by the wizard to skip the "Automatically download the selected models" prompt when nothing is actually missing. An empty selection is treated as not-present. @@ -1571,17 +1503,18 @@ def model_install_hints(audiocpp_dir: Path, def install_models(audiocpp_dir: Path, guidance: List[Tuple[str, str]], - emit=None, cancel=None) -> None: + emit=None, cancel=None) -> int: """Download the (display name, install id) models via the helper script. Runs ``model_manager_v2.py install`` for each de-duped install id in the checkout, streaming to the console (or to EMIT, the in-TUI task view); a - failing install is reported as a warning and does not abort the rest. Used - by the hub's "Download Missing Models" action (see + failing install is reported as a warning and does not abort the rest. + Returns 0 when every download succeeded, 130 when cancelled, 1 when any + failed. Used by the hub's "Download Missing Models" action (see ``missing_model_install_guidance`` for the mapping). """ - _install_models(audiocpp_dir, guidance, download=True, - emit=emit, cancel=cancel) + return _install_models(audiocpp_dir, guidance, download=True, + emit=emit, cancel=cancel) def hand_install_guidance(audiocpp_dir: Path, @@ -1590,7 +1523,7 @@ def hand_install_guidance(audiocpp_dir: Path, Returns a multi-line message listing each missing model's id and the path its files must be placed in (``rel``, resolved against the - AUDIOCPP_DIR checkout). Used when the missing models cannot be mapped to + checkout). Used when the missing models cannot be mapped to a ``model_manager_v2.py install`` command, so the user still knows what to download and where to put it. """ @@ -1653,10 +1586,10 @@ def delete_model_files(server_json: Path, entries: List[dict]) -> int: def uninstall(*, emit=None, cancel=None) -> int: """Remove the audio.cpp backend entirely: stop its server, delete the checkout. - The checkout (``app/audio.cpp``, or wherever ``find_local_checkout`` - resolves it) holds the built binary, the downloaded models, and the - server.json, so removing the directory uninstalls the backend. A running - server this tool started is stopped first (best-effort). + The checkout (``app/audio.cpp``) holds the built binary, the downloaded + models, and the server.json, so removing the directory uninstalls the + backend. A running server this tool started is stopped first + (best-effort). EMIT is accepted for registry symmetry with the other backends but is unused here — this uninstall has no subprocess phase, and its prints are @@ -1666,7 +1599,11 @@ def uninstall(*, emit=None, cancel=None) -> int: completes and the uninstall never tears halfway. Returns the exit code (130 when cancelled before a remaining phase). """ - servers.stop("audiocpp") + # Only stop when a pid file exists: without one this tool never + # started the server, so the "not started by this tool" notice would + # be uninstall-time noise. + if servers.pid_for("audiocpp") is not None: + servers.stop("audiocpp") if common.cancel_requested(cancel): return 130 checkout = find_local_checkout() @@ -1680,29 +1617,17 @@ def uninstall(*, emit=None, cancel=None) -> int: def find_local_checkout() -> Optional[Path]: - """Best-effort location of an audio.cpp checkout with model_specs. + """Return the managed audio.cpp checkout at ``app/audio.cpp``. - Checks the AUDIOCPP_DIR environment variable, then ``app/audio.cpp`` - inside the tts-audiobook-generator root, then an ``audio.cpp`` directory - in or above the current working directory. Returns the path only when it - contains a ``model_specs`` directory. + Returns the path only when it contains a ``model_specs`` directory; + the checkout is installed there by the setup wizard and nowhere else. """ - candidates: List[Path] = [] - env_dir = os.environ.get("AUDIOCPP_DIR") - if env_dir: - candidates.append(Path(os.path.expanduser(env_dir))) - candidates.append(APP_DIR / AUDIOCPP_DIR_NAME) - cwd = Path.cwd() - candidates.append(cwd / AUDIOCPP_DIR_NAME) - candidates.append(cwd.parent / AUDIOCPP_DIR_NAME) - candidates.append(cwd.parent.parent / AUDIOCPP_DIR_NAME) - for candidate in candidates: - try: - resolved = candidate.resolve() - except OSError: - continue - if (resolved / "model_specs").is_dir(): - return resolved + try: + resolved = (APP_DIR / AUDIOCPP_DIR_NAME).resolve() + except OSError: + return None + if (resolved / "model_specs").is_dir(): + return resolved return None @@ -1886,6 +1811,8 @@ def apply_ggml_patches(audiocpp_dir: Path, *, emit=None, cancel=None) -> int: "--whitespace=nowarn", str(patch_path)] check_rc = common.run_console_subprocess( check_argv, emit=emit, cancel=cancel) + if check_rc == 130 or (cancel is not None and cancel.is_set()): + return 130 if check_rc != 0: print(f"[ERROR] {patch['file']}: no longer applies to " f"{patch['target']} (git apply --check exit {check_rc}). " @@ -1898,6 +1825,8 @@ def apply_ggml_patches(audiocpp_dir: Path, *, emit=None, cancel=None) -> int: "--whitespace=nowarn", str(patch_path)] rc = common.run_console_subprocess( apply_argv, emit=emit, cancel=cancel) + if rc == 130 or (cancel is not None and cancel.is_set()): + return 130 if rc != 0: print(f"[ERROR] {patch['file']}: git apply failed (exit {rc})") return rc @@ -1930,20 +1859,19 @@ def build_audiocpp(audiocpp_dir: Path, backend: str, *, if emit is not None: common.record_post_tui_notice(message) return 1 - patch_rc = apply_ggml_patches(audiocpp_dir, emit=emit, cancel=cancel) - if patch_rc != 0: - message = ("[ERROR] ggml build patches could not be applied; " - "aborting audiocpp_server build. See the messages above " - "and re-evaluate app/backends/patches/.") - print(message) - if emit is not None: - common.record_post_tui_notice(message) - return patch_rc argv = ["sh", str(script), "--backend", backend, "--target", "audiocpp_server", "--deployment-build"] command = f"cd {audiocpp_dir} && {shlex.join(argv)}" if emit is None: print(f"[INFO] Building audiocpp_server for {backend} ({command})...") + patch_rc = apply_ggml_patches(audiocpp_dir, cancel=cancel) + if patch_rc == 130 or (cancel is not None and cancel.is_set()): + return 130 + if patch_rc != 0: + print("[ERROR] ggml build patches could not be applied; " + "aborting audiocpp_server build. See the messages above " + "and re-evaluate app/backends/patches/.") + return patch_rc return common.run_console_subprocess(argv, cwd=audiocpp_dir) return _build_audiocpp_tui(emit, cancel, argv, command, audiocpp_dir) @@ -1952,12 +1880,13 @@ def _build_audiocpp_tui(emit, cancel, argv: List[str], command: str, audiocpp_dir: Path) -> int: """Run the build on the TUI path: tee output to a log file. - Every emitted line is also written (and flushed) to + The ggml patch step runs first, inside the same log: every emitted + line (patch status, build output) is also written (and flushed) to ``app/logs/audiocpp_build_<timestamp>.log``. On failure a summary (the - copy-pastable COMMAND and the log path) is emitted into the TUI, written - to the log, and queued as a post-TUI console notice. A cancelled build - (CANCEL set) is not reported as a failure, but its partial output stays - in the log file. + copy-pastable COMMAND and the log path) is emitted into the TUI, + written to the log, and queued as a post-TUI console notice. A + cancelled build (CANCEL set) is not reported as a failure, but its + partial output stays in the log file. """ log_path = common.LOG_DIR / ( f"audiocpp_build_{datetime.now():%Y%m%d_%H%M%S}.log") @@ -1969,9 +1898,29 @@ def _build_audiocpp_tui(emit, cancel, argv: List[str], command: str, log_handle.flush() emit(line) - tee(f"[INFO] Building audiocpp_server ({command})...") - rc = 0 + class _TeeWriter(io.TextIOBase): + """Route print() output from the patch step into the log too.""" + + def write(self, s: str) -> int: + for line in s.splitlines(): + if line: + tee(line) + return len(s) + try: + with contextlib.redirect_stdout(_TeeWriter()): + patch_rc = apply_ggml_patches(audiocpp_dir, emit=tee, + cancel=cancel) + if patch_rc == 130 or (cancel is not None and cancel.is_set()): + return 130 + if patch_rc != 0: + notice = ("[ERROR] ggml build patches could not be applied; " + "aborting audiocpp_server build. See the messages " + "above and re-evaluate app/backends/patches/.") + tee(notice) + common.record_post_tui_notice(notice) + return patch_rc + tee(f"[INFO] Building audiocpp_server ({command})...") rc = common.run_console_subprocess( argv, cwd=audiocpp_dir, emit=tee, cancel=cancel) if rc != 0 and (cancel is None or not cancel.is_set()): @@ -2048,7 +1997,7 @@ def _execute_lanes(settings: dict, args.input_dir = settings["wav_dir"] if settings["include_clone"] and args.input_dir is not None: transcripts, write_prompt = _transcribe( - args, True, plan=settings["plan"], cancel=cancel) + args, plan=settings["plan"], cancel=cancel) elif args.input_dir is not None: print(f"[WARNING] Ignoring {args.input_dir}: no clone-capable " "family selected, so voice presets are not used") @@ -2302,10 +2251,8 @@ def _collect_from_flags(args: argparse.Namespace, the settings dict, or None when the user declined an overwrite (the default-location fallback then also exists). """ - # Checkout: --audiocpp-dir, else a local checkout, else --clone clones one. - audiocpp_dir = args.audiocpp_dir - if audiocpp_dir is None: - audiocpp_dir = find_local_checkout() + # Checkout: ./app/audio.cpp, else --clone clones one there. + audiocpp_dir = find_local_checkout() if audiocpp_dir is None and args.clone: target = APP_DIR / AUDIOCPP_DIR_NAME rc = common.git_clone(AUDIOCPP_GIT_URL, target) @@ -2322,17 +2269,8 @@ def _collect_from_flags(args: argparse.Namespace, audiocpp_dir = target if audiocpp_dir is None: parser.error( - "An audio.cpp checkout is required. Pass --audiocpp-dir PATH, " - "or --clone to clone app/audio.cpp, or run without flags for the " - "TUI wizard.") - audiocpp_dir = Path(audiocpp_dir).resolve() - if not audiocpp_dir.is_dir(): - parser.error(f"audio.cpp checkout not found: {audiocpp_dir}") - root = _resolve_audiocpp_root(audiocpp_dir) - if root is None: - parser.error(f"{audiocpp_dir} has no model_specs/ directory; point " - "--audiocpp-dir at the root of an audio.cpp checkout") - audiocpp_dir = root + "An audio.cpp checkout is required. Pass --clone to clone " + "app/audio.cpp, or run without flags for the TUI wizard.") try: catalog = load_model_catalog(audiocpp_dir) except NotADirectoryError as exc: @@ -2465,11 +2403,6 @@ def build_parser() -> argparse.ArgumentParser: "server.json inside the audio.cpp checkout; an " "existing file is overwritten only with --force " "or a TUI confirm)") - parser.add_argument("--audiocpp-dir", type=normalize_dir_arg, default=None, - help="Path to a local audio.cpp checkout containing a " - "model_specs/ directory (default: detected from " - "AUDIOCPP_DIR or ./app/audio.cpp; in the TUI you can " - "clone one instead)") parser.add_argument("--clone", action="store_true", help="Non-interactive: clone audio.cpp into " "./app/audio.cpp when no checkout is found") |