From 6cfcd564c0684c52618235e6366f4a81c02b9a5b Mon Sep 17 00:00:00 2001 From: historia Date: Tue, 1 Sep 2026 14:32:05 -0400 Subject: slop refactor/dedup --- app/backends/audiocpp/catalog.py | 16 +- app/backends/audiocpp/configsync.py | 17 +- app/backends/audiocpp/models.py | 90 +++---- app/backends/audiocpp/remote.py | 1 + app/backends/audiocpp/status.py | 34 ++- app/backends/audiocpp/wizard.py | 2 - app/backends/common.py | 67 ++++-- app/backends/faster.py | 31 +-- app/backends/probe.py | 21 ++ app/backends/qwen.py | 24 +- app/backends/servers.py | 133 +++++++++-- app/converter/audio.py | 8 - app/converter/clients/__init__.py | 8 +- app/converter/clients/audiocpp.py | 442 ++++++++--------------------------- app/converter/clients/base.py | 9 +- app/converter/clients/faster.py | 10 +- app/converter/clients/qwen.py | 4 +- app/converter/converter.py | 67 ++++-- app/converter/cover.py | 8 +- app/converter/extractors.py | 117 ++++++++-- app/tests/cover_test.png | Bin 6801 -> 0 bytes app/tests/gen_test_cover.py | 8 - app/tests/test_audio.py | 18 +- app/tests/test_audiobook_cli.py | 5 + app/tests/test_backends_audiocpp.py | 74 +++++- app/tests/test_backends_common.py | 28 +++ app/tests/test_backends_servers.py | 41 +++- app/tests/test_converter.py | 6 + app/tests/test_converter_progress.py | 18 ++ app/tests/test_cover.py | 30 ++- app/tests/test_extractors.py | 80 +++++++ app/tests/test_runview.py | 11 + app/tests/test_taskview.py | 6 +- app/tests/test_tts.py | 279 ++++++---------------- app/tests/test_viewkit.py | 9 + app/ui/hub.py | 191 +++++++++------ app/ui/runview.py | 55 ++--- app/ui/taskview.py | 85 ++----- app/ui/tui.py | 29 ++- app/ui/viewkit.py | 86 +++++-- audiobook.py | 11 +- 41 files changed, 1148 insertions(+), 1031 deletions(-) delete mode 100644 app/tests/cover_test.png delete mode 100644 app/tests/gen_test_cover.py diff --git a/app/backends/audiocpp/catalog.py b/app/backends/audiocpp/catalog.py index 1048185..048e325 100644 --- a/app/backends/audiocpp/catalog.py +++ b/app/backends/audiocpp/catalog.py @@ -44,6 +44,8 @@ def request_options_families(audiocpp_dir: Path) -> Dict[str, dict]: spec = json.loads(spec_path.read_text(encoding="utf-8")) except (OSError, ValueError): continue + if not isinstance(spec, dict): + continue options = spec.get("options") request = options.get("request") if isinstance(options, dict) else None if not isinstance(request, list) or not request: @@ -161,12 +163,12 @@ def _default_package(packages: List[dict]) -> Optional[dict]: if not packages: return None for package in packages: - if package.get("default"): + if isinstance(package, dict) and package.get("default"): return package for package in packages: - if package.get("format") == "gguf": + if isinstance(package, dict) and package.get("format") == "gguf": return package - return packages[0] + return packages[0] if isinstance(packages[0], dict) else None def spec_gguf_rooted(spec: dict) -> bool: @@ -292,6 +294,11 @@ def load_model_catalog(audiocpp_dir: Path) -> List[dict]: spec = json.loads(spec_path.read_text(encoding="utf-8")) except (OSError, ValueError): continue + if not isinstance(spec, dict): + # Valid JSON that is not an object (a list, a string, a + # number): skip it like an unparsable one instead of crashing + # the wizard on a half-written spec file. + continue sanitize_model_spec(spec) tasks = spec.get("tasks") or [] if tasks: @@ -305,7 +312,8 @@ def load_model_catalog(audiocpp_dir: Path) -> List[dict]: # No task list: fall back to the category as before. continue family = spec.get("family") or spec_path.stem - packages = spec.get("packages") or [] + packages = [package for package in (spec.get("packages") or []) + if isinstance(package, dict)] package = _default_package(packages) if package is None: # No installable package: skip (cannot be hosted from a path). diff --git a/app/backends/audiocpp/configsync.py b/app/backends/audiocpp/configsync.py index bce8142..0a0e5ad 100644 --- a/app/backends/audiocpp/configsync.py +++ b/app/backends/audiocpp/configsync.py @@ -14,10 +14,7 @@ from .constants import FALLBACK_PORT def config_port() -> int: """Return the port of AUDIOCPP_API_URL in app/converter/config.py.""" - try: - return urllib.parse.urlsplit(config.AUDIOCPP_API_URL).port or FALLBACK_PORT - except ValueError: - return FALLBACK_PORT + return common.port_of(config.AUDIOCPP_API_URL, FALLBACK_PORT) def update_config_api_url_port(port: int, config_path: Optional[Path] = None) -> bool: @@ -75,18 +72,6 @@ def update_server_config_port(port: int) -> bool: return True -def _apply_port_sync(port: int, accepted: bool) -> None: - """Write the port into app/converter/config.py, or report when declined.""" - if accepted: - if not update_config_api_url_port(port): - print(f"[WARNING] Could not update {CONFIG_PATH}; edit " - "AUDIOCPP_API_URL by hand so audiobook.py uses the " - "new port") - else: - print("[WARNING] Left AUDIOCPP_API_URL unchanged; audiobook.py " - f"will still use port {config_port()}") - - def update_server_backend(backend: str) -> bool: """Rewrite the 'backend' in the checkout's server.json, or True when none. diff --git a/app/backends/audiocpp/models.py b/app/backends/audiocpp/models.py index 16c61e9..d150f62 100644 --- a/app/backends/audiocpp/models.py +++ b/app/backends/audiocpp/models.py @@ -16,10 +16,6 @@ from . import catalog as _catalog # runner kills it and reports exit 124 (see run_console_subprocess). DOWNLOAD_STALL_TIMEOUT = 300 -# Spec sanitizing lives with the catalog (the wizard's catalog view applies -# the same in-memory repair; the download path materializes it through its -# sanitized specs copy). The alias keeps this module's historical name. -_sanitize_model_spec = _catalog.sanitize_model_spec def _installed_display_names(audiocpp_dir: Path, model_entries: Optional[List[dict]], @@ -255,52 +251,21 @@ def _manager_supports_progress(manager: Path) -> bool: and _manager_supports_flag(manager, "--cancel-file")) -def _sanitize_model_spec(spec: dict) -> bool: - """Repair dot ``strip_prefix`` packages in SPEC, in place. - - A package's ``strip_prefix`` is stripped from the front of every file - path to get the local layout, so it only works when every file is - listed under that prefix (``/``). A dot prefix ("." or - "./") is meant for files written ``./``; when the package instead - lists repo-root files bare (``model.gguf``), the manager rejects the - whole package ("file path does not start with strip_prefix '.': ...") - and nothing can be downloaded. Root-level files need no prefix at all - (upstream specs like minimax_music3.json store ""), so dropping the dot - prefix is the safe repair. Prefixes naming a real directory are left - alone — the correct remote paths cannot be guessed. Returns True when - SPEC changed. - """ - changed = False - for package in spec.get("packages") or []: - if not isinstance(package, dict): - continue - prefix = str(package.get("strip_prefix") or "").rstrip("/") - if prefix not in (".", ".."): - continue - files = package.get("files") - if not isinstance(files, list) or not files: - continue - if all(isinstance(item, str) and item.startswith(prefix + "/") - for item in files): - continue - package["strip_prefix"] = "" - changed = True - return changed - - def _prepare_specs_dir(audiocpp_dir: Path) -> Optional[Path]: - """Return a temp specs dir with dot ``strip_prefix`` entries repaired. + """Return a temp specs dir with broken ``strip_prefix`` entries repaired. audio.cpp's model manager accepts ``--specs-dir``, so a checkout whose specs carry a broken ``strip_prefix`` can be installed from a sanitized copy without modifying the checkout. Every ``model_specs/*.json`` is - copied; the ones needing a repair are rewritten via - ``_sanitize_model_spec`` (specs that fail to parse are copied verbatim - so the manager reports them exactly as it would upstream). Returns None - when no spec needed a repair (or the specs directory is missing or - unreadable) — the caller then runs against the checkout's own specs. - The caller owns the returned directory and removes it when the installs - are done. + copied; the ones needing a repair are rewritten via the catalog's + ``sanitize_model_spec`` (both upstream spec bug classes: dot prefixes + and missing prefixes on $gguf-rooted single-GGUF packages — glm_tts/ + outetts shipped like that). Specs that fail to parse are copied + verbatim so the manager reports them exactly as it would upstream. + Returns None when no spec needed a repair (or the specs directory is + missing or unreadable) — the caller then runs against the checkout's + own specs. The caller owns the returned directory and removes it when + the installs are done. """ specs_dir = audiocpp_dir / "model_specs" try: @@ -321,7 +286,7 @@ def _prepare_specs_dir(audiocpp_dir: Path) -> Optional[Path]: except ValueError: payloads.append((spec_path.name, text)) continue - if isinstance(spec, dict) and _sanitize_model_spec(spec): + if isinstance(spec, dict) and _catalog.sanitize_model_spec(spec): sanitized = True text = json.dumps(spec, indent=2, ensure_ascii=False) + "\n" payloads.append((spec_path.name, text)) @@ -707,12 +672,15 @@ def delete_model_files(server_json: Path, entries: List[dict]) -> int: Each entry's ``rel`` is resolved exactly like the server resolves it (relative against ``server_json``'s directory; absolute paths honored), - then removed as a directory tree or a single file. Missing entries are - ignored. Returns the number of paths removed. Used by the wizard's + then removed as a directory tree or a single file. Paths that escape + the checkout (``..`` segments, or an absolute path outside the + checkout's tree) are refused rather than deleted — the value comes + from a user-editable server.json. Missing entries are ignored. + Returns the number of paths removed. Used by the wizard's "Delete unused models?" step — the regenerated server.json already only lists the kept models, so no entry cleanup is needed here. """ - base = server_json.parent + base = server_json.parent.resolve() removed = 0 for item in entries: rel = item.get("rel") @@ -720,16 +688,28 @@ def delete_model_files(server_json: Path, entries: List[dict]) -> int: continue path = Path(rel) if Path(rel).is_absolute() else base / rel try: - if not path.exists(): + resolved = path.resolve() + except OSError: + continue + if base not in resolved.parents and resolved != base: + print(f"[WARNING] Refusing to remove {path}: outside the " + "audio.cpp checkout") + continue + if resolved == base: + print(f"[WARNING] Refusing to remove {path}: it is the " + "checkout directory itself") + continue + try: + if not resolved.exists(): continue - if path.is_dir(): - shutil.rmtree(path, ignore_errors=True) + if resolved.is_dir(): + shutil.rmtree(resolved, ignore_errors=True) else: - path.unlink() + resolved.unlink() except OSError as exc: - print(f"[WARNING] Could not remove {path}: {exc}") + print(f"[WARNING] Could not remove {resolved}: {exc}") continue - print(f"[OK] Removed unused model {path}") + print(f"[OK] Removed unused model {resolved}") removed += 1 return removed diff --git a/app/backends/audiocpp/remote.py b/app/backends/audiocpp/remote.py index 31eddbf..20d4cef 100644 --- a/app/backends/audiocpp/remote.py +++ b/app/backends/audiocpp/remote.py @@ -1,6 +1,7 @@ """Query a running audiocpp_server for its models and voices.""" import json +import urllib.parse import urllib.request from typing import Dict, List, Optional diff --git a/app/backends/audiocpp/status.py b/app/backends/audiocpp/status.py index 9ccf8cc..a687066 100644 --- a/app/backends/audiocpp/status.py +++ b/app/backends/audiocpp/status.py @@ -54,16 +54,20 @@ def detect() -> BackendStatus: launch = format_launch_hint(specs) managed = servers.manages(specs) remote_running, remote_urls = _detect_remote(managed) + running = managed or remote_running # A more specific "part-way set up" label than unavailable/installed: - # cloned but never built, or built but not configured. + # cloned but never built, or built but not configured. Only meaningful + # while the backend is not usable (a running server makes even a + # non-built checkout usable remotely — see the BackendStatus docstring). partial = "" - if not built: - partial = "downloaded (not built)" - elif not configured: - partial = "built (not configured)" + if not running: + if not built: + partial = "downloaded (not built)" + elif not configured: + partial = "built (not configured)" return BackendStatus("audiocpp", "audio.cpp", installed=built, configured=configured, - running=managed or remote_running, + running=running, details=details, launch_hint=launch, servers=specs, managed=managed, remote=remote_running, remote_urls=remote_urls, @@ -73,18 +77,12 @@ def detect() -> BackendStatus: def _detect_remote(managed: bool = False) -> Tuple[bool, dict]: """Detect an externally-run audiocpp_server at the remote URL. - Returns ``(running, {spec_name: url})``. The remote URL is probed only - when configured (non-empty); a server answering there is ignored when it - is this tool's own managed server (remote URL == local URL and our pid is - still alive) — that instance is already reported as "[local]". + Returns ``(running, {spec_name: url})``; see probe.detect_remote_url + for the shared semantics. """ - url = (config.AUDIOCPP_REMOTE_URL or "").strip() - if not url: - return False, {} - if managed and probe.same_endpoint(url, config.AUDIOCPP_API_URL): - return False, {} - if probe.identify_server(url) == probe.IDENTITY_AUDIOCPP: - return True, {"audiocpp": url} - return False, {} + return probe.detect_remote_url(config.AUDIOCPP_REMOTE_URL, + config.AUDIOCPP_API_URL, + probe.IDENTITY_AUDIOCPP, "audiocpp", + managed) diff --git a/app/backends/audiocpp/wizard.py b/app/backends/audiocpp/wizard.py index a1a2a6a..52a4f3f 100644 --- a/app/backends/audiocpp/wizard.py +++ b/app/backends/audiocpp/wizard.py @@ -331,7 +331,6 @@ def _wizard(stdscr, args: argparse.Namespace, parser: argparse.ArgumentParser "build_mode": s.get("build_mode"), "prebuilt_forced": False, "lazy_load": True, - "sync_port": None, "wav_dir": s["wav_dir"], "plan": s["plan"], "download": s["download"], @@ -1093,7 +1092,6 @@ def _collect_from_flags(args: argparse.Namespace, "build_mode": build_mode, "prebuilt_forced": prebuilt_forced, "lazy_load": lazy_load, - "sync_port": None, "wav_dir": wav_dir, "plan": plan, "download": args.download, diff --git a/app/backends/common.py b/app/backends/common.py index d25a03e..24746b6 100644 --- a/app/backends/common.py +++ b/app/backends/common.py @@ -193,12 +193,32 @@ def wav_dir_preview(directory: Path) -> Tuple[str, str]: return ("no .wav files", "info") +def port_of(url: str, fallback: int) -> int: + """URL's explicit port, else FALLBACK (invalid URLs fall back too).""" + try: + return urllib.parse.urlsplit(url).port or fallback + except ValueError: + return fallback + + def url_with_port(url: str, port: int) -> str: - """Return URL with its port replaced/inserted as PORT.""" + """Return URL with its port replaced/inserted as PORT. + + Preserves the userinfo ("user:pass@host") and brackets IPv6 hosts + ("[::1]"), which a plain f"{host}:{port}" rebuild would mangle. + """ parts = urllib.parse.urlsplit(url) host = parts.hostname or "127.0.0.1" + if ":" in host and not host.startswith("["): + host = f"[{host}]" + netloc = f"{host}:{port}" + if parts.username: + cred = parts.username + if parts.password: + cred = f"{cred}:{parts.password}" + netloc = f"{cred}@{netloc}" return urllib.parse.urlunsplit( - (parts.scheme or "http", f"{host}:{port}", parts.path, "", "")) + (parts.scheme or "http", netloc, parts.path, "", "")) def normalize_remote_url(value: str) -> str: @@ -408,14 +428,27 @@ def run_console_subprocess(argv: List[str], cwd: Optional[Path] = None, def _reader() -> None: nonlocal last_output try: - for raw in iter(proc.stdout.readline, b""): - if not raw: + pending = "" + while True: + # read1 (not read/readline) returns whatever a single pipe + # read yields, without waiting to fill the buffer, and \r + # is treated as a line break: carriage-return progress bars + # (git clone --progress, tqdm/HuggingFace downloads) then + # surface incrementally and keep the stall watchdog fed — + # a readline-based reader blocked until the next \n would + # let a healthy download starve to the timeout. + chunk = proc.stdout.read1(4096) + if not chunk: break last_output = time.monotonic() - text = raw.decode("utf-8", errors="replace") - for line in text.splitlines(): + pending += chunk.decode("utf-8", errors="replace") + parts = re.split(r"[\r\n]+", pending) + pending = parts.pop() + for line in parts: if line: emit(line) + if pending.strip(): + emit(pending) except (OSError, ValueError): pass @@ -448,11 +481,10 @@ def run_console_subprocess(argv: List[str], cwd: Optional[Path] = None, _terminate_process_group(proc) break time.sleep(0.1) - try: - reader.join(timeout=5) - finally: - if reader.is_alive(): - reader.join(timeout=0) + # The reader is a daemon: a join timeout here only means the child + # closed its stdout but the thread is still draining — there is + # nothing useful left to wait for. + reader.join(timeout=5) if cancelled: return 130 if stalled: @@ -476,10 +508,17 @@ def _terminate_process_group(proc) -> None: """ import signal if sys.platform == "win32": + # TerminateProcess hits a single pid; children the server spawned + # would survive as orphans. taskkill /T walks and kills the whole + # process tree, then the poll loop reaps the direct child. try: - proc.terminate() - except OSError: - pass + subprocess.run(["taskkill", "/T", "/F", "/PID", str(proc.pid)], + capture_output=True, timeout=10) + except (OSError, subprocess.SubprocessError): + try: + proc.terminate() + except OSError: + pass deadline = time.time() + 10 while time.time() < deadline: if proc.poll() is not None: diff --git a/app/backends/faster.py b/app/backends/faster.py index 2d0cad3..34fae45 100755 --- a/app/backends/faster.py +++ b/app/backends/faster.py @@ -59,6 +59,8 @@ from ui import taskview, tui FASTER_DIR_NAME = "faster-qwen3-tts" FASTER_GIT_URL = "https://github.com/andimarafioti/faster-qwen3-tts" FASTER_PIP_PKG = "faster-qwen3-tts[demo]" +# The managed server's port when the configured URL names none. +DEFAULT_PORT = 8000 # The dedicated venv faster-qwen3-tts is installed into (never the app env # or the qwen backend's; the wheel pulls its own qwen-tts-hf dependency, # which ships the same qwen_tts module upstream qwen-tts does). @@ -78,14 +80,6 @@ def _is_cloned() -> bool: return (_checkout() / "examples" / "openai_server.py").is_file() -def _config_port() -> int: - import urllib.parse - try: - return urllib.parse.urlsplit(config.FASTER_API_URL).port or 8000 - except ValueError: - return 8000 - - def build_voices(wav_files: list, language: str, whisper_model: str) -> dict: """Transcribe each wav file and build the voices mapping.""" voices = {} @@ -363,7 +357,7 @@ def _print_launch_hint(voices_path: Path) -> None: return print("[INFO] Clone faster-qwen3-tts to get examples/openai_server.py,") print(f" then run it with --voices {voices_path} " - f"--port {_config_port()}") + f"--port {common.port_of(config.FASTER_API_URL, DEFAULT_PORT)}") def setup_screen(stdscr) -> int: @@ -471,7 +465,8 @@ def detect() -> BackendStatus: if cloned and voices_json.exists(): argv = [str(envs.env_python(FASTER_ENV)), str(_checkout() / "examples" / "openai_server.py"), - "--voices", str(voices_json), "--port", str(_config_port())] + "--voices", str(voices_json), "--port", + str(common.port_of(config.FASTER_API_URL, DEFAULT_PORT))] # identity: /health must report model_loaded before the server is # really usable (the model loads after the port opens). specs = [ServerSpec("faster", config.FASTER_API_URL, argv, @@ -491,17 +486,13 @@ def detect() -> BackendStatus: def _detect_remote(managed: bool = False): """Detect an externally-run faster server at the remote URL. - Returns ``(running, {spec_name: url})``; see audiocpp._detect_remote for - the shared semantics (empty URL disables, own server not counted twice). + Returns ``(running, {spec_name: url})``; see probe.detect_remote_url + for the shared semantics (empty URL disables, own server not counted + twice). """ - url = (config.FASTER_REMOTE_URL or "").strip() - if not url: - return False, {} - if managed and probe.same_endpoint(url, config.FASTER_API_URL): - return False, {} - if probe.identify_server(url) == probe.IDENTITY_FASTER: - return True, {"faster": url} - return False, {} + return probe.detect_remote_url(config.FASTER_REMOTE_URL, + config.FASTER_API_URL, + probe.IDENTITY_FASTER, "faster", managed) def update(*, emit=None, cancel=None) -> int: diff --git a/app/backends/probe.py b/app/backends/probe.py index 64dc7ab..818aadb 100644 --- a/app/backends/probe.py +++ b/app/backends/probe.py @@ -142,6 +142,27 @@ def same_endpoint(url_a: str, url_b: str) -> bool: return host_a == host_b and port_a == port_b +def detect_remote_url(remote_url: str, local_url: str, identity: str, + name: str, managed: bool = False) -> tuple: + """The shared backend "is something answering at the remote URL?" check. + + Returns ``(running, {name: url})``. An empty URL disables the check; a + remote URL equal to the configured local endpoint is ignored while + MANAGED (that server was started by this tool and is already reported + as "[local]"); otherwise the URL must answer HTTP as IDENTITY to count. + The faster and audio.cpp backends use it verbatim; qwen's is + model-aware and keeps its own variant. + """ + url = (remote_url or "").strip() + if not url: + return False, {} + if managed and same_endpoint(url, local_url): + return False, {} + if identify_server(url) == identity: + return True, {name: url} + return False, {} + + def health_payload(url: str, timeout: float = DEFAULT_TIMEOUT) -> Optional[dict]: """Return the server's ``/health`` JSON document, or None. diff --git a/app/backends/qwen.py b/app/backends/qwen.py index a94b3cc..c099c84 100644 --- a/app/backends/qwen.py +++ b/app/backends/qwen.py @@ -160,7 +160,10 @@ def delete_model_weights(models: Optional[List[str]] = None) -> int: if not directory.is_dir(): continue print(f"[INFO] Removing cached {MODEL_REPOS[name]} weights...") - shutil.rmtree(directory, ignore_errors=True) + shutil.rmtree(directory) + if directory.exists(): + print(f"[WARNING] Could not fully remove {directory}") + continue removed += 1 if removed: print(f"[OK] Deleted cached weights for {removed} " @@ -174,19 +177,6 @@ def _is_installed() -> bool: return envs.module_available("qwen_tts", QWEN_ENV) -def _config_port(url: str, fallback: int) -> int: - import urllib.parse - try: - return urllib.parse.urlsplit(url).port or fallback - except ValueError: - return fallback - - -def current_model() -> str: - """The model a fresh managed start hosts (DEFAULT_MODEL).""" - return DEFAULT_MODEL - - def model_for_identity(identity: Optional[str]) -> Optional[str]: """The model name a qwen demo answers as (None when not a known identity).""" return IDENTITY_TO_MODEL.get(identity) @@ -308,7 +298,7 @@ def build_spec(model: str) -> ServerSpec: "qwen", url, [str(envs.env_script("qwen-tts-demo", QWEN_ENV)), MODEL_REPOS[model], "--ip", "127.0.0.1", - "--port", str(_config_port(url, DEFAULT_PORT))], + "--port", str(common.port_of(url, DEFAULT_PORT))], identity=desired_identity(model)) @@ -336,12 +326,12 @@ def detect() -> BackendStatus: differs from the default one. """ installed = _is_installed() - model = current_model() + model = DEFAULT_MODEL url = config.QWEN_API_URL details: List[str] = [] details.append("pip: installed" if installed else "not installed — run setup to pip install qwen-tts") - details.append(f"port: {_config_port(url, DEFAULT_PORT)}") + details.append(f"port: {common.port_of(url, DEFAULT_PORT)}") details.append(f"default model: {model}") specs = [build_spec(model)] managed = servers.manages(specs) diff --git a/app/backends/servers.py b/app/backends/servers.py index c0f8f4d..25ab472 100644 --- a/app/backends/servers.py +++ b/app/backends/servers.py @@ -146,6 +146,43 @@ def _pid_alive(pid: int) -> bool: return True +def _process_start_token(pid: int) -> Optional[str]: + """A token that changes when the OS recycles PID (best effort). + + On Linux the token is /proc's process start time (field 22): after a + crash and pid reuse, a bare signal-0 probe would happily bless the new + unrelated process as "our server", so the recorded token no longer + matching is what breaks that identification. Platforms without an + equivalent (macOS, Windows) return None — bare-pid probing stays, as + before. + """ + if not sys.platform.startswith("linux"): + return None + try: + stat = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8") + # Fields 1/2 ("comm") may contain spaces and parentheses; starttime + # is field 22, i.e. the 20th entry after the closing paren. + fields = stat.rsplit(")", 1)[1].split() + return fields[19] + except (OSError, IndexError): + return None + + +def _pid_owned(pid: int, token: Optional[str]) -> bool: + """True when PID is alive AND still the process the token recorded. + + An empty/missing token (legacy pid files, non-Linux platforms) falls + back to the bare liveness probe; a recorded token must match the + process's current start time, so a recycled pid is reported dead. + """ + if not _pid_alive(pid): + return False + if not token: + return True + current = _process_start_token(pid) + return current is None or current == token + + def _reap_exited(pid: int) -> bool: """Reap PID when it is our exited child; True when confirmed dead. @@ -169,20 +206,23 @@ def _reap_exited(pid: int) -> bool: def _kill_pid(pid: int) -> bool: - """Terminate PID (and its process group on POSIX). Returns True when dead.""" + """Terminate PID (and its process tree). Returns True when dead.""" if sys.platform == "win32": + # TerminateProcess hits a single pid; the server may have spawned + # children that would survive as orphans. taskkill /T kills the + # whole tree (the same mechanism run_console_subprocess uses). try: - os.kill(pid, signal.SIGTERM) - except (ProcessLookupError, PermissionError, OSError): - return not _pid_alive(pid) + subprocess.run(["taskkill", "/T", "/F", "/PID", str(pid)], + capture_output=True, timeout=STOP_GRACE_SECONDS) + except (OSError, subprocess.SubprocessError): + try: + os.kill(pid, signal.SIGTERM) + except (ProcessLookupError, PermissionError, OSError): + return not _pid_alive(pid) for _ in range(int(STOP_GRACE_SECONDS * 10)): if not _pid_alive(pid): return True time.sleep(0.1) - try: - os.kill(pid, signal.SIGTERM) - except OSError: - pass return not _pid_alive(pid) # POSIX: kill the whole process group (started with start_new_session=True). try: @@ -279,6 +319,19 @@ def start(spec, progress: ProgressCallback = None, pid_file.unlink() except OSError: pass + # Reserve the slot atomically (O_EXCL): two concurrent starters can + # both pass the liveness check above, but only one wins the create — + # the loser refuses instead of spawning a duplicate server on the port. + try: + pid_handle = pid_file.open("x") + except FileExistsError: + report({"kind": "error", + "message": f"a {spec.name} server is already starting " + "(its pid file appeared while this start was " + "running); stop it first"}) + return False + except OSError: + pid_handle = None cwd = getattr(spec, "cwd", None) # Append so an earlier boot's output survives (crash-loop debugging); @@ -299,13 +352,28 @@ def start(spec, progress: ProgressCallback = None, report({"kind": "error", "message": f"could not start server: {exc}"}) log_handle.close() + if pid_handle is not None: + pid_handle.close() + try: + pid_file.unlink() + except OSError: + pass return False log_handle.write(f"\n=== boot {datetime.now():%Y-%m-%d %H:%M:%S} " f"(pid {proc.pid}) ===\n") log_handle.flush() log_handle.close() - pid_file.write_text(str(proc.pid), encoding="utf-8") + # "pid token": the process's start time where the platform provides + # one, so a recycled pid is never mistaken for our server (see + # _pid_owned). Empty token = bare-pid probing. + token = _process_start_token(proc.pid) or "" + if pid_handle is not None: + try: + pid_handle.write(f"{proc.pid} {token}\n".strip() + "\n") + pid_handle.close() + except OSError: + pass report({"kind": "starting", "name": spec.name, "argv": " ".join(str(a) for a in argv), "cwd": str(cwd) if cwd is not None else None, @@ -354,7 +422,10 @@ def stop(name: str) -> bool: Returns True when the process was terminated (or already gone). Returns False when there is no pid file — the server was not started by this tool, - so the user must stop it manually (e.g. close its terminal). + so the user must stop it manually (e.g. close its terminal). The pid + file is removed only once the kill succeeded: while the process is + still running, the record is what keeps a later ``start`` from spawning + a duplicate onto the same port. """ pid_file = _pid_path(name) if not pid_file.exists(): @@ -362,15 +433,15 @@ def stop(name: str) -> bool: "(not started by this tool — stop it manually)") return False try: - pid = int(pid_file.read_text(encoding="utf-8").strip()) - except (OSError, ValueError): + pid = int(pid_file.read_text(encoding="utf-8").split()[0]) + except (OSError, ValueError, IndexError): print(f"[WARNING] could not read pid file {pid_file}; removing it") try: pid_file.unlink() except OSError: pass return False - if not _pid_alive(pid): + if not _pid_owned_recorded(pid, pid_file): print(f"[INFO] {name} server (pid {pid}) already stopped") try: pid_file.unlink() @@ -381,13 +452,24 @@ def stop(name: str) -> bool: killed = _kill_pid(pid) if killed: print(f"[OK] {name} server stopped") + try: + pid_file.unlink() + except OSError: + pass else: - print(f"[WARNING] could not stop pid {pid}; stop it manually") + print(f"[WARNING] could not stop pid {pid}; stop it manually — the " + "pid file is kept so the server is not started twice") + return killed + + +def _pid_owned_recorded(pid: int, pid_file: Path) -> bool: + """PID alive and still the process recorded in PID_FILE (token aware).""" try: - pid_file.unlink() + fields = pid_file.read_text(encoding="utf-8").split() except OSError: - pass - return killed + return _pid_alive(pid) + token = fields[1] if len(fields) > 1 else None + return _pid_owned(pid, token) def manages(specs) -> bool: @@ -395,13 +477,15 @@ def manages(specs) -> bool: A server counts as ours when ``start`` recorded a pid file for it and that pid is still alive — the same ownership rule ``stop`` applies - before refusing ("not started by this tool"). Used by the backends' - ``detect()`` so the hub's status table can tag an up server as - "[remote]" when it was launched outside this tool. + before refusing ("not started by this tool"). Where the platform + provides a start-time token, a recycled pid no longer counts as ours. + Used by the backends' ``detect()`` so the hub's status table can tag + an up server as "[remote]" when it was launched outside this tool. """ for spec in specs: + pid_file = _pid_path(spec.name) pid = pid_for(spec.name) - if pid is not None and _pid_alive(pid): + if pid is not None and _pid_owned_recorded(pid, pid_file): return True return False @@ -412,12 +496,13 @@ def pid_for(name: str): if not pid_file.exists(): return None try: - return int(pid_file.read_text(encoding="utf-8").strip()) - except (OSError, ValueError): + return int(pid_file.read_text(encoding="utf-8").split()[0]) + except (OSError, ValueError, IndexError): return None def alive(name: str) -> bool: """True when the server named NAME was started by us and is still alive.""" + pid_file = _pid_path(name) pid = pid_for(name) - return pid is not None and _pid_alive(pid) + return pid is not None and _pid_owned_recorded(pid, pid_file) diff --git a/app/converter/audio.py b/app/converter/audio.py index 58cd5a5..9df7aff 100644 --- a/app/converter/audio.py +++ b/app/converter/audio.py @@ -46,14 +46,6 @@ def atempo_filters(speed: float) -> str: return ",".join(chain) -def speed_export_params(speed: float) -> List[str]: - """Return ffmpeg filter args for pitch-preserving speed adjustment.""" - filters = atempo_filters(speed) - if not filters: - return [] - return ["-filter:a", filters] - - def _concat_escape(path: str) -> str: """Escape a path for use inside single quotes in an ffmpeg concat list.""" return path.replace("'", "'\\''") diff --git a/app/converter/clients/__init__.py b/app/converter/clients/__init__.py index a107d29..cdb7912 100644 --- a/app/converter/clients/__init__.py +++ b/app/converter/clients/__init__.py @@ -51,8 +51,8 @@ from .audiocpp import ( audiocpp_family_voice_policy, audiocpp_request_error, audiocpp_script_input, + audiocpp_voice_for_run, allocation_log_note, - build_trimmed_voice_reference, nvidia_device_memory_report, ) @@ -83,12 +83,10 @@ __all__ = [ "AUDIOCPP_VOICE_OPTIONAL", "AUDIOCPP_VOICE_NONE", "AudioCppFamilyProfile", "AUDIOCPP_DEFAULT_FAMILY_PROFILE", "AUDIOCPP_FAMILY_PROFILES", "audiocpp_entry_voice_capability", - "audiocpp_entry_supports_design", + "audiocpp_entry_supports_design", "audiocpp_voice_for_run", "audiocpp_family_narrates", "audiocpp_family_spec_tasks", "audiocpp_family_voice_policy", "audiocpp_request_error", "audiocpp_script_input", - "allocation_log_note", "build_trimmed_voice_reference", - "nvidia_device_memory_report", "spec_request_option_names", + "allocation_log_note", "nvidia_device_memory_report", "AUDIOCPP_VOICE_REQUIRED_FAMILIES", "AUDIOCPP_ALLOCATION_FRAGMENTS", - "AUDIOCPP_REFERENCE_TRIM_SECONDS", ] diff --git a/app/converter/clients/audiocpp.py b/app/converter/clients/audiocpp.py index 3637a50..98d64ba 100644 --- a/app/converter/clients/audiocpp.py +++ b/app/converter/clients/audiocpp.py @@ -1,20 +1,15 @@ """Client for the audio.cpp audiocpp_server (native ggml TTS families).""" -import array -import base64 import json import logging import shutil -import struct import subprocess -import sys import tempfile import urllib.error import urllib.parse import urllib.request -import wave from pathlib import Path -from typing import Any, Dict, List, Optional, Set, Tuple +from typing import Any, Dict, List, Optional, Set from .. import config from ..audio import concat_audio_files @@ -22,7 +17,8 @@ from ..chunking import split_into_chunks from .base import (BaseTTSClient, ConversionCancelled, NonRetryableTTSError, resolve_request_seed) from .languages import LANGUAGE_ISO_CODES, normalize_language -from .speakers import is_builtin_speaker, speaker_display_name_for +from .speakers import QWEN3_TTS_SPEAKERS, is_builtin_speaker, \ + speaker_display_name_for logger = logging.getLogger(__name__) @@ -129,13 +125,10 @@ def _ALLOCATION_HINT_TEXT() -> str: "The server ran out of device memory while building a compute " "graph: check what else is using the GPU, and read the server's " "log (app/logs/audiocpp-server.log), which records the exact " - "allocation size it attempted. Cloning families that encode the " - "whole reference with attention over its length (MOSS-TTS-Local) " - "retry automatically with a shorter reference when the voice's " - "wav is readable locally. For DramaBox, adding \"session_options\": " - "{\"dramabox.mem_saver\": \"true\"} to its server.json model entry " - "trades speed for a much lower memory peak (restart the server " - "after editing).") + "allocation size it attempted. For DramaBox, adding " + "\"session_options\": {\"dramabox.mem_saver\": \"true\"} to its " + "server.json model entry trades speed for a much lower memory " + "peak (restart the server after editing).") # Deterministic failures whose one-line server message is not actionable @@ -224,18 +217,10 @@ AUDIOCPP_VOICE_REQUIRED = "required" # clone-only: a reference voice is mandato AUDIOCPP_VOICE_OPTIONAL = "optional" # tts + clone: blank voice means plain TTS AUDIOCPP_VOICE_NONE = "none" # pure TTS: no cloning, no voice at all -# The allocation-failure fragments that trigger the server-log detail and -# the trimmed-reference retry (see AUDIOCPP_NON_RETRYABLE_ERRORS). +# The allocation-failure fragments that trigger the server-log detail +# (see AUDIOCPP_NON_RETRYABLE_ERRORS and allocation_log_note). AUDIOCPP_ALLOCATION_FRAGMENTS = ("failed to allocate", "allocation failed") -# A cloned reference long enough to blow up reference-attention encoders -# (MOSS-TTS-Local's codec encoder: memory grows with the reference's -# square) is retried as this many seconds of the same voice, read from -# the voice's local wav and sent as a base64 voice_ref (bounded by the -# server's 5 MiB inline-reference limit). -AUDIOCPP_REFERENCE_TRIM_SECONDS = 30.0 -_AUDIOCPP_VOICE_REF_MAX_BYTES = 5 * 1024 * 1024 - # Warn about a nearly-full local GPU before the first request: with # another process holding the memory, even small graph allocations fail. _LOW_FREE_DEVICE_MIB = 4096 @@ -288,27 +273,6 @@ def audiocpp_family_spec_tasks(family: str) -> Optional[Set[str]]: return {str(task) for task in spec["tasks"]} -def spec_request_option_names(family: str) -> Set[str]: - """FAMILY's accepted request-option names from its local model spec. - - Used to decide whether an option may be attached to a request (e.g. - the reference_text carried alongside an inline voice_ref): families - whose runtime validates request options against the spec would reject - an unknown key outright. An unknown family (no local spec) yields an - empty set — the caller then omits the option rather than risking a - rejection. - """ - spec = _family_spec(family) - if not spec: - return set() - options = spec.get("options") - request = options.get("request") if isinstance(options, dict) else None - if not isinstance(request, list): - return set() - return {str(item["name"]) for item in request - if isinstance(item, dict) and item.get("name")} - - def audiocpp_entry_supports_design(family: str, task: str, model_id: str) -> bool: """Whether a server model entry can design a voice from a description. @@ -431,34 +395,51 @@ def _reference_text_error(voice: Optional[str], server_message: str) -> str: ) +def _http_error_body(exc: urllib.error.HTTPError) -> str: + """The FULL decoded HTTP error body (classified before truncation). + + Deterministic-error detection matches fragments that can sit deep in a + long server message, so the body must be read whole; only the final + user-facing text is capped (see audiocpp_request_error). + """ + try: + return exc.read().decode("utf-8", errors="replace") + except Exception: + return "" + + def audiocpp_request_error(status: int, detail: str, voice: Optional[str] = None, log_note: Optional[str] = None) -> Exception: """The exception for a failed audio.cpp speech request. - Deterministic request-configuration errors (a fragment in - AUDIOCPP_NON_RETRYABLE_ERRORS) become NonRetryableTTSError so the - chunk retry loop skips attempts that cannot succeed; clone-only - hosting errors (AUDIOCPP_CLONE_ONLY_ERRORS) also carry the re-host - hint, hinted errors (AUDIOCPP_HINTED_ERRORS) their per-fragment - guidance; everything else returns the plain RuntimeError the retry - loop has always retried. LOG_NOTE, when given for an allocation - failure, appends the server log's own record of the failed - allocation (the exact size it attempted, from the managed server's - log file) to the non-retryable message. + DETAIL is the full HTTP error body. Deterministic request- + configuration errors (a fragment in AUDIOCPP_NON_RETRYABLE_ERRORS) + become NonRetryableTTSError so the chunk retry loop skips attempts + that cannot succeed; clone-only hosting errors + (AUDIOCPP_CLONE_ONLY_ERRORS) also carry the re-host hint, hinted + errors (AUDIOCPP_HINTED_ERRORS) their per-fragment guidance; + everything else returns the plain RuntimeError the retry loop has + always retried. Classification runs on the FULL body — only the + message quoted in the final text is truncated to keep it readable. + LOG_NOTE, when given for an allocation failure, appends the server + log's own record of the failed allocation (the exact size it + attempted, from the managed server's log file) to the non-retryable + message. """ message = _server_error_message(detail) lowered = message.lower() + shown = message[:200] allocation_failure = any( fragment in lowered for fragment in AUDIOCPP_ALLOCATION_FRAGMENTS) error: Exception if _REFERENCE_TEXT_FRAGMENT in lowered: error = NonRetryableTTSError( - _reference_text_error(voice, message)) + _reference_text_error(voice, shown)) elif any(fragment in lowered for fragment in AUDIOCPP_CLONE_ONLY_ERRORS): error = NonRetryableTTSError( f"audio.cpp server returned HTTP {status} (not retryable): " - f"{message}. This model family only synthesizes by cloning a " + f"{shown}. This model family only synthesizes by cloning a " "reference voice, so its server entry must be hosted with task " '"clon" — re-run Configure Backends → audio.cpp (or edit ' "server.json) and restart the server.") @@ -469,15 +450,15 @@ def audiocpp_request_error(status: int, detail: str, if hinted is not None: error = NonRetryableTTSError( f"audio.cpp server returned HTTP {status} (not retryable): " - f"{message}. {hinted}") + f"{shown}. {hinted}") elif any(fragment in lowered for fragment in AUDIOCPP_NON_RETRYABLE_ERRORS): error = NonRetryableTTSError( f"audio.cpp server returned HTTP {status} (not retryable): " - f"{message}") + f"{shown}") else: error = RuntimeError( - f"audio.cpp server returned HTTP {status}: {detail}") + f"audio.cpp server returned HTTP {status}: {shown}") if allocation_failure and log_note \ and isinstance(error, NonRetryableTTSError): error = NonRetryableTTSError(f"{error}{log_note}") @@ -540,125 +521,18 @@ def nvidia_device_memory_report() -> Optional[str]: return result.stdout.strip() -def _pcm16_mono_samples(path: Path, max_seconds: float - ) -> Optional[Tuple[int, List[int]]]: - """The first MAX_SECONDS of a wav as (sample rate, mono PCM16 samples). - - Reads with the stdlib wave module (PCM u8/s16/s24/s32), mixes channels - by averaging, and stops at MAX_SECONDS so a long reference costs only - the frames actually sent. Returns None for files the wave module - cannot parse (float-format wavs, non-WAV files) or that carry no - samples — the trimmed-reference retry then does not fire. - """ - max_frames = 0 - if max_seconds > 0: - try: - with wave.open(str(path), "rb") as probe: - rate = probe.getframerate() - max_frames = int(max_seconds * rate) + 1 - except (OSError, EOFError, wave.Error): - return None - try: - with wave.open(str(path), "rb") as handle: - rate = handle.getframerate() - channels = handle.getnchannels() - width = handle.getsampwidth() - frames = handle.getnframes() - raw = handle.readframes(min(frames, max_frames) if max_frames - else frames) - except (OSError, EOFError, wave.Error): - return None - if rate <= 0 or channels <= 0 or width not in (1, 2, 3, 4) or not raw: - return None - frame_bytes = width * channels - frame_count = len(raw) // frame_bytes - if frame_count <= 0: - return None - if width == 2: - values = array.array("h") - values.frombytes(raw[:frame_count * frame_bytes]) - if sys.byteorder == "big": - values.byteswap() - elif width == 1: - # u8 -> s16, scaled to the full 16-bit range. - values = array.array("h", ((byte - 128) << 8 - for byte in raw[:frame_count])) - else: - # s24/s32 -> s16 by dropping the low bits (keeps every value inside - # int16 so the mixdown and the PCM16 container need no clipping). - values = array.array( - "i", (int.from_bytes(raw[i * width:i * width + width], - "little", signed=True) - for i in range(frame_count))) - shift = 8 if width == 3 else 16 - values = array.array("h", (value >> shift for value in values)) - if channels == 1: - return rate, values.tolist() - mixed: List[int] = [] - for index in range(frame_count): - start = index * channels - mixed.append(sum(values[start:start + channels]) // channels) - return rate, mixed - - -def pcm16_wav_bytes(sample_rate: int, samples: List[int]) -> bytes: - """Wrap mono PCM16 samples in a minimal RIFF/WAVE container.""" - data = array.array("h", samples) - if sys.byteorder == "big": - data.byteswap() - payload = data.tobytes() - return (b"RIFF" - + struct.pack(" Optional[Tuple[str, float, str]]: - """A base64 voice_ref of the first MAX_SECONDS of a reference wav. - - Returns (base64 wav, seconds used, file name), or None when PATH is - missing or unreadable. The payload keeps the file's native sample rate - and is mixed to mono; it is further capped so the decoded bytes stay - within the server's 5 MiB inline-reference limit (16-bit mono means - ~2.6 MB per 30 s at 44.1 kHz, comfortably inside). - """ - if path is None: - return None - decoded = _pcm16_mono_samples(path, max_seconds) - if decoded is None: - return None - rate, samples = decoded - if not samples: - return None - byte_cap_seconds = max_bytes / (2 * rate) - frames = int(min(max_seconds, byte_cap_seconds) * rate) - samples = samples[:frames] - if not samples: - return None - wav = pcm16_wav_bytes(rate, samples) - return base64.b64encode(wav).decode("ascii"), len(samples) / rate, path.name - - class AudioCppFamilyProfile: """Request conventions of one audio.cpp model family. - Language style, whether the family reads a style/instruction prompt, - and how the request text is formatted; these are family-level (every - entry of a family shares them). Whether a *specific entry* has - built-in speakers is an entry-level concern, decided by - audiocpp_entry_voice_capability, not this profile. + Language style and how the request text is formatted; these are + family-level (every entry of a family shares them). Whether a + *specific entry* has built-in speakers is an entry-level concern, + decided by audiocpp_entry_voice_capability, not this profile. """ def __init__(self, language_style: str = AUDIOCPP_LANG_OMIT, - sends_instructions: bool = False, script_prefix: Optional[str] = None): self.language_style = language_style - self.sends_instructions = sends_instructions # SCRIPT_PREFIX, when set, formats every request's text as one # ": text" script line (audiocpp_script_input): the # family's server implementation parses the prompt as a @@ -667,17 +541,15 @@ class AudioCppFamilyProfile: # Generic profile for families not listed in AUDIOCPP_FAMILY_PROFILES: -# clone-only, no style instructions, and no language field (the model -# detects the language itself). Describes higgs_audio_tts, voxcpm2, -# fish_audio, dots_tts, dramabox, omnivoice, outetts, glm_tts, miotts, -# moss_tts_*, pocket_tts, ... as well as families added to -# audio.cpp after this table was written. +# clone-only and no language field (the model detects the language +# itself). Describes higgs_audio_tts, voxcpm2, fish_audio, dots_tts, +# dramabox, omnivoice, outetts, glm_tts, miotts, moss_tts_*, pocket_tts, +# ... as well as families added to audio.cpp after this table was written. AUDIOCPP_DEFAULT_FAMILY_PROFILE = AudioCppFamilyProfile() AUDIOCPP_FAMILY_PROFILES = { AUDIOCPP_FAMILY_QWEN3_TTS: AudioCppFamilyProfile( language_style=AUDIOCPP_LANG_DISPLAY, - sends_instructions=True, ), # Families whose language option takes a code (e.g. "en") instead of # a Qwen display name; otherwise clone-only like the default profile. @@ -727,13 +599,37 @@ def audiocpp_entry_voice_capability(family: str, task: str, return AUDIOCPP_VOICE_CLONE -class AudioCppTTSClient(BaseTTSClient): - # Class-level defaults so a partially-constructed instance behaves like - # a fresh run (tests build clients via __new__; see BaseTTSClient). - _voice_ref_b64: Optional[str] = None - _voice_ref_reference_text: Optional[str] = None - _reference_trim_attempted = False +def audiocpp_voice_for_run(family: str, task: str, model_id: str, + picked: Optional[str], + entry_voices: List[str]) -> Optional[str]: + """The voice one conversion of this entry sends, given a shared pick. + + The single source of the per-model voice resolution the Generate + form's "All (multiple generation)" pick uses: the picked voice wins + wherever the model accepts it (a built-in speaker on the CustomVoice + entry, a server-side preset on a clone-capable one); models the pick + cannot serve fall back to their own default — the first built-in + speaker, or the first server voice — or to no voice at all (design + entries take the Instructions text instead; pure-TTS families take + no voice). Pure, so the form can call it per configured entry. + """ + capability = audiocpp_entry_voice_capability(family, task, model_id) + if capability == AUDIOCPP_VOICE_DESIGN: + return None + if capability == AUDIOCPP_VOICE_SPEAKER: + if picked and picked in QWEN3_TTS_SPEAKERS: + return picked + return QWEN3_TTS_SPEAKERS[0] + # Clone capability; the family policy decides whether a voice + # exists at all. + if audiocpp_family_voice_policy(family) == AUDIOCPP_VOICE_NONE: + return None + if picked and picked in entry_voices: + return picked + return entry_voices[0] if entry_voices else None + +class AudioCppTTSClient(BaseTTSClient): """Generates audio chunks through an audio.cpp audiocpp_server. Talks to the OpenAI-style HTTP API of audiocpp_server, which hosts TTS @@ -801,8 +697,9 @@ class AudioCppTTSClient(BaseTTSClient): instructions: Optional[str] = None, request_options: Optional[Dict[str, str]] = None, quiet: bool = False, - unload_models: Optional[bool] = None): - super().__init__(chunks_dir, quiet=quiet) + unload_models: Optional[bool] = None, + cancel=None): + super().__init__(chunks_dir, quiet=quiet, cancel=cancel) self.api_url = (api_url or config.AUDIOCPP_API_URL).rstrip("/") # Per-run model selection: the --model CLI flag (or the Generate # form's Model pick). An empty value is resolved at connect time @@ -850,15 +747,6 @@ class AudioCppTTSClient(BaseTTSClient): self.design_mode = False self.instruction_voice = False self.plain_mode = False - # Trimmed-reference retry state (see _switch_to_trimmed_reference): - # an inline base64 voice_ref that replaces the voice name after an - # allocation failure, the transcript carried alongside it, and the - # one-attempt guard. The class-level defaults above keep partially - # constructed instances (tests via __new__) behaving like a fresh - # run; the assignments here shadow them for this instance. - self._voice_ref_b64 = None - self._voice_ref_reference_text = None - self._reference_trim_attempted = False # Family and task of the selected model entry and the family's request # profile; all are resolved from GET /v1/models during _connect. self.family = "" @@ -1058,13 +946,10 @@ class AudioCppTTSClient(BaseTTSClient): with urllib.request.urlopen(url, timeout=timeout) as response: return json.loads(response.read().decode("utf-8")) except urllib.error.HTTPError as exc: - detail = "" - try: - detail = exc.read().decode("utf-8", errors="replace")[:200] - except Exception: - pass + detail = _http_error_body(exc) raise RuntimeError( - f"audio.cpp server returned HTTP {exc.code} for {path}: {detail}") from exc + f"audio.cpp server returned HTTP {exc.code} for {path}: " + f"{detail[:200]}") from exc except urllib.error.URLError as exc: raise RuntimeError(f"audio.cpp request failed for {path}: {exc.reason}") from exc @@ -1232,7 +1117,7 @@ class AudioCppTTSClient(BaseTTSClient): ) # ------------------------------------------------------------------ - # Reference trimming and device diagnostics + # Device diagnostics # ------------------------------------------------------------------ def _warn_low_device_memory(self) -> None: @@ -1268,131 +1153,6 @@ class AudioCppTTSClient(BaseTTSClient): "condition usually mean another process is using " "the GPU.") - def _voice_wav_path(self) -> Optional[Path]: - """The selected voice's reference wav on this machine, when readable. - - Resolved from the local checkout's server.json exactly like the - server resolves the request's voice name: the entry's - ``voice_presets[name].voice_ref`` first, then ``voice_dir/.wav``. - Only preset-mode runs with a locally readable file return a path — - remote-only servers (or voice names that only exist server-side) - yield None and the trimmed-reference retry does not fire. - """ - if not self.voice: - return None - name = self.voice - if not name or name in (".", "..") or "/" in name or "\\" in name: - return None - try: - # Imported lazily: backends.audiocpp imports this package, so a - # module-level import would cycle. - from backends.audiocpp.build import find_local_checkout - checkout = find_local_checkout() - except Exception: # noqa: BLE001 - best effort - return None - if checkout is None: - return None - server_json = checkout / "server.json" - try: - data = json.loads(server_json.read_text(encoding="utf-8")) - except (OSError, ValueError): - return None - if not isinstance(data, dict): - return None - for entry in data.get("models") or []: - if not isinstance(entry, dict) or entry.get("id") != self.model_id: - continue - presets = entry.get("voice_presets") - if isinstance(presets, dict): - preset = presets.get(name) - if isinstance(preset, dict): - ref = preset.get("voice_ref") - if isinstance(ref, str) and ref: - path = Path(ref) - if not path.is_absolute(): - path = server_json.parent / ref - if path.is_file(): - return path - voice_dir = data.get("voice_dir") - if isinstance(voice_dir, str) and voice_dir: - candidate = Path(voice_dir) / f"{name}.wav" - if candidate.is_file(): - return candidate - return None - - def _voice_transcript(self) -> Optional[str]: - """The voice library transcript for the selected voice, or None. - - Read from the voice directory's prompt_text mapping (the same file - the server consults when it resolves a voice NAME); only carried - alongside an inline voice_ref, where the server's own injection is - bypassed. - """ - if not self.voice: - return None - try: - # Imported lazily: backends.audiocpp imports this package, so a - # module-level import would cycle. - from backends.common import PROMPT_TEXT_FILENAME, read_prompt_text - from backends.audiocpp.build import find_local_checkout - checkout = find_local_checkout() - except Exception: # noqa: BLE001 - best effort - return None - if checkout is None: - return None - try: - data = json.loads((checkout / "server.json") - .read_text(encoding="utf-8")) - except (OSError, ValueError): - return None - voice_dir = data.get("voice_dir") if isinstance(data, dict) else None - if not isinstance(voice_dir, str) or not voice_dir: - return None - try: - return (read_prompt_text(Path(voice_dir) / PROMPT_TEXT_FILENAME) - .get(self.voice) or None) - except OSError: - return None - - def _switch_to_trimmed_reference(self, server_message: str) -> bool: - """Switch the cloning reference to a trimmed local wav, once. - - Some families encode the whole reference with attention over its - length (MOSS-TTS-Local's codec encoder: the required memory grows - with the reference's square), so a long voice reference fails the - graph allocation regardless of how much VRAM the device has. When - the selected voice resolves to a locally readable wav, replace the - voice name with an inline base64 voice_ref cut to - AUDIOCPP_REFERENCE_TRIM_SECONDS and retry the request once; the - trimmed reference then applies to the rest of the run. Only fires - on allocation-failure messages for preset-mode runs; everything - else keeps the original behavior. - """ - lowered = server_message.lower() - if self._reference_trim_attempted \ - or not any(fragment in lowered - for fragment in AUDIOCPP_ALLOCATION_FRAGMENTS) \ - or self.design_mode or self.instruction_voice \ - or self.plain_mode or not self.voice: - return False - self._reference_trim_attempted = True - trimmed = build_trimmed_voice_reference(self._voice_wav_path()) - if trimmed is None: - return False - b64, seconds, source = trimmed - self._voice_ref_b64 = b64 - transcript = self._voice_transcript() - if transcript and "reference_text" in spec_request_option_names( - self.family): - self._voice_ref_reference_text = transcript - self._report( - f"[INFO] {source}'s family encodes the whole reference with " - f"attention over its length; retrying with the first " - f"{seconds:.0f}s of voice '{self.voice}' as the cloning " - "reference (the trimmed reference applies to the rest of this " - "run).") - return True - # ------------------------------------------------------------------ # HTTP requests # ------------------------------------------------------------------ @@ -1413,13 +1173,8 @@ class AudioCppTTSClient(BaseTTSClient): # Design models take no voice field (the voice comes from the # instruction); instruction-voice runs on families without built-in # speakers omit it too, since no speaker or preset was requested; - # plain-TTS runs (no reference voice needed) omit it likewise. A - # trimmed-reference retry replaces the voice name with an inline - # base64 voice_ref (see _switch_to_trimmed_reference). - if self._voice_ref_b64 is not None: - payload["voice_ref"] = {"type": "base64", - "data": self._voice_ref_b64} - elif not self.design_mode and not self.instruction_voice \ + # plain-TTS runs (no reference voice needed) omit it likewise. + if not self.design_mode and not self.instruction_voice \ and not self.plain_mode: payload["voice"] = self.voice if self.profile.language_style == AUDIOCPP_LANG_DISPLAY: @@ -1440,17 +1195,10 @@ class AudioCppTTSClient(BaseTTSClient): # Explicit voice-design or style instruction (required for task # "vdes" entries; a Ctrl/style control on families that read it). payload["instructions"] = self.instructions - options = dict(self.request_options) - if self._voice_ref_reference_text \ - and "reference_text" not in options: - # The server only injects the voice library's transcript when it - # resolves the voice NAME; an inline voice_ref bypasses that, so - # carry the transcript explicitly for families that accept it. - options["reference_text"] = self._voice_ref_reference_text - if options: + if self.request_options: # Generic per-model controls (--option KEY=VALUE): forwarded # verbatim; the model ignores keys it does not know. - payload["options"] = options + payload["options"] = dict(self.request_options) request = urllib.request.Request( url, data=json.dumps(payload).encode("utf-8"), headers={"Content-Type": "application/json"}, method="POST") @@ -1459,16 +1207,8 @@ class AudioCppTTSClient(BaseTTSClient): with urllib.request.urlopen(request, timeout=timeout) as response: wav = response.read() except urllib.error.HTTPError as exc: - detail = "" - try: - detail = exc.read().decode("utf-8", errors="replace")[:200] - except Exception: - pass + detail = _http_error_body(exc) message = _server_error_message(detail) - if self._switch_to_trimmed_reference(message): - # One retry with the trimmed reference; if that fails too the - # error below carries the server log's allocation detail. - return self._request_wav(text) raise audiocpp_request_error(exc.code, detail, voice=self.voice, log_note=allocation_log_note( diff --git a/app/converter/clients/base.py b/app/converter/clients/base.py index 8a0b4e4..9cf6979 100644 --- a/app/converter/clients/base.py +++ b/app/converter/clients/base.py @@ -68,14 +68,17 @@ class BaseTTSClient: cancel = None quiet = False - def __init__(self, chunks_dir: Path, quiet: bool = False): + def __init__(self, chunks_dir: Path, quiet: bool = False, + cancel: Optional[threading.Event] = None): self.chunks_dir = Path(chunks_dir) # Quiet silences console prints (the run view owns the screen). self.quiet = bool(quiet) # Set by the converter when the run is cancellable (the TUI run # view): a threading.Event that, once set, aborts the run between - # requests (and interrupts retry back-off sleeps). - self.cancel = None + # requests (and interrupts retry back-off sleeps). Assigned here — + # before the subclass's connect logic runs — so a cancel pressed + # while the client is still connecting is not lost. + self.cancel = cancel def _report(self, message: str) -> None: """Print a console line unless quiet (the run view owns the screen).""" diff --git a/app/converter/clients/faster.py b/app/converter/clients/faster.py index 48c26ab..eeecee9 100644 --- a/app/converter/clients/faster.py +++ b/app/converter/clients/faster.py @@ -31,8 +31,8 @@ class FasterTTSClient(BaseTTSClient): def __init__(self, chunks_dir: Path, voice: Optional[str] = None, api_url: Optional[str] = None, - quiet: bool = False): - super().__init__(chunks_dir, quiet=quiet) + quiet: bool = False, cancel=None): + super().__init__(chunks_dir, quiet=quiet, cancel=cancel) # The voice is per-run (--voice / the Generate form's Voice pick); # there is no configured default. self.voice = (voice or "").strip() @@ -85,10 +85,12 @@ class FasterTTSClient(BaseTTSClient): except urllib.error.HTTPError as exc: detail = "" try: - detail = exc.read().decode("utf-8", errors="replace")[:200] + detail = exc.read().decode("utf-8", errors="replace") except Exception: pass - raise RuntimeError(f"Faster TTS server returned HTTP {exc.code}: {detail}") from exc + raise RuntimeError( + f"Faster TTS server returned HTTP {exc.code}: {detail[:200]}" + ) from exc except urllib.error.URLError as exc: raise RuntimeError(f"Faster TTS request failed: {exc.reason}") from exc if not pcm: diff --git a/app/converter/clients/qwen.py b/app/converter/clients/qwen.py index 17f14c5..dd1c8ba 100644 --- a/app/converter/clients/qwen.py +++ b/app/converter/clients/qwen.py @@ -33,8 +33,8 @@ class QwenTTSClient(BaseTTSClient): voice_clone_ref_text: Optional[str] = None, skip_transcription: bool = False, language: Optional[str] = None, api_url: Optional[str] = None, instructions: Optional[str] = None, quiet: bool = False, - voice: Optional[str] = None): - super().__init__(chunks_dir, quiet=quiet) + voice: Optional[str] = None, cancel=None): + super().__init__(chunks_dir, quiet=quiet, cancel=cancel) if voice_mode not in VOICE_MODES: raise ValueError( f"Unknown voice mode: {voice_mode!r} (expected one of {VOICE_MODES})" diff --git a/app/converter/converter.py b/app/converter/converter.py index a10a57d..32cd342 100644 --- a/app/converter/converter.py +++ b/app/converter/converter.py @@ -66,6 +66,13 @@ DEBUG_FOLDER = APP_DIR / "debug" # --debug dumps, kept across runs AUDIO_FORMATS = ("mp3", "m4b", "ogg", "flac") SUPPORTED_FORMATS = [".txt", ".pdf", ".epub"] +# Device names Windows cannot use as a file name (with or without an +# extension); sanitized output names matching these get a prefix. +_WINDOWS_RESERVED_NAMES = frozenset( + {"CON", "PRN", "AUX", "NUL"} + | {f"COM{i}" for i in range(1, 10)} + | {f"LPT{i}" for i in range(1, 10)}) + def _console_log_filter(record: logging.LogRecord) -> bool: """Keep httpx/httpcore request logs and file-only traceback dumps out @@ -258,7 +265,7 @@ class AudiobookConverter: # configured on the server, so no local reference audio is needed. self.tts = FasterTTSClient(chunks_dir=CHUNKS_FOLDER, voice=voice, api_url=api_url, - quiet=quiet) + quiet=quiet, cancel=cancel) elif backend == BACKEND_AUDIOCPP: # --voice picks the voice: a built-in speaker name on the # CustomVoice entry, or a server-side preset (cloning) @@ -273,7 +280,8 @@ class AudiobookConverter: instructions=instructions, request_options=self.request_options, api_url=api_url, quiet=quiet, - unload_models=unload_models) + unload_models=unload_models, + cancel=cancel) else: # Qwen: the voice mode picks the request shape (built-in # speaker, clone from a reference .wav, or a designed voice); @@ -290,9 +298,12 @@ class AudiobookConverter: api_url=api_url, quiet=quiet, voice=voice, + cancel=cancel, ) self._progress = progress - self.tts.cancel = cancel + # The converter's own handle on the run's cancel event (also passed + # to the client, so a cancel during connect-time work is honored). + self._cancel = cancel def _emit(self, event: dict) -> None: """Send one progress event (a no-op without a progress callback).""" @@ -306,7 +317,11 @@ class AudiobookConverter: def _check_cancelled(self) -> None: """Raise ConversionCancelled when the run's cancel event is set.""" - cancel = getattr(getattr(self, "tts", None), "cancel", None) + cancel = getattr(self, "_cancel", None) + if not isinstance(cancel, threading.Event): + # Converters built without __init__ (tests): fall back to the + # client's event, the pre-constructor-arg wiring. + cancel = getattr(getattr(self, "tts", None), "cancel", None) if isinstance(cancel, threading.Event) and cancel.is_set(): raise ConversionCancelled("Cancelled by user") @@ -344,10 +359,17 @@ class AudiobookConverter: @staticmethod def _sanitize_filename(name: str, fallback: str = "chapter") -> str: - """Make a chapter title safe to use as part of a file name.""" + """Make a chapter title safe to use as part of a file name. + + Reserved Windows device names (CON, NUL, COM1, ...) are suffixed + so the resulting name is writable on every platform. + """ cleaned = re.sub(r'[\\/:*?"<>|]', " ", name) cleaned = re.sub(r"\s+", " ", cleaned).strip().strip(".") - return cleaned[:80] or fallback + cleaned = cleaned[:80] or fallback + if cleaned.upper() in _WINDOWS_RESERVED_NAMES: + return f"{fallback}_{cleaned}" + return cleaned def _narrator_tag(self) -> str: """Narrator name used in output file names (see compute_narrator_tag).""" @@ -480,15 +502,20 @@ class AudiobookConverter: stem = output_name or f"{file_path.stem}_{self._narrator_tag()}" # The output files this book will produce (single final file, - # or one per chapter). Reported on the book_done/book_failed - # events so the run view can list them in its summary. + # or one per chapter — each with its speed-adjusted copy when + # SPEED != 1.0, see audio.combine_chunks). Reported on the + # book_done/book_failed events so the run view can list them + # in its summary. + speed_tag = ("" if abs(self.speed - 1.0) < audio.SPEED_EPSILON + else f"_{self.speed:g}") if self.output_format == "m4b" or self.single_file \ or len(sections) == 1: - self.current_outputs = [f"{stem}.{self.output_format}"] + self.current_outputs = [f"{stem}{speed_tag}." + f"{self.output_format}"] else: self.current_outputs = [ f"{stem}_{index:02d}_" - f"{self._sanitize_filename(section.title)}." + f"{self._sanitize_filename(section.title)}{speed_tag}." f"{self.output_format}" for index, section in enumerate(sections, 1)] @@ -930,7 +957,11 @@ class AudiobookConverter: self._say(f"[INFO] Converting {len(planned)} of {len(book_files)} book(s)") - results = {} + # Per-book outcome ({book file name: ok}), published on the + # instance so multi-book orchestrators (the "All" run) can count + # partial success after run() returns False (a failed book aborts + # the rest, but earlier books still count). + self.results = {} cancelled = False for index, (book_file, output_name) in enumerate(planned, 1): self._check_cancelled() @@ -938,7 +969,7 @@ class AudiobookConverter: "name": book_file.name}) try: success = self.convert_book(book_file, output_name=output_name) - results[book_file.name] = success + self.results[book_file.name] = success self._emit({"kind": "book_done", "name": book_file.name, "ok": bool(success), "files": list(getattr(self, "current_outputs", []))}) @@ -949,21 +980,21 @@ class AudiobookConverter: break except KeyboardInterrupt: self._say("\n[WARNING] Conversion interrupted by user") - results[book_file.name] = False + self.results[book_file.name] = False break except Exception as exc: logger.error("Unexpected error: %s", exc) - results[book_file.name] = False + self.results[book_file.name] = False self._emit({"kind": "book_failed", "name": book_file.name, "error": str(exc), "files": list(getattr(self, "current_outputs", []))}) - if not results.get(book_file.name): + if not self.results.get(book_file.name): logger.error("Conversion of %s failed; aborting the remaining books", book_file.name) break - successful = sum(results.values()) - total = len(results) + successful = sum(self.results.values()) + total = len(self.results) # A cancelled run is not a successful run on either path (the TUI # event consumer and the console summary report it consistently). ok = not cancelled and total > 0 and successful == total @@ -979,7 +1010,7 @@ class AudiobookConverter: print(f"Total: {total} | Success: {successful} | Failed: {total - successful}") print("=" * 70) - for filename, success in results.items(): + for filename, success in self.results.items(): status = "[OK]" if success else "[FAIL]" print(f"{status} {filename}") diff --git a/app/converter/cover.py b/app/converter/cover.py index b2d3cb5..9bf9237 100644 --- a/app/converter/cover.py +++ b/app/converter/cover.py @@ -122,6 +122,10 @@ _FONT = { _GLYPH_WIDTH = 5 _GLYPH_HEIGHT = 7 +# Any character outside the embedded font (accented letters, CJK, +# Cyrillic, ...) renders as an empty box: silently dropping it would +# blank those titles while still counting their width for centering. +_UNKNOWN_GLYPH = [0x1F, 0x11, 0x11, 0x11, 0x11, 0x11, 0x1F] _TEXT_SCALE = 6 # render each font pixel as a 6x6 block _TEXT_MARGIN = 60 # horizontal padding when wrapping _TEXT_COLOR = (255, 255, 255) @@ -194,9 +198,7 @@ def _render_line(pixels: List[List[Tuple[int, int, int]]], text: str, x0: int, y stays tinted by the gradient behind it). """ for char_index, char in enumerate(text): - glyph = _FONT.get(char) - if glyph is None: - continue + glyph = _FONT.get(char, _UNKNOWN_GLYPH) x_off = x0 + char_index * (_GLYPH_WIDTH + 1) * _TEXT_SCALE for gy, bits in enumerate(glyph): for gx in range(_GLYPH_WIDTH): diff --git a/app/converter/extractors.py b/app/converter/extractors.py index f05d451..8b01372 100644 --- a/app/converter/extractors.py +++ b/app/converter/extractors.py @@ -6,7 +6,7 @@ import re import zipfile from html import unescape from pathlib import Path -from typing import List, NamedTuple +from typing import List, NamedTuple, Optional try: from bs4 import BeautifulSoup @@ -39,8 +39,6 @@ def extract_text(file_path: Path) -> str: return _extract_txt(file_path) if extension == ".pdf": return _extract_pdf(file_path) - if extension == ".epub": - return extract_epub(file_path) raise ValueError(f"Unsupported file format: {extension}") @@ -186,19 +184,91 @@ def _read_epub_ebooklib(file_path: Path): def _read_epub_zipfile(file_path: Path): - """Read EPUB HTML members as (title, html) pairs, ordered by filename.""" + """Read EPUB HTML members as (title, html) pairs, in spine order. + + Fallback for EPUBs ebooklib cannot read. The package's OPF describes + the reading order (its ```` itemrefs reference manifest items + by id), so documents are emitted in that order; the manifest's + ``properties="nav"`` item (the table of contents) and any document + outside the spine are skipped so the TOC is never narrated as a + chapter. EPUBs without a parsable OPF fall back to natural filename + order over every HTML member. + """ items = [] with zipfile.ZipFile(file_path, "r") as epub_zip: - for file_name in sorted(epub_zip.namelist(), key=_natural_key): - if file_name.lower().endswith((".html", ".xhtml", ".htm")): - try: - content = epub_zip.read(file_name).decode("utf-8", errors="ignore") - items.append((Path(file_name).stem, content)) - except Exception as exc: - logger.debug("Skipping EPUB member %r: %s", file_name, exc) + names = epub_zip.namelist() + html_names = [name for name in names + if name.lower().endswith((".html", ".xhtml", ".htm"))] + order = _epub_spine_order(epub_zip, html_names) + if order is None: + order = sorted(html_names, key=_natural_key) + for file_name in order: + try: + content = epub_zip.read(file_name).decode("utf-8", errors="ignore") + items.append((Path(file_name).stem, content)) + except Exception as exc: + logger.debug("Skipping EPUB member %r: %s", file_name, exc) return items +def _epub_spine_order(epub_zip: zipfile.ZipFile, html_names: List[str]): + """The EPUB's HTML members in spine order, or None when unparsable. + + Parses the package OPF (located via META-INF/container.xml, else the + only *.opf member): manifest item id -> href, then the spine's + idrefs. Returns member paths limited to HTML_NAMES; nav documents + (``properties`` containing "nav") and non-HTML items are excluded. + """ + container = "META-INF/container.xml" + opf_name = None + try: + rootfile = epub_zip.read(container).decode("utf-8", errors="ignore") + match = re.search(r"full-path\s*=\s*[\"']([^\"']+)[\"']", rootfile) + if match and match.group(1) in epub_zip.namelist(): + opf_name = match.group(1) + except (KeyError, OSError): + pass + if opf_name is None: + opf_candidates = [name for name in epub_zip.namelist() + if name.lower().endswith(".opf")] + if len(opf_candidates) != 1: + return None + opf_name = opf_candidates[0] + try: + opf = epub_zip.read(opf_name).decode("utf-8", errors="ignore") + except (KeyError, OSError): + return None + + def attr(tag: str, name: str) -> Optional[str]: + match = re.search(rf"\b{name}\s*=\s*[\"']([^\"']*)[\"']", tag) + return match.group(1) if match else None + + base = "/".join(opf_name.split("/")[:-1]) + item_tags = re.findall(r"]*>", opf) + + def is_nav(item_tag: str) -> bool: + properties = attr(item_tag, "properties") or "" + return "nav" in properties.split() + + order: List[str] = [] + for ref_tag in re.findall(r"]*>", opf): + idref = attr(ref_tag, "idref") + if not idref: + continue + match = next((item_tag for item_tag in item_tags + if attr(item_tag, "id") == idref), None) + if match is None or is_nav(match): + continue + href = attr(match, "href") + if not href: + continue + path = (f"{base}/{href}" if base else href) + path = re.sub(r"#.*$", "", path) + if path in html_names and path not in order: + order.append(path) + return order or None + + def _read_epub_manual(file_path: Path): """Last-resort read of any markup-looking EPUB member.""" skipped_extensions = (".jpg", ".jpeg", ".png", ".gif", ".css", ".js") @@ -219,15 +289,17 @@ def _read_epub_manual(file_path: Path): def clean_text(text: str) -> str: """Normalize whitespace and strip standalone page numbers. - Page numbers are removed only when they appear as a short number alone on - its own line (before whitespace collapsing), so inline numbers like - "42 years", "1,000" or "3.5" are preserved. + Page numbers are removed only when a short number (up to three digits) + appears alone on its own line (before whitespace collapsing), so inline + numbers like "42 years", "1,000" or "3.5" are preserved, as are + four-digit standalone lines, which are usually years ("1984") or + chapter numbers rather than page numbers. """ if not text: return "" # Standalone page numbers (digits alone on a line) must go BEFORE the # newline-collapsing step below. - text = re.sub(r"(?m)^\s*\d{1,4}\s*$", " ", text) + text = re.sub(r"(?m)^\s*\d{1,3}\s*$", " ", text) text = re.sub(r"\s+", " ", text) return text.strip() @@ -256,14 +328,6 @@ def clean_html(html_content: str) -> str: return html_content.strip() -def extract_epub(file_path: Path) -> str: - """Extract the book's text from EPUB, trying several methods in order.""" - chapters = _extract_epub_chapters(file_path) - if not chapters: - raise RuntimeError("All EPUB extraction methods failed") - return "\n\n".join(section.text for section in chapters) - - def _natural_key(name: str): """Sort key that orders numeric runs numerically (chapter2 before chapter10).""" return [int(part) if part.isdigit() else part.lower() @@ -286,10 +350,15 @@ def _extract_txt(file_path: Path) -> str: return clean_text(data.decode("utf-8-sig")) # No BOM: UTF-16 without BOM is common on Windows; detect via NUL bytes. + # A real UTF-16 file of ASCII-range text has a NUL at every other byte + # position, so require a substantial NUL share before committing to + # UTF-16: a lone stray NUL in a UTF-8/cp1252 file must not flip the + # whole book into mojibake (the decode ladder below handles that). sample = data[:4096] even_nuls = sum(1 for i, b in enumerate(sample) if b == 0 and i % 2 == 0) odd_nuls = sum(1 for i, b in enumerate(sample) if b == 0 and i % 2 == 1) - if even_nuls or odd_nuls: + threshold = max(len(sample) // 4, 1) + if even_nuls >= threshold or odd_nuls >= threshold: encoding = "utf-16-be" if even_nuls > odd_nuls else "utf-16-le" return clean_text(data.decode(encoding)) diff --git a/app/tests/cover_test.png b/app/tests/cover_test.png deleted file mode 100644 index 0c252db..0000000 Binary files a/app/tests/cover_test.png and /dev/null differ diff --git a/app/tests/gen_test_cover.py b/app/tests/gen_test_cover.py deleted file mode 100644 index 292469a..0000000 --- a/app/tests/gen_test_cover.py +++ /dev/null @@ -1,8 +0,0 @@ -import sys -from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) -from converter.cover import generate_cover - -p = generate_cover('The Count of Monte Cristo', - Path(__file__).resolve().parent / 'cover_test.png') -print('written:', p) diff --git a/app/tests/test_audio.py b/app/tests/test_audio.py index 6c67294..75de97c 100644 --- a/app/tests/test_audio.py +++ b/app/tests/test_audio.py @@ -22,34 +22,34 @@ from converter.audio import ( build_m4b_chapters_command, cleanup_chunks, concat_audio_files, - speed_export_params, + atempo_filters, verify_output_duration, ) -class SpeedExportParamsTests(unittest.TestCase): +class AtempoFiltersTests(unittest.TestCase): def test_normal_speed_no_filter(self): - self.assertEqual(speed_export_params(1.0), []) + self.assertEqual(atempo_filters(1.0), "") def test_simple_speedup(self): - self.assertEqual(speed_export_params(1.5), ["-filter:a", "atempo=1.5"]) + self.assertEqual(atempo_filters(1.5), "atempo=1.5") def test_simple_slowdown(self): - self.assertEqual(speed_export_params(0.75), ["-filter:a", "atempo=0.75"]) + self.assertEqual(atempo_filters(0.75), "atempo=0.75") def test_chained_speedup_beyond_2x(self): - self.assertEqual(speed_export_params(3.0), ["-filter:a", "atempo=2.0,atempo=1.5"]) + self.assertEqual(atempo_filters(3.0), "atempo=2.0,atempo=1.5") def test_chained_slowdown_below_half(self): - self.assertEqual(speed_export_params(0.25), ["-filter:a", "atempo=0.5,atempo=0.5"]) + self.assertEqual(atempo_filters(0.25), "atempo=0.5,atempo=0.5") def test_zero_speed_rejected(self): with self.assertRaises(ValueError): - speed_export_params(0) + atempo_filters(0) def test_negative_speed_rejected(self): with self.assertRaises(ValueError): - speed_export_params(-1.5) + atempo_filters(-1.5) class CleanupChunksTests(unittest.TestCase): diff --git a/app/tests/test_audiobook_cli.py b/app/tests/test_audiobook_cli.py index f1eb8e8..8076c8e 100644 --- a/app/tests/test_audiobook_cli.py +++ b/app/tests/test_audiobook_cli.py @@ -390,12 +390,16 @@ class AllModelsConvertTests(unittest.TestCase): ctor_kwargs.append(ckwargs) inst = MagicMock() made.append(inst) + result = True if make_run is not None: inst.run.side_effect = make_run(ckwargs) else: result = states[len(made) - 1] \ if len(made) <= len(states) else True inst.run.return_value = result + # Per-book outcomes the All-run loop counts on the CLI path + # (see audiobook._convert_each_model). + inst.results = {"book.txt": bool(result)} return inst fake_class = MagicMock(side_effect=make_instance) @@ -550,6 +554,7 @@ class AllModelsConvertTests(unittest.TestCase): def make_instance(*args, **ckwargs): inst = MagicMock() inst.run.return_value = True + inst.results = {"book.txt": True} return inst fake_class.side_effect = make_instance with patch.object(audiobook, "setup_logging"), \ diff --git a/app/tests/test_backends_audiocpp.py b/app/tests/test_backends_audiocpp.py index 18c38c4..9392ba1 100644 --- a/app/tests/test_backends_audiocpp.py +++ b/app/tests/test_backends_audiocpp.py @@ -410,6 +410,56 @@ class LoadModelCatalogTests(unittest.TestCase): self.assertIn("mystery_tts", [entry["family"] for entry in catalog]) + def test_non_object_spec_json_is_skipped_not_fatal(self): + # Valid JSON that is not an object (the "crash on bad model_specs" + # fix class): skipped like an unparsable spec, never an + # AttributeError out of the wizard. + for payload in ('["a list"]', '"a string"', "42", "null"): + (self.checkout / "model_specs" / "broken.json") \ + .write_text(payload, encoding="utf-8") + catalog = make_server.catalog.load_model_catalog(self.checkout) + self.assertNotIn("broken", + [entry["family"] for entry in catalog]) + + def test_non_dict_package_entries_are_skipped(self): + (self.checkout / "model_specs" / "weird_pkg.json").write_text( + json.dumps({"family": "weird_pkg", "tasks": ["tts"], + "packages": ["not-a-dict", + {"id": "wp", "format": "gguf", + "files": ["m.gguf"], + "target_directory": "weird_pkg"}]}), + encoding="utf-8") + catalog = make_server.catalog.load_model_catalog(self.checkout) + entry = next(e for e in catalog if e["family"] == "weird_pkg") + self.assertEqual(entry["install_id"], "wp") + + def test_download_path_materializes_the_catalog_repair(self): + # Regression for the shadowed-sanitizer bug: the download path's + # staged specs copy must apply the catalog sanitizer's SECOND bug + # class (missing strip_prefix on $gguf-rooted single-GGUF packages + # nested under a repo directory — glm_tts/outetts), which the old + # dot-only models.py repair did not. + (self.checkout / "model_specs" / "glm_like.json").write_text( + json.dumps({"family": "glm_like", "tasks": ["tts"], + "sources": [{"format": "gguf", + "roots": {"tokenizer": "$gguf"}}], + "packages": [{"id": "glm_q8", "format": "gguf", + "files": ["Text to audio (TTS)/" + "GLM-TTS_Q8.gguf"]}, + {"id": "glm_other", "format": "safetensors", + "files": ["tokenizer_merges"], + "default": True}]}), + encoding="utf-8") + staging = make_server.models._prepare_specs_dir(self.checkout) + self.assertIsNotNone(staging, "nested-GGUF repair was not staged") + try: + repaired = json.loads( + (staging / "glm_like.json").read_text(encoding="utf-8")) + finally: + shutil.rmtree(staging, ignore_errors=True) + self.assertEqual(repaired["packages"][0]["strip_prefix"], + "Text to audio (TTS)") + def test_families_sorted_alphabetically_by_display_name(self): catalog = make_server.catalog.load_model_catalog(self.checkout) names = [entry["display_name"].lower() for entry in catalog] @@ -1095,48 +1145,53 @@ class InstallModelsTests(unittest.TestCase): class SanitizeModelSpecTests(unittest.TestCase): - """The dot strip_prefix repair and the --specs-dir staging copy.""" + """The strip_prefix repairs (both upstream bug classes) and staging.""" def test_dot_prefix_dropped_when_file_is_bare(self): spec = {"packages": [{"files": ["model.gguf"], "strip_prefix": "."}]} - self.assertTrue(make_server.models._sanitize_model_spec(spec)) + self.assertTrue(make_server.catalog.sanitize_model_spec(spec)) self.assertEqual(spec["packages"][0]["strip_prefix"], "") def test_slash_dot_prefix_normalized_like_dot(self): spec = {"packages": [{"files": ["model.gguf"], "strip_prefix": "./"}]} - self.assertTrue(make_server.models._sanitize_model_spec(spec)) + self.assertTrue(make_server.catalog.sanitize_model_spec(spec)) self.assertEqual(spec["packages"][0]["strip_prefix"], "") def test_dot_prefix_kept_when_files_carry_it(self): spec = {"packages": [{"files": ["./model.gguf"], "strip_prefix": "."}]} - self.assertFalse(make_server.models._sanitize_model_spec(spec)) + self.assertFalse(make_server.catalog.sanitize_model_spec(spec)) self.assertEqual(spec["packages"][0]["strip_prefix"], ".") def test_real_directory_prefix_untouched(self): spec = {"packages": [{"files": ["model.gguf"], "strip_prefix": "Kroko-ASR-GGUF"}]} - self.assertFalse(make_server.models._sanitize_model_spec(spec)) + self.assertFalse(make_server.catalog.sanitize_model_spec(spec)) self.assertEqual(spec["packages"][0]["strip_prefix"], "Kroko-ASR-GGUF") def test_valid_prefix_untouched(self): spec = {"packages": [{"files": ["Kroko-ASR-GGUF/model.gguf"], "strip_prefix": "Kroko-ASR-GGUF"}]} - self.assertFalse(make_server.models._sanitize_model_spec(spec)) + self.assertFalse(make_server.catalog.sanitize_model_spec(spec)) - def test_missing_or_empty_files_untouched(self): + def test_malformed_files_package_repaired_too(self): + # The catalog sanitizer also drops a dot prefix when the package's + # files list is missing, empty, or malformed — nothing can match a + # dot prefix, so the repair is safe there as well (a dot prefix + # with files that all carry it is kept, see above). spec = {"packages": [{"strip_prefix": "."}, {"files": [], "strip_prefix": "."}, {"files": "model.gguf", "strip_prefix": "."}]} - self.assertFalse(make_server.models._sanitize_model_spec(spec)) + self.assertTrue(make_server.catalog.sanitize_model_spec(spec)) + self.assertEqual({p["strip_prefix"] for p in spec["packages"]}, {""}) def test_only_broken_packages_repaired(self): spec = {"packages": [ {"files": ["model.gguf"], "strip_prefix": "."}, {"files": ["./model.gguf"], "strip_prefix": "."}, ]} - self.assertTrue(make_server.models._sanitize_model_spec(spec)) + self.assertTrue(make_server.catalog.sanitize_model_spec(spec)) self.assertEqual([p["strip_prefix"] for p in spec["packages"]], ["", "."]) @@ -3368,7 +3423,6 @@ class ExecuteLanesTests(unittest.TestCase): "include_clone": False, "wav_dir": None, "plan": None, - "sync_port": None, "delete_unused": False, "unused_entries": [], "model_entries": [], diff --git a/app/tests/test_backends_common.py b/app/tests/test_backends_common.py index 08ffc2c..68a5543 100644 --- a/app/tests/test_backends_common.py +++ b/app/tests/test_backends_common.py @@ -51,6 +51,34 @@ class RunConsoleSubprocessStreamingTests(unittest.TestCase): on_cancel=lambda: touched.append(True)) self.assertEqual(touched, [True]) + def test_carriage_return_progress_streams_incrementally(self): + # tqdm/HuggingFace-style \r-only progress: a readline-based reader + # blocked until the next \n, so the updates arrived in one burst + # (or the stall watchdog fired first). Each \r segment must be + # emitted as its own line. + lines = [] + rc = common.run_console_subprocess( + [sys.executable, "-c", + "import sys, time\n" + "for i in range(4):\n" + " sys.stdout.write(f'pct {i}\\r'); sys.stdout.flush()\n" + " time.sleep(0.2)\n" + "sys.stdout.write('done\\n'); sys.stdout.flush()\n"], + emit=lines.append, stall_timeout=1.0) + self.assertEqual(rc, 0) + self.assertEqual(lines, [f"pct {i}" for i in range(4)] + ["done"]) + + def test_url_with_port_preserves_userinfo_and_ipv6(self): + self.assertEqual( + common.url_with_port("http://user:pass@host:8000", 8080), + "http://user:pass@host:8080") + self.assertEqual( + common.url_with_port("http://[::1]:8000", 8080), + "http://[::1]:8080") + self.assertEqual( + common.url_with_port("http://host:8000/path", 8080), + "http://host:8080/path") + class RunConsoleSubprocessStallTests(unittest.TestCase): """The no-output watchdog: a silent child is killed and reported 124.""" diff --git a/app/tests/test_backends_servers.py b/app/tests/test_backends_servers.py index ab05eed..61897d4 100644 --- a/app/tests/test_backends_servers.py +++ b/app/tests/test_backends_servers.py @@ -65,10 +65,45 @@ class StartTests(unittest.TestCase): ok = servers.start(self.spec) self.assertTrue(ok) mk.assert_called_once() - # Pid file written. + # Pid file written (first field is the pid; the optional second + # field is the start-time ownership token, absent on this platform). self.assertEqual( - (self.dir / "test-server.pid").read_text(encoding="utf-8"), - "4242") + (self.dir / "test-server.pid").read_text(encoding="utf-8") + .split()[0], "4242") + + def test_pid_file_records_a_start_time_token_where_available(self): + proc = MagicMock() + proc.pid = 5150 + proc.poll.return_value = None + with patch.object(servers, "LOG_DIR", self.dir), \ + patch("subprocess.Popen", return_value=proc), \ + patch.object(servers, "_process_start_token", + return_value="12345"), \ + patch("backends.common.server_running", + side_effect=[False, True]), \ + patch("time.sleep"): + self.assertTrue(servers.start(self.spec)) + fields = (self.dir / "test-server.pid") \ + .read_text(encoding="utf-8").split() + self.assertEqual(fields, ["5150", "12345"]) + + def test_recycled_pid_with_mismatched_token_is_not_ours(self): + # The pid is alive but its start time differs from the recorded + # token: an unrelated process now owns this pid, so manages/alive + # must report not-ours (and never kill it). + pid_file = self.dir / "test-server.pid" + pid_file.write_text("4242 111\n", encoding="utf-8") + with patch.object(servers, "LOG_DIR", self.dir), \ + patch.object(servers, "_pid_alive", return_value=True), \ + patch.object(servers, "_process_start_token", + return_value="999"): + self.assertFalse(servers.alive("test")) + self.assertFalse(servers.manages([self.spec])) + with patch.object(servers, "LOG_DIR", self.dir), \ + patch.object(servers, "_pid_alive", return_value=True), \ + patch.object(servers, "_process_start_token", + return_value="111"): + self.assertTrue(servers.alive("test")) def test_returns_false_when_process_exits_early(self): proc = MagicMock() diff --git a/app/tests/test_converter.py b/app/tests/test_converter.py index 41b9cad..0b0eb0d 100644 --- a/app/tests/test_converter.py +++ b/app/tests/test_converter.py @@ -38,6 +38,12 @@ class SanitizeFilenameTests(unittest.TestCase): def test_empty_falls_back(self): self.assertEqual(AudiobookConverter._sanitize_filename("///"), "chapter") + def test_windows_reserved_device_names_are_suffixed(self): + for name in ("CON", "nul", "COM1", "lpt2"): + result = AudiobookConverter._sanitize_filename(name) + self.assertNotEqual(result.upper(), name.upper()) + self.assertTrue(result.startswith("chapter_"), result) + class ConfigurationValidationTests(unittest.TestCase): def test_invalid_voice_mode_rejected(self): diff --git a/app/tests/test_converter_progress.py b/app/tests/test_converter_progress.py index d77e9e0..0a44725 100644 --- a/app/tests/test_converter_progress.py +++ b/app/tests/test_converter_progress.py @@ -225,6 +225,24 @@ class CancelTests(unittest.TestCase): converter = self.fixture.build(cancel=threading.Event()) converter._check_cancelled() # no raise + def test_cancel_event_reaches_the_client_before_connect(self): + # The cancel event must be wired into the TTS client at + # construction time — connect-time work (health check, model + # listing, voice validation) is otherwise un-cancellable. + cancel = threading.Event() + cancel.set() + client = MagicMock() + client.cancel = None + with patch.object(converter_mod, "QwenTTSClient", + return_value=client) as mock_qwen: + AudiobookConverter(voice_mode=VOICE_MODE_CUSTOM, + backend=BACKEND_QWEN, voice="Vivian", + cancel=cancel) + self.assertIs(mock_qwen.call_args.kwargs["cancel"], cancel) + with self.assertRaises(ConversionCancelled): + converter_mod.AudiobookConverter._check_cancelled( + MagicMock(tts=None, _cancel=cancel)) + if __name__ == "__main__": unittest.main() diff --git a/app/tests/test_cover.py b/app/tests/test_cover.py index f19db5a..ca8b9a2 100644 --- a/app/tests/test_cover.py +++ b/app/tests/test_cover.py @@ -9,6 +9,7 @@ from pathlib import Path from converter.cover import ( _random_light_color, + _render_line, _text_width, _wrap_title, generate_cover, @@ -117,11 +118,13 @@ class GenerateCoverTests(unittest.TestCase): _, _, rows = _decode_png(data) self.assertEqual(_black_pixels(rows), 0) - def test_unrenderable_title_degrades_to_gradient(self): - # CJK glyphs are not in the bitmap font; no crash, no text pixels + def test_unrenderable_title_renders_placeholder_boxes(self): + # CJK glyphs are not in the bitmap font: they used to vanish + # (blank cover); now each renders as a black-outlined placeholder + # box so the title is visibly present. _, data = self._write("书名") _, _, rows = _decode_png(data) - self.assertEqual(_black_pixels(rows), 0) + self.assertGreater(_black_pixels(rows), 0) def test_write_failure_returns_none(self): result = generate_cover("Hello", Path("/nonexistent_dir/cover.png")) @@ -187,3 +190,24 @@ class DropShadowTests(unittest.TestCase): if __name__ == "__main__": unittest.main() + + +class NonAsciiTitleTests(unittest.TestCase): + """Characters outside the bitmap font render as boxes, not blanks.""" + + def test_unknown_characters_are_rendered_as_placeholder_boxes(self): + # Regression: unknown chars used to be skipped in rendering while + # still counted for width, so e.g. Japanese titles produced a + # blank cover with no warning. + with tempfile.TemporaryDirectory() as tmp: + path = generate_cover("日本語", Path(tmp) / "c.png", + seed=1) + self.assertIsNotNone(path) + raw = path.read_bytes() + self.assertTrue(raw.startswith(b"\x89PNG")) + + def test_placeholder_draws_pixels_for_every_unknown_char(self): + pixels = [[(255, 255, 255)] * 60 for _ in range(60)] + _render_line(pixels, "éé", 2, 2, color=(0, 0, 0)) + dark = sum(1 for row in pixels for pixel in row if pixel == (0, 0, 0)) + self.assertGreater(dark, 2 * 5 * 7 - 6) # both boxes' outlines drawn diff --git a/app/tests/test_extractors.py b/app/tests/test_extractors.py index 64666ba..619fe5a 100644 --- a/app/tests/test_extractors.py +++ b/app/tests/test_extractors.py @@ -53,6 +53,86 @@ class TxtExtractionTests(unittest.TestCase): with self.assertRaises(ValueError): extract_text(path) + def test_utf16_without_bom_detected(self): + self.assertEqual(self._extract("chapter one".encode("utf-16-le")), + "chapter one") + + def test_lone_nul_does_not_flip_to_utf16(self): + # A single stray NUL byte in an otherwise-ASCII UTF-8 file must not + # switch the whole book to a UTF-16 decode (mojibake): the text + # comes back readable instead. + self.assertEqual(self._extract(b"hello world\x00rest"), + "hello world\x00rest") + + def test_standalone_page_numbers_removed_but_years_kept(self): + from converter.extractors import clean_text + + cleaned = clean_text("Chapter 1\n\n42\n\nIt was 1984.") + self.assertNotIn("42", cleaned) + self.assertIn("1984", cleaned) + kept = clean_text("It was the year\n\n1984\n\nwhen it began.") + self.assertIn("1984", kept) + + +class EpubZipfileFallbackTests(unittest.TestCase): + """The no-ebooklib EPUB fallback: spine order, no TOC narration.""" + + @staticmethod + def _write_epub(path: Path): + import zipfile + + container = ("" + "" + "" + "") + opf = ("" + "" + "" + "" + "" + "" + "" + "" + "" + "") + with zipfile.ZipFile(path, "w") as zf: + zf.writestr("mimetype", "application/epub+zip") + zf.writestr("META-INF/container.xml", container) + zf.writestr("OEBPS/content.opf", opf) + zf.writestr("OEBPS/nav.xhtml", + "

Contents

") + zf.writestr("OEBPS/text/chapterA.xhtml", + "

Alpha text.

") + zf.writestr("OEBPS/text/chapterB.xhtml", + "

Beta text.

") + + def test_spine_order_and_no_nav(self): + from converter.extractors import _read_epub_zipfile + + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "book.epub" + self._write_epub(path) + items = _read_epub_zipfile(path) + + titles = [title for title, _ in items] + self.assertNotIn("nav", titles) + # Spine order (B before A) beats filename sort (A before B). + self.assertEqual(titles, ["chapterB", "chapterA"]) + + def test_unparsable_opf_falls_back_to_filename_order(self): + import zipfile + + from converter.extractors import _read_epub_zipfile + + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "book.epub" + with zipfile.ZipFile(path, "w") as zf: + zf.writestr("a.xhtml", "

A

") + zf.writestr("b.xhtml", "

B

") + items = _read_epub_zipfile(path) + + self.assertEqual([title for title, _ in items], ["a", "b"]) + def _build_test_epub(path: Path, chapters=(("One", "First chapter text."), ("Two", "Second chapter text."))) -> None: diff --git a/app/tests/test_runview.py b/app/tests/test_runview.py index 4e3f9fe..6add83f 100644 --- a/app/tests/test_runview.py +++ b/app/tests/test_runview.py @@ -277,6 +277,17 @@ class LogAppenderTests(_FakeTui, unittest.TestCase): self.assertTrue(lines[0].endswith(" - one")) self.assertTrue(lines[2].endswith(" - three")) + def test_carriage_return_progress_splits_into_lines(self): + # tqdm/git-style \r-only progress: each segment becomes its own + # log line instead of one ever-growing buffered line. + path, appender = self._appender() + appender.write("pct 0\rpct 1\rpct 2\r") + appender.flush() + with open(path, encoding="utf-8") as handle: + lines = handle.read().splitlines() + self.assertEqual(len(lines), 3) + self.assertTrue(lines[2].endswith(" - pct 2")) + def test_blank_lines_and_empty_path_are_skipped(self): path, appender = self._appender() appender.write("\n\n") diff --git a/app/tests/test_taskview.py b/app/tests/test_taskview.py index 78cc2f1..2b3e542 100644 --- a/app/tests/test_taskview.py +++ b/app/tests/test_taskview.py @@ -277,7 +277,9 @@ class StateTransitionTests(_FakeTui, unittest.TestCase): view.handle_event({"kind": "step_cancelled", "index": 0}) view.handle_event({"kind": "finish", "phase": "cancelled", "rc": 1}) self.assertEqual(view.phase, "cancelled") - self.assertEqual(view._result_rc(), 1) + # 130: the CLI's user-cancel code, so callers can flash + # "cancelled" instead of "failed" (see hub._download_models_action). + self.assertEqual(view._result_rc(), 130) # The step interrupted by cancel is marked cancelled, not failed. self.assertEqual(view._step_mark(0), ("[x]", "warn")) self.assertEqual(view._step_mark(1), ("[ ]", "dim")) @@ -677,7 +679,7 @@ class LanesViewTests(_FakeTui, unittest.TestCase): view._drain() self.assertEqual(view.phase, "cancelled") self.assertTrue(view.cancelled) - self.assertEqual(view._result_rc(), 1) + self.assertEqual(view._result_rc(), 130) def test_split_render_draws_both_lane_titles(self): view, screen = self.make_view(self._two_lanes()) diff --git a/app/tests/test_tts.py b/app/tests/test_tts.py index 45e3812..39b407d 100644 --- a/app/tests/test_tts.py +++ b/app/tests/test_tts.py @@ -41,14 +41,15 @@ from converter.clients import ( VOICE_MODES, AudioCppTTSClient, FasterTTSClient, + QWEN3_TTS_SPEAKERS, QwenTTSClient, audiocpp_entry_voice_capability, audiocpp_family_narrates, audiocpp_family_voice_policy, audiocpp_request_error, audiocpp_script_input, + audiocpp_voice_for_run, allocation_log_note, - build_trimmed_voice_reference, nvidia_device_memory_report, normalize_language, transcribe_reference_audio_detailed, @@ -2310,7 +2311,7 @@ class BackendWiringTests(unittest.TestCase): backend=BACKEND_FASTER, voice="narrator") mock_faster.assert_called_once_with(chunks_dir=converter_mod.CHUNKS_FOLDER, voice="narrator", api_url=None, - quiet=False) + quiet=False, cancel=None) mock_qwen.assert_not_called() mock_audiocpp.assert_not_called() @@ -2327,7 +2328,7 @@ class BackendWiringTests(unittest.TestCase): instructions=None, request_options={}, api_url=None, quiet=False, - unload_models=None) + unload_models=None, cancel=None) mock_faster.assert_not_called() mock_qwen.assert_not_called() @@ -2341,7 +2342,7 @@ class BackendWiringTests(unittest.TestCase): instructions=None, request_options={}, api_url=None, quiet=False, - unload_models=None) + unload_models=None, cancel=None) def test_audiocpp_backend_model_id_is_wired_through(self): with patch("converter.converter.AudioCppTTSClient") as mock_audiocpp: @@ -2353,7 +2354,7 @@ class BackendWiringTests(unittest.TestCase): voice="narrator", language=config.LANGUAGE, model_id="higgs", instructions=None, request_options={}, api_url=None, quiet=False, - unload_models=None) + unload_models=None, cancel=None) def test_audiocpp_backend_instructions_and_options_are_wired_through(self): with patch("converter.converter.AudioCppTTSClient") as mock_audiocpp: @@ -2368,7 +2369,8 @@ class BackendWiringTests(unittest.TestCase): model_id=None, instructions="A warm adult narrator", request_options={"emotion": "neutral", "speed": "1.1"}, - api_url=None, quiet=False, unload_models=None) + api_url=None, quiet=False, unload_models=None, + cancel=None) def test_qwen_backend_uses_qwen_client(self): with patch("converter.converter.FasterTTSClient") as mock_faster, \ @@ -2420,7 +2422,7 @@ class BackendWiringTests(unittest.TestCase): voice="narrator", language=config.LANGUAGE, model_id=None, instructions=None, request_options={}, api_url="http://10.0.0.5:8080", quiet=False, - unload_models=None) + unload_models=None, cancel=None) with patch("converter.converter.FasterTTSClient") as mock_faster: AudiobookConverter(voice_mode=VOICE_MODE_CLONE, backend=BACKEND_FASTER, voice="narrator", @@ -2428,7 +2430,7 @@ class BackendWiringTests(unittest.TestCase): mock_faster.assert_called_once_with(chunks_dir=converter_mod.CHUNKS_FOLDER, voice="narrator", api_url="http://10.0.0.5:8000", - quiet=False) + quiet=False, cancel=None) with patch("converter.converter.QwenTTSClient") as mock_qwen: AudiobookConverter(voice_mode=VOICE_MODE_CUSTOM, backend=BACKEND_QWEN, voice="Vivian", @@ -2438,7 +2440,8 @@ class BackendWiringTests(unittest.TestCase): voice_mode=VOICE_MODE_CUSTOM, voice_clone_ref_audio=None, voice_clone_ref_text=None, skip_transcription=False, language=config.LANGUAGE, instructions=None, - api_url="http://10.0.0.5:7860", quiet=False, voice="Vivian") + api_url="http://10.0.0.5:7860", quiet=False, voice="Vivian", + cancel=None) def test_audiocpp_clone_mode_does_not_require_reference(self): # Cloning is server-side for the audiocpp backend, so the @@ -2548,77 +2551,6 @@ if __name__ == "__main__": unittest.main() -class TrimmedVoiceReferenceTests(unittest.TestCase): - """build_trimmed_voice_reference: a bounded inline cloning reference.""" - - def setUp(self): - self._tmp = tempfile.TemporaryDirectory() - self.dir = Path(self._tmp.name) - - def tearDown(self): - self._tmp.cleanup() - - @staticmethod - def _wav_bytes(rate, channels, sampwidth, frames, fill): - if sampwidth == 1: - payload = bytes(fill & 0xFF for _ in range(frames * channels)) - else: - payload = fill.to_bytes(sampwidth, "little", signed=True) \ - * frames * channels - return (b"RIFF" + struct.pack(" None: - """Run one backend's dedicated configure screen on this session.""" + def _run_screen(self, info, runner) -> None: + """Run INFO's screen function (setup or configure) on this session. + + Shared wrapper for _run_setup/_run_configure: a WizardCancelled is + a normal exit, any other exception flashes instead of taking the + hub down. + """ try: - info.configure_screen(self.stdscr) + runner(self.stdscr) except tui.WizardCancelled: pass except Exception as exc: # noqa: BLE001 - keep the hub alive tui.flash(self.stdscr, str(exc), "err") + def _run_configure(self, info) -> None: + """Run one backend's dedicated configure screen on this session.""" + self._run_screen(info, info.configure_screen) + def _run_setup(self, info) -> None: """Run one backend's setup wizard on this session (no stack frame).""" - try: - info.setup_screen(self.stdscr) - except tui.WizardCancelled: - pass - except Exception as exc: # noqa: BLE001 - keep the hub alive - tui.flash(self.stdscr, str(exc), "err") + self._run_screen(info, info.setup_screen) def screen_install(self): """Pick a backend to install and run its setup inline. @@ -443,7 +448,12 @@ class _Hub: pass except Exception as exc: # noqa: BLE001 - keep the hub alive view._cancel.set() - view._worker.join(timeout=30) + worker = view._worker + if worker is not None and worker.is_alive(): + try: + worker.join(timeout=30) + except RuntimeError: + pass tui.flash(self.stdscr, f"The run view failed: {exc}", "err") finally: try: @@ -465,12 +475,14 @@ class _Hub: result = tui.form(self.stdscr, "Settings", fields, back_value=tui.Wizard.BACK) if not (result is tui.Wizard.BACK or result is None): - # Save pressed: apply as before, no prompt. + # Save pressed: apply, and on failure loop back into the + # form with the edits intact instead of discarding them. try: _apply_settings(result) + return tui.Wizard.BACK except ValueError as exc: tui.flash(self.stdscr, str(exc), "err") - return tui.Wizard.BACK + continue # q/Esc (or the Cancel button) left the form without saving: # with no edits there is nothing to keep, so go straight back; # otherwise ask whether the edits should be preserved. @@ -591,11 +603,15 @@ def _server_action_step(spec, action: str): def work(emit, cancel): inner = sys.stdout # the task view's line-writer, when run in TUI - with contextlib.redirect_stdout(logging_kit.TeeWriter(logf, inner)): - if action == "start": - ok = servers.start(spec, cancel=cancel) - else: - ok = servers.stop(spec.name) + try: + with contextlib.redirect_stdout( + logging_kit.TeeWriter(logf, inner)): + if action == "start": + ok = servers.start(spec, cancel=cancel) + else: + ok = servers.stop(spec.name) + finally: + logf.close() return 0 if ok else 1 return taskview.TaskStep(title, work), log_path @@ -779,7 +795,7 @@ def _status_mark(status: Optional[BackendStatus]) -> Tuple[str, str, str]: name_kind = "dim" if not status.installed else "body" return (status.partial, "warn", name_kind) if status is not None and status.installed: - if status.models_missing and not status.running: + if status.models_missing: return ("installed (models missing)", "warn", "body") return ("installed", "ok", "body") return ("unavailable", "err", "dim") @@ -938,6 +954,39 @@ def _convert_form(stdscr) -> Optional[tuple]: return fields, builders, statuses +def _tui_confirm(stdscr) -> Callable: + """The overwrite-confirm callback the TUI pre-flight hands the converter. + + Asks with tui.confirm (the console input() would scribble over + curses); the cancel answer raises _BackToForm so the caller returns + to the Generate form. + """ + def confirm(message: str, default: bool) -> bool: + answer = tui.confirm(stdscr, message, default=default, + cancel_value=_CANCEL) + if answer is _CANCEL: + raise _BackToForm() + return answer + return confirm + + +def _check_preflight_plan(stdscr, book_files: list, planned: dict) -> bool: + """The shared nothing-to-convert flashes; True when there is a plan. + + PLANNED maps a run key (a model id, or "" for the single-model run) + to that run's plan; a run happens when any of them is non-empty. + """ + if not book_files: + tui.flash(stdscr, "No books to convert. Add a .txt, .pdf or .epub " + "file to the input folder first.") + return False + if not any(planned.values()): + tui.flash(stdscr, "Nothing to convert — every existing output was " + "kept.") + return False + return True + + def _preflight(stdscr, cmd: tuple) -> bool: """Run the overwrite checks in the TUI; stash the plan on the command. @@ -957,14 +1006,6 @@ def _preflight(stdscr, cmd: tuple) -> bool: voice_mode = voice_mode_for(backend, kwargs.get("voice"), kwargs.get("clone"), kwargs.get("instructions")) - - def confirm(message: str, default: bool) -> bool: - answer = tui.confirm(stdscr, message, default=default, - cancel_value=_CANCEL) - if answer is _CANCEL: - raise _BackToForm() - return answer - with contextlib.redirect_stdout(io.StringIO()): book_files, planned = AudiobookConverter.preflight_overwrites( backend=backend, voice=kwargs.get("voice"), @@ -972,14 +1013,8 @@ def _preflight(stdscr, cmd: tuple) -> bool: voice_clone_ref_audio=kwargs.get("clone"), output_format=kwargs.get("output_format") or config.AUDIO_FORMAT, instructions=kwargs.get("instructions"), - confirm=confirm) - if not book_files: - tui.flash(stdscr, "No books to convert. Add a .txt, .pdf or .epub " - "file to the input folder first.") - return False - if not planned: - tui.flash(stdscr, "Nothing to convert — every existing output was " - "kept.") + confirm=_tui_confirm(stdscr)) + if not _check_preflight_plan(stdscr, book_files, {"": planned}): return False kwargs["book_files"] = book_files kwargs["planned"] = planned @@ -1002,13 +1037,6 @@ def _preflight_all(stdscr, backend: str, kwargs: dict) -> bool: model_voices = kwargs.get("model_voices") or {} instructions = kwargs.get("instructions") - def confirm(message: str, default: bool) -> bool: - answer = tui.confirm(stdscr, message, default=default, - cancel_value=_CANCEL) - if answer is _CANCEL: - raise _BackToForm() - return answer - book_files: list = [] planned_by_model: dict = {} with contextlib.redirect_stdout(io.StringIO()): @@ -1021,18 +1049,12 @@ def _preflight_all(stdscr, backend: str, kwargs: dict) -> bool: voice_clone_ref_audio=kwargs.get("clone"), output_format=kwargs.get("output_format") or config.AUDIO_FORMAT, - instructions=instructions, confirm=confirm, + instructions=instructions, confirm=_tui_confirm(stdscr), name_tag=AudiobookConverter.compute_model_tag(model_id)) if not book_files: book_files = books planned_by_model[model_id] = planned - if not book_files: - tui.flash(stdscr, "No books to convert. Add a .txt, .pdf or .epub " - "file to the input folder first.") - return False - if not any(planned_by_model.values()): - tui.flash(stdscr, "Nothing to convert — every existing output was " - "kept.") + if not _check_preflight_plan(stdscr, book_files, planned_by_model): return False kwargs["book_files"] = book_files kwargs["planned_by_model"] = planned_by_model @@ -1348,31 +1370,16 @@ def _audiocpp_fields(stdscr, api_url: Optional[str] = None, def all_voice_for(model_id: str, picked: Optional[str]) -> Optional[str]: """The voice to send for MODEL_ID in an "All" run. - The picked voice wins wherever the model accepts it; models the - pick cannot serve fall back to their own default: the first - built-in speaker (CustomVoice) or first server voice (cloning), - or no voice at all (design entries and voice-less clone families - — the client then designs the voice from Instructions or - synthesizes plainly). + Single-sourced in audiocpp_voice_for_run (the same rules the + client documents): the pick wins where the model accepts it, and + models it does not fit fall back to their own default — the + first built-in speaker, first server voice, or no voice at all. """ entry = next((m for m in models if m.get("id") == model_id), models[0]) - capability = entry_capability(entry) - if capability == AUDIOCPP_VOICE_DESIGN: - return None - if capability == AUDIOCPP_VOICE_SPEAKER: - if picked and picked in QWEN3_TTS_SPEAKERS: - return picked - return QWEN3_TTS_SPEAKERS[0] - # Clone capability; the family policy decides whether a voice - # exists at all. - if audiocpp_family_voice_policy( - entry.get("family") or "") == AUDIOCPP_VOICE_NONE: - return None - voices = voices_for(model_id) - if picked and picked in voices: - return picked - return voices[0] if voices else None + return audiocpp_voice_for_run( + entry.get("family") or "", entry.get("task") or "tts", + entry.get("id") or "", picked, voices_for(model_id)) def all_voice_problem() -> Optional[str]: """Why an "All" run cannot start with the current settings, or None. @@ -1543,9 +1550,11 @@ def _audiocpp_fields(stdscr, api_url: Optional[str] = None, """The entry's capability words in fixed column order. Column 1 voices plain synthesis ("speaker" for built-in speakers, - "tts" for families that need no voice at all), column 2 is - "clone" when the entry clones a reference, column 3 "design" when - it can design a voice from an Instructions description. + "tts" for families that need no voice at all) — or the family's + kind when it cannot narrate text at all ("s2s", speech-to-speech). + Column 2 is "clone" when the entry clones a reference, column 3 + "design" when it can design a voice from an Instructions + description. """ family = entry.get("family") or "" task = entry.get("task") or "tts" @@ -1555,6 +1564,10 @@ def _audiocpp_fields(stdscr, api_url: Optional[str] = None, return ("speaker", "", "") if capability == AUDIOCPP_VOICE_DESIGN: return ("", "", "design") + if audiocpp_family_narrates(family) is False: + # Speech-to-speech-only family: labeling it "tts" would be the + # exact opposite of the truth. + return ("s2s", "", "") # The generic clone capability is refined by the family's voice # policy: pure-TTS families need no voice at all, mixed families # may run with or without one, clone-only families (and unknown @@ -1630,6 +1643,24 @@ def _audiocpp_fields(stdscr, api_url: Optional[str] = None, or (model_capability(fs) == AUDIOCPP_VOICE_CLONE and model_voice_policy(fs) == AUDIOCPP_VOICE_NONE)) + def model_validate(value) -> Optional[str]: + """Refuse a single-model pick that cannot synthesize narration. + + Speech-to-speech-only families (e.g. PersonaPlex) fail every + request regardless of hosting: the "All" path skips them, so the + single-model pick must refuse them too rather than start a doomed + run (the Voice field is hidden there, so voice_validate never + runs). The All pick is validated by voice_validate / + instructions_validate instead. + """ + if value == AUDIOCPP_MODEL_ALL: + return None + entry = model_entry(fields) + if audiocpp_family_narrates(entry.get("family") or "") is False: + return (f"'{entry.get('id')}' is speech-to-speech, not TTS: it " + "cannot turn text into audio. Pick a TTS model") + return None + def instructions_validate(value) -> Optional[str]: """Refuse a blank Instructions when the run needs it for a voice. @@ -1658,7 +1689,8 @@ def _audiocpp_fields(stdscr, api_url: Optional[str] = None, # The pick menu shows the padded capability table; the form row # collapses its column padding back to the two-space gutter. "compact_label": True, - "on_change": reset_voice}, + "on_change": reset_voice, + "validate": model_validate}, # The label tracks the entry's capability: a built-in speaker on # CustomVoice, otherwise the name of a server-side voice to clone. # Hidden on design entries (the voice is described) and on @@ -1755,8 +1787,9 @@ def _qwen_fields(remote_modes: Optional[list] = None, CustomVoice, a Clone .wav directory browser (default ./voices) + Voice- to-clone .wav picker on Base, Instructions on VoiceDesign — and MAPPER turns a submitted form values dict into the qwen converter - kwargs. qwen always has options to offer, so it never signals - unavailability. PREFIX namespaces the field keys ("" for the managed + kwargs. None (form omitted) when a filtered remote model list comes + back empty — no known mode matched what the remote demo reported. + PREFIX namespaces the field keys ("" for the managed entry) so two entries of this backend can share one form without overwriting each other. @@ -1785,6 +1818,10 @@ def _qwen_fields(remote_modes: Optional[list] = None, model_choices = [(label, value) for (label, value) in model_choices if dict(mode_keys)[value] in available] by_value = {value: label for label, value in model_choices} + if not model_choices: + # Nothing the remote demo can be hosting (its reported model name + # matched no known mode): the form cannot offer a model pick. + return None default_mode = "custom" if "custom" in by_value else model_choices[0][1] speakers = list(qwen_backend.QWEN_SPEAKERS) default_speaker = speakers[0] diff --git a/app/ui/runview.py b/app/ui/runview.py index 21ac08c..005db59 100644 --- a/app/ui/runview.py +++ b/app/ui/runview.py @@ -37,6 +37,7 @@ from typing import Callable, List, Optional from backends import common, servers from ui import tui +from ui import viewkit from ui.viewkit import (TERMINAL_PHASES as _TERMINAL, DRAW_TIMEOUT_MS as _DRAW_TIMEOUT_MS, ScreenView, _box, _fit, _format_elapsed, _sep, @@ -58,42 +59,23 @@ _SERVER_STATES = { _MONITOR_INTERVAL = 2.0 -class _LogAppender: +class _LogAppender(viewkit.LineSplitter): """A file-like that appends redirected console output to the run's log. The run view owns the screen, so anything a conversion prints to stdout/stderr outside the progress events would otherwise be swallowed - silently; this mirrors it line by line into the run's dated log file + silently; this mirrors it line by line (\\n and \\r — see + viewkit.LineSplitter) into the run's dated log file (RunConfig.log_path, the audiobook_ day stream), prefixed with the same timestamp format the converter's log records use. Best-effort: write errors are swallowed, and an empty path disables logging. """ def __init__(self, path: str): + super().__init__(self._append_line) self._path = path - self._buffer = "" - - def write(self, text: str) -> int: - if not text: - return 0 - self._buffer += text - while True: - cut = self._buffer.find("\n") - if cut < 0: - break - line, self._buffer = self._buffer[:cut], self._buffer[cut + 1:] - self._append(line) - return len(text) - - def flush(self) -> None: - if self._buffer: - self._append(self._buffer) - self._buffer = "" - - def isatty(self) -> bool: - return False - def _append(self, line: str) -> None: + def _append_line(self, line: str) -> None: if not self._path or not line.strip(): return try: @@ -274,6 +256,10 @@ class RunView(ScreenView): if event.get("cancelled"): self.cancelled = True self._finish("cancelled") + elif total == 0 and ok == 0 and not self.error_message: + # An empty run (no books found, or all skipped): a clean + # no-op, not a failure — there was nothing that could fail. + self._finish("done") elif total and ok >= total and not self.error_message: self._finish("done") else: @@ -404,21 +390,30 @@ class RunView(ScreenView): going. """ self._blocking() - answer = tui.confirm(self.scr, "Cancel processing?", default=False, - cancel_value=False) - if not answer: + try: + answer = tui.confirm(self.scr, "Cancel processing?", default=False, + cancel_value=False) + finally: self._nonblocking() + if not answer: return False self.cancelling = True self._cancel.set() # Wind the worker down BEFORE offering the server stop: killing the # server under a still-running request turns the cancellation into # request failures (reported as "failed" instead of "cancelled"). - self._worker.join(timeout=60) + # The join is best-effort — a wedged worker delays but cannot veto + # the flow below. + self._join_worker() # When this run booted the server, offer to shut it down too (the # boot path kills it itself when cancelled before ready); by now - # the worker is done, so nothing is mid-request. - self._confirm_stop_server() + # the worker is done (or wedged beyond saving), so nothing further + # is mid-request from this view's side. + self._blocking() + try: + self._confirm_stop_server() + finally: + self._nonblocking() self._drain() self.render() # One more key press acknowledges the final screen. diff --git a/app/ui/taskview.py b/app/ui/taskview.py index 3831ceb..e280673 100644 --- a/app/ui/taskview.py +++ b/app/ui/taskview.py @@ -53,10 +53,11 @@ from typing import Callable, List, Optional, Tuple import logging_kit from ui import tui +from ui import viewkit from ui.viewkit import (TERMINAL_PHASES as _TERMINAL, DRAW_TIMEOUT_MS as _DRAW_TIMEOUT_MS, - ScreenView, _box, _fit, _format_elapsed, _sep, - _text) + ScreenView, _box, _fit, _format_elapsed, _rect_box, + _sep, _text) # How many recent output lines the tail keeps in memory. The on-screen tail # draws as many as fit (see render); the full run is also mirrored to the @@ -411,9 +412,14 @@ class TaskView(ScreenView): return self._result_rc() def _result_rc(self) -> int: - """The exit code for the whole run (cancelled counts as failure).""" + """The exit code for the whole run: 0 ok, 130 cancelled, else first rc. + + 130 (the CLI's Ctrl-C code) distinguishes a user cancel from a + plain step failure, so callers like the hub can flash "cancelled" + instead of "failed". + """ if self.cancelled: - return 1 + return 130 return next((rc for rc in self.results if rc), 0) def _on_stop(self) -> None: @@ -432,7 +438,7 @@ class TaskView(ScreenView): return False self.cancelling = True self._cancel.set() - self._worker.join(timeout=60) + self._join_worker() return True # ------------------------------------------------------------------ @@ -542,42 +548,13 @@ class TaskView(ScreenView): # Small helpers (module-level for testability) # --------------------------------------------------------------------------- -class _LineWriter: - """A file-like object that forwards writes to a per-line callback. - - Handles carriage-return progress updates (git/tqdm) by treating ``\r`` - as a line terminator too, so the last full line always reflects the - latest progress. - """ - - def __init__(self, emit: Callable[[str], None]): - self._emit = emit - self._buffer = "" - - def write(self, text: str) -> int: - if not text: - return 0 - self._buffer += text - while True: - cut = _find_line_end(self._buffer) - if cut < 0: - break - line, self._buffer = self._buffer[:cut], self._buffer[cut + 1:] - if line: - self._emit(line) - return len(text) - - def flush(self) -> None: - if self._buffer: - self._emit(self._buffer) - self._buffer = "" - - def isatty(self) -> bool: - return False +class _LineWriter(viewkit.LineSplitter): + """A file-like that forwards writes to a per-line callback (see + viewkit.LineSplitter for the \\r/\\n splitting).""" def _find_line_end(text: str) -> int: - """Index of the earliest ``\n`` or ``\r`` in TEXT, else -1.""" + """Index of the earliest ``\\n`` or ``\\r`` in TEXT, else -1.""" newline = text.find("\n") carriage = text.find("\r") if newline < 0: @@ -607,23 +584,6 @@ def _fmt_bytes(size: float) -> str: return f"{value:.1f}GB" -def _rect_box(scr, curses, theme, x: int, y: int, w: int, h: int) -> None: - """Draw a box around the rectangle ``(x, y, w, h)``.""" - border = theme["border"] - try: - scr.addch(y, x, curses.ACS_ULCORNER, border) - scr.addch(y, x + w - 1, curses.ACS_URCORNER, border) - scr.addch(y + h - 1, x, curses.ACS_LLCORNER, border) - scr.addch(y + h - 1, x + w - 1, curses.ACS_LRCORNER, border) - scr.hline(y, x + 1, curses.ACS_HLINE, w - 2, border) - scr.hline(y + h - 1, x + 1, curses.ACS_HLINE, w - 2, border) - for yy in range(y + 1, y + h - 1): - scr.addch(yy, x, curses.ACS_VLINE, border) - scr.addch(yy, x + w - 1, curses.ACS_VLINE, border) - except Exception: - pass - - class _ThreadRouter: """A file-like object that routes writes to a per-thread writer. @@ -846,9 +806,9 @@ class LanesView(_GetchModes): return self._clock() def _result_rc(self) -> int: - """The exit code for the whole run (cancelled counts as failure).""" + """The exit code for the whole run: 0 ok, 130 cancelled, else first rc.""" if self.cancelled: - return 1 + return 130 for lane in self._lanes: for rc in lane.results: if rc: @@ -906,7 +866,11 @@ class LanesView(_GetchModes): return key def _prompt_cancel(self) -> bool: - """Esc/q: confirm cancel, then wait for both workers to wind down.""" + """Esc/q: confirm cancel, then wait for the workers to wind down. + + Best-effort joins (see ScreenView._join_worker): a wedged lane + worker is left to its daemon fate rather than blocking the view. + """ self._blocking() try: answer = tui.confirm(self.scr, "Cancel this step?", default=False, @@ -918,8 +882,9 @@ class LanesView(_GetchModes): self.cancelling = True self._cancel.set() for lane in self._lanes: - if lane.worker is not None: - lane.worker.join(timeout=60) + worker = lane.worker + if worker is not None and worker.is_alive(): + worker.join(timeout=60) return True # -- drawing ----------------------------------------------------- diff --git a/app/ui/tui.py b/app/ui/tui.py index 1e95353..60a77dc 100644 --- a/app/ui/tui.py +++ b/app/ui/tui.py @@ -399,11 +399,11 @@ class Frame: # -- drawing --------------------------------------------------------- def _row_width(self, row: dict) -> int: - """Logical width of a row, including its indent.""" + """Display width of a row (wide chars count 2), plus its indent.""" if row["segments"] is not None: - return sum(len(text) for text, _ in row["segments"]) \ + return sum(_disp_width(text) for text, _ in row["segments"]) \ + 2 * row["indent"] - return len(row["text"]) + 2 * row["indent"] + return _disp_width(row["text"]) + 2 * row["indent"] def _status_extra(self) -> int: """Rows the status block needs beyond its single bottom row. @@ -417,19 +417,24 @@ class Frame: return max(0, len(self.status[0].split("\n")) - 1) def _measure(self, width: int) -> int: - """Dialog width: widest row plus frame, capped to the screen.""" - longest = max(len(self.title) + 4, len(self.footer) + 4, 40) + """Dialog width: widest row plus frame, capped to the screen. + + Measured in display columns (wide chars count 2), so CJK text + gets a dialog wide enough to hold it untruncated. + """ + longest = max(_disp_width(self.title) + 4, + _disp_width(self.footer) + 4, 40) for row in self.rows: longest = max(longest, self._row_width(row) + 4) if self.status: # Measure per line: a multi-line status must not widen the # dialog to the combined length of its lines. - longest = max(longest, max(len(line) for line + longest = max(longest, max(_disp_width(line) for line in self.status[0].split("\n")) + 6) if self.buttons: labels, _ = self.buttons longest = max(longest, - sum(len(label) + 6 for label in labels) + 4) + sum(_disp_width(label) + 6 for label in labels) + 4) return min(longest + 4, width - 2) def _flatten(self, usable: int @@ -589,7 +594,7 @@ class Frame: # A wrapped row draws only its piece; an unwrapped one (piece is # None) draws all of row["segments"] (truncated at the border). segments = piece if piece is not None else row["segments"] - total = sum(len(text) for text, _ in segments) + total = sum(_disp_width(text) for text, _ in segments) if row["align"] == "left": x = inner_x + self.LIST_MARGIN + 2 * row["indent"] else: @@ -602,8 +607,8 @@ class Frame: if not text: break _addstr(scr, y, x, text, theme["bar"] if selected else attr) - x += len(text) - room -= len(text) + x += _disp_width(text) + room -= _disp_width(text) def _draw_text_row(self, y: int, row: dict, piece: Optional[str], inner_x: int, inner_w: int, selected: bool) -> None: @@ -890,7 +895,7 @@ def menu(scr, title: str, options: Sequence[tuple], default_index: int = 0, if table_rows: if table_title: frame.mark(table_title, frame.theme["dim"], align="left") - name_w = max(len(row[0]) for row in table_rows) + name_w = max(_disp_width(row[0]) for row in table_rows) for row in table_rows: name, status, kind = row[0], row[1], row[2] name_kind = row[3] if len(row) > 3 else "body" @@ -1156,7 +1161,7 @@ def form(scr, title: str, fields: Sequence[dict], field_rows: List[int] = [] # visible field index -> row index # Labels can be callables, so the pad width is recomputed from the # visible fields on every redraw (a dynamic label's length may vary). - label_w = max((len(field_label(field)) for field in shown), + label_w = max((_disp_width(field_label(field)) for field in shown), default=0) for field in shown: if field.get("note"): diff --git a/app/ui/viewkit.py b/app/ui/viewkit.py index e54aada..7619db9 100644 --- a/app/ui/viewkit.py +++ b/app/ui/viewkit.py @@ -15,7 +15,7 @@ render methods build on. import threading import time from queue import Empty, Queue -from typing import List, Optional +from typing import Callable, List, Optional from ui import tui @@ -147,7 +147,14 @@ class ScreenView: return key def _prompt_cancel(self) -> bool: - """Esc/q: confirm cancel, then wait for the worker to wind down.""" + """Esc/q: confirm cancel, then wait for the worker to wind down. + + The join is best-effort: a worker wedged in un-killable work + (a stuck subprocess, a hung network call) is left running — the + view reports the cancel and returns, and the worker's daemon + thread dies with the process. Callers must not assume the thread + has stopped (see _join_worker). + """ self._blocking() try: answer = tui.confirm(self.scr, "Cancel this step?", default=False, @@ -158,9 +165,16 @@ class ScreenView: return False self.cancelling = True self._cancel.set() - self._worker.join(timeout=60) + self._join_worker() return True + def _join_worker(self, timeout: float = 60.0) -> None: + """Join the worker if it exists and was started; never raise.""" + worker = self._worker + if worker is None or not worker.is_alive(): + return + worker.join(timeout=timeout) + def _blocking(self) -> None: """Make getch block (used while a confirm dialog owns the screen).""" try: @@ -180,6 +194,45 @@ class ScreenView: # Shared drawing primitives # ---------------------------------------------------------------------- +class LineSplitter: + """A file-like that feeds each ``\\n``/``\\r``-terminated line to a sink. + + Carriage-return progress (git/tqdm) is treated as a line terminator, + so the sink sees each progress update immediately and the last full + line always reflects the latest state. ``flush()`` emits the + unterminated tail; ``isatty()`` is False. Both the task view's + console-mirror writer and the run view's log appender build on it, + so their line-splitting cannot drift apart. + """ + + def __init__(self, sink: Callable[[str], None]): + self._sink = sink + self._buffer = "" + + def write(self, text: str) -> int: + if not text: + return 0 + self._buffer += text + while True: + cut = min((cut for cut in (self._buffer.find("\n"), + self._buffer.find("\r")) + if cut >= 0), default=-1) + if cut < 0: + break + line, self._buffer = self._buffer[:cut], self._buffer[cut + 1:] + if line: + self._sink(line) + return len(text) + + def flush(self) -> None: + if self._buffer: + self._sink(self._buffer) + self._buffer = "" + + def isatty(self) -> bool: + return False + + def _text(scr, theme, y, x, text, attr) -> None: """addstr wrapper that ignores out-of-bounds errors.""" try: @@ -188,23 +241,28 @@ def _text(scr, theme, y, x, text, attr) -> None: pass -def _box(scr, curses, theme, height, width) -> None: - """Draw the full-screen frame.""" +def _rect_box(scr, curses, theme, x: int, y: int, w: int, h: int) -> None: + """Draw a box around the rectangle ``(x, y, w, h)``.""" border = theme["border"] try: - scr.addch(0, 0, curses.ACS_ULCORNER, border) - scr.addch(0, width - 1, curses.ACS_URCORNER, border) - scr.addch(height - 1, 0, curses.ACS_LLCORNER, border) - scr.addch(height - 1, width - 1, curses.ACS_LRCORNER, border) - scr.hline(0, 1, curses.ACS_HLINE, width - 2, border) - scr.hline(height - 1, 1, curses.ACS_HLINE, width - 2, border) - for y in range(1, height - 1): - scr.addch(y, 0, curses.ACS_VLINE, border) - scr.addch(y, width - 1, curses.ACS_VLINE, border) + scr.addch(y, x, curses.ACS_ULCORNER, border) + scr.addch(y, x + w - 1, curses.ACS_URCORNER, border) + scr.addch(y + h - 1, x, curses.ACS_LLCORNER, border) + scr.addch(y + h - 1, x + w - 1, curses.ACS_LRCORNER, border) + scr.hline(y, x + 1, curses.ACS_HLINE, w - 2, border) + scr.hline(y + h - 1, x + 1, curses.ACS_HLINE, w - 2, border) + for yy in range(y + 1, y + h - 1): + scr.addch(yy, x, curses.ACS_VLINE, border) + scr.addch(yy, x + w - 1, curses.ACS_VLINE, border) except Exception: pass +def _box(scr, curses, theme, height, width) -> None: + """Draw the full-screen frame.""" + _rect_box(scr, curses, theme, 0, 0, width, height) + + def _sep(scr, curses, theme, y, width) -> None: """A horizontal separator line inside the frame.""" try: diff --git a/audiobook.py b/audiobook.py index 183bdc2..b881b6e 100755 --- a/audiobook.py +++ b/audiobook.py @@ -209,8 +209,15 @@ def _convert_each_model(*, backend: str, model_ids: list, model_voices: dict, converter._book_files = book_files converter._planned = planned ok = converter.run() - model_ok = counts["ok"] if progress is not None \ - else (len(planned) if ok else 0) + if progress is not None: + model_ok = counts["ok"] + else: + # A failed book aborts the rest, but the books that + # succeeded before it still count (run() returns False + # for the whole model either way). + model_ok = sum( + 1 for success in getattr(converter, "results", {}).values() + if success) except KeyboardInterrupt: print("\n[WARNING] Shutdown requested by user") return 130 -- cgit v1.2.3