From 077c82d9614328ff267632298da6c88009313337 Mon Sep 17 00:00:00 2001 From: historia Date: Fri, 11 Sep 2026 17:41:33 -0400 Subject: fix: consistently rename to producer --- lib/project/src/voiceforge/__init__.py | 3 - lib/project/src/voiceforge/__main__.py | 3 - lib/project/src/voiceforge/ai.py | 157 ------------------------ lib/project/src/voiceforge/ai_worker.py | 161 ------------------------- lib/project/src/voiceforge/audio.py | 128 -------------------- lib/project/src/voiceforge/cli.py | 193 ------------------------------ lib/project/src/voiceforge/config.py | 118 ------------------- lib/project/src/voiceforge/pipeline.py | 199 ------------------------------- lib/project/src/voiceforge/progress.py | 40 ------- lib/project/src/voiceforge/setup.py | 203 -------------------------------- 10 files changed, 1205 deletions(-) delete mode 100644 lib/project/src/voiceforge/__init__.py delete mode 100644 lib/project/src/voiceforge/__main__.py delete mode 100644 lib/project/src/voiceforge/ai.py delete mode 100644 lib/project/src/voiceforge/ai_worker.py delete mode 100644 lib/project/src/voiceforge/audio.py delete mode 100644 lib/project/src/voiceforge/cli.py delete mode 100644 lib/project/src/voiceforge/config.py delete mode 100644 lib/project/src/voiceforge/pipeline.py delete mode 100644 lib/project/src/voiceforge/progress.py delete mode 100644 lib/project/src/voiceforge/setup.py (limited to 'lib/project/src/voiceforge') diff --git a/lib/project/src/voiceforge/__init__.py b/lib/project/src/voiceforge/__init__.py deleted file mode 100644 index 1d8663d..0000000 --- a/lib/project/src/voiceforge/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""VoiceForge: local speech cleanup and mastering.""" - -__version__ = "0.2.0" diff --git a/lib/project/src/voiceforge/__main__.py b/lib/project/src/voiceforge/__main__.py deleted file mode 100644 index 4e28416..0000000 --- a/lib/project/src/voiceforge/__main__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .cli import main - -main() diff --git a/lib/project/src/voiceforge/ai.py b/lib/project/src/voiceforge/ai.py deleted file mode 100644 index 493be65..0000000 --- a/lib/project/src/voiceforge/ai.py +++ /dev/null @@ -1,157 +0,0 @@ -"""Automatic, isolated DeepFilterNet3 inference (Linux, Python 3.11 worker). - -Public progress callback: (stage: str, completed: float | None, total: float | None). -Known totals are audio frames or download bytes (identified by stage); -unknown totals use elapsed seconds as completed. -First use downloads Python, wheels, and upstream model weights into the -VoiceForge data directory (VOICEFORGE_HOME prefix or the XDG data location). -""" -from __future__ import annotations - -import ctypes -import fcntl -import math -import os -from pathlib import Path -import platform -import shutil -import time - -from .setup import Progress, data_dir, ensure_uv, run_process, tool_env - - -def _cuda_available() -> bool: - # Probe the driver without importing Torch or installing CUDA in the host env. - if os.environ.get("CUDA_VISIBLE_DEVICES") in {"", "-1"}: - return False - try: - driver = ctypes.CDLL("libcuda.so.1") - count = ctypes.c_int() - return driver.cuInit(0) == 0 and driver.cuDeviceGetCount(ctypes.byref(count)) == 0 and count.value > 0 - except (OSError, AttributeError): - return False - - -def _flavor(device: str) -> str: - if device not in {"auto", "cpu", "cuda"}: - raise ValueError("device must be 'auto', 'cpu', or 'cuda'") - if device == "cuda" or (device == "auto" and _cuda_available()): - if platform.machine() not in {"x86_64", "AMD64"}: - raise RuntimeError("The pinned CUDA 12.1 backend requires Linux x86_64.") - return "cu121" - return "cpu" - - -def _worker_env() -> dict[str, str]: - # Same isolation as tool_env(); kept as a named seam for worker launches. - return tool_env() - - -def _interpreter_is_local(python: Path, root: Path) -> bool: - """Whether the environment's interpreter belongs to this prefix. - - An installation duplicated by copying (while the original remained) keeps - interpreter links that resolve into the original directory. Such an - environment must be rebuilt, or the copy would silently depend on files - outside this prefix and break when the original is removed. - """ - try: - return python.resolve().is_relative_to(root) - except OSError: - return False - - -def ensure_ai(device: str = "auto", progress: Progress | None = None, - verify: bool = False) -> Path: - """Install/validate the backend and model; return its Python executable. - - CPU and CUDA environments are separate. Explicit CUDA never falls back; - auto uses Torch's availability check in the selected worker. Downloads trust - PyPI, download.pytorch.org, Astral's Python distribution, and upstream DF. - verify forces a real self-test even when the environment is marked ready - ('setup' uses this); ordinary processing trusts the marker so a ready - environment does not pay for a second model load per run. - """ - flavor = _flavor(device) - try: - libc = os.confstr("CS_GNU_LIBC_VERSION") or "" - except (ValueError, OSError): - libc = "" - if platform.system() != "Linux" or not libc.startswith("glibc ") or tuple( - int(part) for part in libc.split()[1].split(".")[:2] - ) < (2, 28): - raise RuntimeError("The pinned DeepFilterNet3 backend requires Linux with glibc >= 2.28 " - "(for example Ubuntu 22.04+ or Debian 12+). " - "Alpine/musl is not supported by its prebuilt wheels.") - root = data_dir() - root.mkdir(parents=True, exist_ok=True) - environment = root / f"ai-df-0.5.6-torch-2.5.1-{flavor}-py311-v1" - python = environment / "bin/python" - marker = environment / ".voiceforge-ready" - started = time.monotonic() - # Serialize bootstrap/model downloads, including calls from separate CLIs. - with (root / "ai.lock").open("a") as lock: - while True: - try: - fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) - break - except BlockingIOError: - if progress: - progress("Waiting for AI setup", time.monotonic() - started, None) - time.sleep(0.25) - ready = (marker.is_file() and python.is_file() - and _interpreter_is_local(python, root)) - if not ready: - # A moved, copied, or half-installed environment cannot be - # repaired in place: interpreter links reference absolute paths. - # Rebuild it; the shared wheel caches make this fast, usually - # offline. - if environment.exists(): - shutil.rmtree(environment) - uv = ensure_uv(progress) - env = _worker_env() - run_process([uv, "venv", "--python", "3.11", "--managed-python", - "--relocatable", "--allow-existing", str(environment)], - "Preparing AI Python 3.11", progress, env=env) - run_process([uv, "pip", "install", "--python", str(python), - "--index-url", f"https://download.pytorch.org/whl/{flavor}", - "torch==2.5.1", "torchaudio==2.5.1"], - f"Installing PyTorch ({flavor})", progress, env=env) - run_process([uv, "pip", "install", "--python", str(python), - "--only-binary", ":all:", - "--index-url", "https://pypi.org/simple", "deepfilternet==0.5.6", - "numpy==1.26.4", "soundfile==0.12.1"], - "Installing DeepFilterNet3", progress, env=env) - # Check a freshly built environment (and any explicit setup) with a - # real model load and inference; this also downloads the weights. - if verify or not ready: - run_process([str(python), "-I", str(Path(__file__).with_name("ai_worker.py")), - "--device", device, "--model-dir", str(root / "models/df3-0.5.6"), - "--check"], "Checking DeepFilterNet3", progress, - env=_worker_env()) - marker.touch() - return python - - -def denoise(source: Path, target: Path, device: str = "auto", strength: float = 1.0, - progress: Progress | None = None) -> None: - """Denoise mono 48 kHz WAV to float WAV, preserving the exact sample count. - - strength is a dry/wet mix in [0, 1]. The worker publishes output atomically; - errors/cancellation leave an existing target untouched. Chunked inference is - bounded-memory, not bit-identical to inference over an entire recording. - """ - if not math.isfinite(strength) or not 0 <= strength <= 1: - raise ValueError("strength must be finite and between 0 and 1") - source, target = Path(source).resolve(), Path(target).resolve() - if source == target: - raise ValueError("source and target must be different files") - if not source.is_file(): - raise FileNotFoundError(source) - if not target.parent.is_dir(): - raise FileNotFoundError(target.parent) - python = ensure_ai(device, progress) - run_process([str(python), "-I", str(Path(__file__).with_name("ai_worker.py")), - "--device", device, "--source", str(source), "--target", str(target), - "--model-dir", str(data_dir() / "models/df3-0.5.6"), - "--strength", str(strength)], "Denoising", progress, env=_worker_env()) diff --git a/lib/project/src/voiceforge/ai_worker.py b/lib/project/src/voiceforge/ai_worker.py deleted file mode 100644 index 99ec9f5..0000000 --- a/lib/project/src/voiceforge/ai_worker.py +++ /dev/null @@ -1,161 +0,0 @@ -"""Private script executed only by the isolated backend Python.""" -from __future__ import annotations - -import argparse -import hashlib -import json -import os -from pathlib import Path -import signal -import tempfile -import urllib.request -import zipfile - - -def report(stage: str, completed: float | None = None, total: float | None = None) -> None: - print("VOICEFORGE_PROGRESS " + json.dumps([stage, completed, total]), flush=True) - - -def ensure_model(directory: Path) -> Path: - """Install a release-pinned, hash-checked archive atomically under setup lock.""" - model = directory / "DeepFilterNet3" - if (model / "config.ini").is_file() and (model / "checkpoints/model_120.ckpt.best").is_file(): - return model - if directory.exists(): - raise RuntimeError(f"Incomplete model directory: {directory}. Remove it and retry AI setup.") - directory.parent.mkdir(parents=True, exist_ok=True) - url = "https://raw.githubusercontent.com/Rikorose/DeepFilterNet/v0.5.6/models/DeepFilterNet3.zip" - expected = "49c52edc8947ae1f9bf50d81530beaf3a2c3245aeaf34b6f31ff535cd22284d2" - # Release archive size is fixed alongside its hash, even without Content-Length. - expected_bytes = 7986207 - downloaded = 0 - report("Downloading DeepFilterNet3 model (bytes)", 0, expected_bytes) - with tempfile.TemporaryDirectory(prefix=".df3-", dir=directory.parent) as staging: - archive = Path(staging) / "model.zip" - digest = hashlib.sha256() - with urllib.request.urlopen(url, timeout=60) as response, archive.open("wb") as output: - while chunk := response.read(256 * 1024): - downloaded += len(chunk) - if downloaded > expected_bytes: - raise RuntimeError("DeepFilterNet3 model download exceeds the expected size") - digest.update(chunk) - output.write(chunk) - report("Downloading DeepFilterNet3 model (bytes)", downloaded, expected_bytes) - if downloaded != expected_bytes or digest.hexdigest() != expected: - raise RuntimeError("DeepFilterNet3 model checksum mismatch; refusing to load weights") - report("Verifying and extracting DeepFilterNet3 model") - unpacked = Path(staging) / "unpacked" - with zipfile.ZipFile(archive) as bundle: - for member in bundle.infolist(): - destination = (unpacked / member.filename).resolve() - if not destination.is_relative_to(unpacked.resolve()): - raise RuntimeError("Unsafe path in model archive") - bundle.extractall(unpacked) - os.replace(unpacked, directory) - return model - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--device", choices=("auto", "cpu", "cuda"), required=True) - parser.add_argument("--check", action="store_true") - parser.add_argument("--model-dir", type=Path, required=True) - parser.add_argument("--source", type=Path) - parser.add_argument("--target", type=Path) - parser.add_argument("--strength", type=float, default=1.0) - args = parser.parse_args() - - # SIGTERM from the parent takes the same cleanup path as Ctrl-C. - def terminate(signum, frame): - raise KeyboardInterrupt - - signal.signal(signal.SIGTERM, terminate) - - import numpy as np - import soundfile as sf - import torch - - torch.set_num_threads(min(4, os.cpu_count() or 1)) - available = torch.cuda.is_available() - if args.device == "cuda" and not available: - raise RuntimeError("CUDA was explicitly requested, but PyTorch cannot use it. " - "Check the NVIDIA driver and CUDA_VISIBLE_DEVICES, or select cpu.") - device = "cuda" if args.device != "cpu" and available else "cpu" - # df.utils.get_device() consults DEVICE on every call, including model creation. - os.environ["DEVICE"] = device - from df.enhance import enhance, init_df - from df.model import ModelParams - from libdf import DF - - model_path = ensure_model(args.model_dir) - report(f"Loading DeepFilterNet3 ({device})") - model, state, _ = init_df(str(model_path), log_level="ERROR", log_file=None) - if state.sr() != 48000: - raise RuntimeError("DeepFilterNet3 model must use 48 kHz") - if args.check: - # Exercise both model execution and the compiled libdf/NumPy ABI. - sample = enhance(model, state, torch.zeros(1, 4800), pad=True) - if sample.shape != (1, 4800) or not torch.isfinite(sample).all(): - raise RuntimeError("DeepFilterNet3 self-test failed") - report(f"DeepFilterNet3 ready ({device})", 1, 1) - return - if args.source is None or args.target is None: - parser.error("--source and --target are required unless --check is used") - if not np.isfinite(args.strength) or not 0 <= args.strength <= 1: - parser.error("--strength must be in [0, 1]") - - temporary = None - try: - with sf.SoundFile(args.source) as source: - if source.samplerate != 48000 or source.channels != 1 or source.format not in {"WAV", "WAVEX", "RF64"}: - raise ValueError("AI input must be a mono 48 kHz WAV; convert it before denoising") - frames = len(source) - fd, name = tempfile.mkstemp(prefix=".voiceforge-ai-", suffix=".wav", dir=args.target.parent) - os.close(fd) - temporary = Path(name) - # RF64 avoids RIFF's 4 GiB limit on very long recordings. - output_format = "RF64" if frames * 4 > 0xFFFFFFFF - 4096 else "WAV" - with sf.SoundFile(temporary, "w", samplerate=48000, channels=1, - format=output_format, subtype="FLOAT") as target: - block, context, overlap = 480000, 96000, 2400 - previous = None - params = ModelParams() - report(f"Denoising ({device})", 0, frames) - for start in range(0, frames, block): - end = min(start + block, frames) - left, right = max(0, start - context), min(frames, end + overlap + context) - source.seek(left) - audio = source.read(right - left, dtype="float32") - if not np.isfinite(audio).all(): - raise ValueError("Input contains non-finite audio samples") - if args.strength: - # Fresh STFT state per window; context warms normalization and - # recurrent layers. Pad to a whole hop before delay compensation. - state = DF(sr=params.sr, fft_size=params.fft_size, - hop_size=params.hop_size, nb_bands=params.nb_erb, - min_nb_erb_freqs=params.min_nb_freqs) - padded = np.pad(audio, (0, (-len(audio)) % params.hop_size)) - wet = enhance(model, state, torch.from_numpy(padded).unsqueeze(0), - pad=True).squeeze(0).numpy()[:len(audio)] - if len(wet) != len(audio) or not np.isfinite(wet).all(): - raise RuntimeError("DeepFilterNet3 returned invalid audio") - audio = audio * (1 - args.strength) + wet * args.strength - kept = audio[start - left:min(frames, end + overlap) - left].copy() - if previous is not None and args.strength: - n = len(previous) - fade = np.linspace(0, 1, n, dtype=np.float32) - kept[:n] = previous * (1 - fade) + kept[:n] * fade - target.write(kept[:end - start]) - previous = kept[end - start:].copy() - report(f"Denoising ({device})", end, frames) - if target.tell() != frames: - raise RuntimeError("Output frame count does not match input") - os.replace(temporary, args.target) - temporary = None - finally: - if temporary is not None: - temporary.unlink(missing_ok=True) - - -if __name__ == "__main__": - main() diff --git a/lib/project/src/voiceforge/audio.py b/lib/project/src/voiceforge/audio.py deleted file mode 100644 index 7fee0c2..0000000 --- a/lib/project/src/voiceforge/audio.py +++ /dev/null @@ -1,128 +0,0 @@ -"""Streaming measurement and FFmpeg execution. No shell interpolation.""" -from collections import deque -import json -import math -import os -from pathlib import Path -import selectors -import signal -import subprocess -import time - -import numpy as np -import soundfile as sf - -from .setup import terminate_group - - -def ffmpeg_run(ffmpeg: str, args: list[str], stage: str, duration: float, - progress) -> str: - command = [ffmpeg, "-hide_banner", "-nostdin", "-y", "-nostats", - "-progress", "pipe:1"] + args - progress(stage, 0, duration or None) - diagnostics = deque(maxlen=128) - pending = b"" - completed = 0.0 - last_advance = time.monotonic() - with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - start_new_session=True) as process: - try: - with selectors.DefaultSelector() as selector: - for stream in (process.stdout, process.stderr): - os.set_blocking(stream.fileno(), False) - selector.register(stream, selectors.EVENT_READ) - while selector.get_map() or process.poll() is None: - for key, _ in selector.select(timeout=0.2): - data = os.read(key.fd, 8192) - if not data: - selector.unregister(key.fileobj) - continue - if key.fileobj is process.stderr: - diagnostics.append(data) - else: - pending += data - while b"\n" in pending: - line, pending = pending.split(b"\n", 1) - if line.startswith(b"out_time_us="): - try: - value = max(0.0, int(line.split(b"=", 1)[1]) / 1e6) - except ValueError: - continue - if value > completed: - last_advance = time.monotonic() - completed = value - label = stage - if time.monotonic() - last_advance > 60: - label += " (no new audio progress for 60s; still waiting)" - progress(label, min(completed, duration), duration or None) - code = process.wait() - output = b"".join(diagnostics).decode(errors="replace") - if code: - raise RuntimeError(f"{stage} failed (exit {code}):\n{output[-8000:]}") - progress(stage, duration, duration or None) - return output - except BaseException: - terminate_group(process) - raise - - -def measure(path: Path, progress, stage="Analyzing audio") -> dict: - """100ms RMS histogram is bounded in memory, including multi-hour input.""" - with sf.SoundFile(path) as source: - if source.channels not in (1, 2): - raise ValueError("Only mono or stereo voice recordings are supported") - if not len(source): - raise ValueError("The recording is empty") - peaks = np.zeros(source.channels) - sums = np.zeros(source.channels) - squares = np.zeros(source.channels) - clipped = 0 - histogram = np.zeros(121, dtype=np.int64) - frames = 0 - progress(stage, 0, len(source)) - for audio in source.blocks(blocksize=max(1, source.samplerate // 10), - dtype="float64", always_2d=True): - if not np.isfinite(audio).all(): - raise ValueError("Recording contains non-finite samples") - peaks = np.maximum(peaks, np.max(np.abs(audio), axis=0)) - sums += audio.sum(axis=0) - squares += (audio * audio).sum(axis=0) - clipped += int(np.count_nonzero(np.abs(audio) >= 0.9999)) - rms = float(np.sqrt(np.mean(audio * audio))) - db = 20 * math.log10(max(rms, 1e-6)) - histogram[int(np.clip(round(db) + 120, 0, 120))] += 1 - frames += len(audio) - progress(stage, frames, len(source)) - cumulative = histogram.cumsum() - def percentile(p): - return int(np.searchsorted(cumulative, max(1, math.ceil(cumulative[-1] * p)))) - 120 - quiet, speech = percentile(0.1), percentile(0.8) - if speech - quiet < 15: - # Pause-dominated recordings: the 80th percentile of all blocks - # sits in room tone, so estimate the level from the loud tail - # instead of mistaking sparse active speech for silence. - speech = max(speech, percentile(0.98)) - noise_available = speech - quiet >= 15 and quiet > -110 - return {"frames": frames, "duration_seconds": frames / source.samplerate, - "sample_rate": source.samplerate, "channels": source.channels, - "subtype": source.subtype, - "sample_peak_dbfs": float(20 * np.log10(max(float(peaks.max()), 1e-12))), - "channel_rms_dbfs": (20 * np.log10(np.maximum(np.sqrt(squares / frames), 1e-12))).tolist(), - "dc_offset": (sums / frames).tolist(), "near_full_scale_samples": clipped, - "speech_level_estimate_dbfs": speech, - "quiet_blocks_dbfs": quiet, - "noise_floor_estimate_dbfs": quiet if noise_available else None, - "noise_floor_confidence": "low (quiet-block heuristic, not speech recognition)" if noise_available else "insufficient room tone"} - - -def loudness(ffmpeg, source, settings, duration, progress, stage="Measuring loudness"): - output = ffmpeg_run(ffmpeg, ["-i", str(source), "-map", "0:a:0", "-af", - f"loudnorm=I={settings.target_lufs}:TP={settings.true_peak_db}:LRA={settings.loudness_range}:print_format=json", - "-f", "null", "-"], stage, duration, progress) - start, end = output.rfind("{"), output.rfind("}") - if start < 0 or end < start: - raise RuntimeError("FFmpeg did not return loudness measurements") - raw = json.loads(output[start:end + 1]) - return {key: (float(value) if math.isfinite(float(value)) else None) - for key, value in raw.items() if key != "normalization_type"} | { - "normalization_type": raw.get("normalization_type")} diff --git a/lib/project/src/voiceforge/cli.py b/lib/project/src/voiceforge/cli.py deleted file mode 100644 index 52cd1bd..0000000 --- a/lib/project/src/voiceforge/cli.py +++ /dev/null @@ -1,193 +0,0 @@ -"""Command-line entry point; configuration errors fail before model downloads.""" -import argparse -from dataclasses import asdict, fields -import json -import math -import os -from pathlib import Path -import subprocess -import sys -import tempfile -import tomllib - -from rich.console import Console - -from . import __version__ -from .audio import ffmpeg_run, loudness, measure -from .config import PROFILES, Settings, resolve, toml -from .pipeline import check_target, process, targets_for -from .progress import Display -from .setup import data_dir, ensure_ffmpeg - - -def parser(): - app = argparse.ArgumentParser(prog="producer", description="Local speech cleanup and podcast mastering. Originals are never overwritten.") - app.add_argument("--version", action="version", version=__version__) - sub = app.add_subparsers(dest="command", required=True) - for name in ("process", "preview", "config", "analyze"): - cmd = sub.add_parser(name) - cmd.add_argument("--config", type=Path, help="TOML configuration file") - cmd.add_argument("--profile", choices=PROFILES, default=None) - cmd.add_argument("--set", action="append", default=[], metavar="KEY=VALUE", help="Override any setting using TOML value syntax") - for field in fields(Settings): - if field.name == "profile": - continue - default = getattr(Settings(), field.name) - option = "--" + field.name.replace("_", "-") - if isinstance(default, bool): - cmd.add_argument(option, action=argparse.BooleanOptionalAction, default=None) - else: - cmd.add_argument(option, type=type(default), default=None) - cmd.add_argument("--no-denoise", dest="denoiser", action="store_const", const="none") - if name == "config": - cmd.add_argument("--json", action="store_true") - else: - cmd.add_argument("inputs", type=Path, nargs="+" if name == "process" else 1) - if name in ("process", "preview"): - cmd.add_argument("--output-dir", type=Path, default=None, - help="Directory for generated outputs (default: beside each input)") - cmd.add_argument("--overwrite", action="store_true", help="Replace generated outputs, never input files") - if name == "process": - cmd.add_argument("-o", "--output", type=Path, help="Explicit WAV destination (one input only)") - if name == "preview": - cmd.add_argument("--start", type=float, default=0) - cmd.add_argument("--duration", type=float, default=30) - cmd.add_argument("--profiles", nargs="+", choices=PROFILES, default=["natural", "narrator", "radio"]) - sub.add_parser("doctor", help="Check tools and driver without downloading AI") - setup = sub.add_parser("setup", help="Download and self-test the isolated AI backend") - setup.add_argument("--device", choices=("auto", "cpu", "cuda"), default="auto") - return app - - -def settings_for(args, profile=None): - overrides = {field.name: getattr(args, field.name) for field in fields(Settings) - if field.name != "profile" and getattr(args, field.name, None) is not None} - for item in args.set: - key, sep, value = item.partition("=") - if not sep: - raise ValueError("--set expects KEY=VALUE") - # Strings can be supplied without TOML quotes for CLI convenience. - try: - parsed = tomllib.loads("value = " + value)["value"] - except tomllib.TOMLDecodeError: - parsed = value - overrides[key.strip()] = parsed - return resolve(args.config, profile or args.profile, overrides) - - -def require_input(source: Path) -> None: - if not source.is_file() or source.suffix.lower() != ".wav": - raise ValueError(f"Input must be an existing WAV file: {source}") - - -def main(): - app = parser() - args = app.parse_args() - console = Console(stderr=True) - try: - if args.command == "doctor": - from .ai import _cuda_available - ffmpeg = ensure_ffmpeg() - version = subprocess.run([ffmpeg, "-version"], capture_output=True, text=True, check=True).stdout.splitlines()[0] - console.print(version, markup=False) - console.print(f"FFmpeg: {ffmpeg}\nData: {data_dir()}\nCUDA driver available: {_cuda_available()}\nAI is installed and self-tested by './producer.sh setup'.", markup=False) - return - if args.command == "setup": - from .ai import ensure_ai - with Display() as display: - ensure_ffmpeg() - python = ensure_ai(args.device, display.update, verify=True) - console.print(f"AI ready: {python}", markup=False) - return - settings = settings_for(args) - if args.command == "config": - print(json.dumps(asdict(settings), indent=2) if args.json else toml(settings), end="\n" if args.json else "") - return - with Display() as display: - if args.command == "analyze": - stats = measure(args.inputs[0], display.update) - stats["loudness"] = loudness(ensure_ffmpeg(), args.inputs[0], settings, stats["duration_seconds"], display.update) - print(json.dumps(stats, indent=2, allow_nan=False)) - return - if args.command == "process": - if args.output and len(args.inputs) != 1: - raise ValueError("--output requires exactly one input") - if args.output and args.output_dir: - raise ValueError("--output and --output-dir cannot be combined") - # Without an explicit destination, outputs are written beside - # each input; --output-dir keeps every result in one directory. - outputs = [args.output or (args.output_dir or p.parent) / f"{p.stem}.{settings.profile}.wav" - for p in args.inputs] - # Preflight the whole plan so a predictable conflict cannot - # fail a batch after earlier inputs have already been published. - sources = {p.resolve() for p in args.inputs} - claimed: set[Path] = set() - for source, output in zip(args.inputs, outputs): - require_input(source) - targets = targets_for(output.absolute(), settings) - resolved = {target.resolve() for target in targets} - if len(resolved) < len(targets) or resolved & claimed: - raise ValueError("Inputs have duplicate output names; process them separately with --output") - claimed |= resolved - if resolved & sources or any( - target.exists() and os.path.samefile(source, target) for target in targets): - raise ValueError("An output would overwrite an input recording") - for target in targets: - check_target(source, target, args.overwrite) - for source, output in zip(args.inputs, outputs): - report = process(source, output, settings, display.update, args.overwrite) - console.print(f"Written: {output} | {report['master_loudness']['input_i']} LUFS | {report['master_loudness']['input_tp']} dBTP", markup=False) - for warning in report["warnings"]: - console.print(f"Warning: {warning}", markup=False) - return - if not math.isfinite(args.start) or args.start < 0 or not 0 < args.duration <= 300: - raise ValueError("Preview start must be >= 0 and duration must be in (0, 300] seconds") - if len(set(args.profiles)) != len(args.profiles): - raise ValueError("Preview profiles must be unique") - if any(item.partition("=")[0].strip() == "profile" for item in args.set): - raise ValueError("Preview profiles come from --profiles; remove the profile override from --set") - source = args.inputs[0] - require_input(source) - original = source.resolve() - output_dir = args.output_dir or source.parent - # Preflight every preview destination before extracting the excerpt. - claimed: set[Path] = set() - for profile in args.profiles: - output = output_dir / f"{source.stem}.{profile}.preview.wav" - if output.resolve() == original: - raise ValueError("Preview output would overwrite the original") - targets = [output, output.with_suffix(".report.json")] - resolved = {target.resolve() for target in targets} - if len(resolved) < len(targets) or resolved & claimed: - raise ValueError("Preview outputs have duplicate names") - claimed |= resolved - for target in targets: - check_target(original, target, args.overwrite) - output_dir.mkdir(parents=True, exist_ok=True) - with tempfile.TemporaryDirectory(prefix=".preview-", dir=output_dir) as tmp: - excerpt = Path(tmp) / "excerpt.wav" - ffmpeg_run(ensure_ffmpeg(), ["-ss", str(args.start), "-i", str(original), "-t", str(args.duration), - "-map", "0:a:0", "-c:a", "pcm_f32le", str(excerpt)], "Extracting preview", args.duration, display.update) - for profile in args.profiles: - configured = settings_for(args, profile) - # Equal loudness prevents a louder preset winning an unfair A/B. - configured.normalize = True - configured.limiter = True - output = output_dir / f"{source.stem}.{profile}.preview.wav" - report = process(excerpt, output, configured, display.update, args.overwrite, - provenance={"original": str(original), "preview_start_seconds": args.start, - "requested_duration_seconds": args.duration, - "normalization_forced": True}) - console.print(f"Preview: {output} | {report['master_loudness']['input_i']} LUFS | {report['master_loudness']['input_tp']} dBTP", markup=False) - for warning in report["warnings"]: - console.print(f"Warning: {warning}", markup=False) - except KeyboardInterrupt: - console.print("Cancelled; temporary files removed. Completed outputs may remain.") - raise SystemExit(130) - except (ValueError, OSError, RuntimeError, subprocess.SubprocessError) as error: - console.print(f"Error: {error}", markup=False) - raise SystemExit(1) - - -if __name__ == "__main__": - main() diff --git a/lib/project/src/voiceforge/config.py b/lib/project/src/voiceforge/config.py deleted file mode 100644 index 9cbd59d..0000000 --- a/lib/project/src/voiceforge/config.py +++ /dev/null @@ -1,118 +0,0 @@ -"""Flat, strict TOML configuration; CLI overrides are applied last.""" -from dataclasses import asdict, dataclass, fields -import math -from pathlib import Path -import tomllib - - -@dataclass -class Settings: - profile: str = "natural" - denoiser: str = "deepfilter" - device: str = "auto" - denoise_strength: float = 0.85 - fft_reduction_db: float = 8.0 - channel: str = "auto" - highpass: bool = True - highpass_hz: float = 70.0 - lowpass: bool = False - lowpass_hz: float = 16000.0 - hum_hz: int = 0 - leveling: bool = True - level_target_db: float = -23.0 - max_gain_db: float = 18.0 - expansion: bool = False - expansion_threshold_db: float = -50.0 - expansion_ratio: float = 1.5 - expansion_range_db: float = 12.0 - eq: bool = True - warmth_db: float = 0.5 - mud_db: float = -1.5 - presence_db: float = 1.0 - compression: bool = True - compressor_threshold_db: float = -21.0 - compressor_ratio: float = 2.0 - compressor_attack_ms: float = 15.0 - compressor_release_ms: float = 150.0 - deess: bool = True - deess_intensity: float = 0.15 - deess_amount: float = 0.4 - normalize: bool = True - target_lufs: float = -19.0 - loudness_range: float = 9.0 - limiter: bool = True - true_peak_db: float = -1.5 - sample_rate: int = 48000 - mp3: bool = False - mp3_bitrate: int = 192 - - -PROFILES = { - "natural": {}, - "narrator": {"compressor_ratio": 1.6, "compressor_attack_ms": 25.0, - "presence_db": 0.5, "warmth_db": 1.0, "loudness_range": 11.0}, - "radio": {"compressor_ratio": 3.5, "compressor_threshold_db": -24.0, - "compressor_attack_ms": 8.0, "compressor_release_ms": 100.0, - "warmth_db": 2.0, "mud_db": -2.0, "presence_db": 2.0, - "deess_intensity": 0.25, "loudness_range": 6.0}, - "cleanup-only": {"leveling": False, "eq": False, "compression": False, - "deess": False, "normalize": False, "limiter": False}, -} - -RANGES = { - "denoise_strength": (0, 1), "fft_reduction_db": (0.01, 30), - "highpass_hz": (20, 300), "lowpass_hz": (4000, 22000), - "level_target_db": (-40, -12), "max_gain_db": (0, 30), - "expansion_threshold_db": (-90, -20), "expansion_ratio": (1, 4), - "expansion_range_db": (0, 40), "warmth_db": (-12, 12), - "mud_db": (-12, 12), "presence_db": (-12, 12), - "compressor_threshold_db": (-50, -5), "compressor_ratio": (1, 10), - "compressor_attack_ms": (0.1, 200), "compressor_release_ms": (10, 2000), - "deess_intensity": (0, 1), "deess_amount": (0, 1), - "target_lufs": (-30, -12), "loudness_range": (1, 20), - "true_peak_db": (-9, -0.5), -} - - -def resolve(path: Path | None = None, profile: str | None = None, - overrides: dict | None = None) -> Settings: - values = tomllib.loads(path.read_text()) if path else {} - chosen = (overrides or {}).get("profile", profile or values.get("profile", "natural")) - if not isinstance(chosen, str) or chosen not in PROFILES: - raise ValueError(f"Unknown profile {chosen!r}; choose {', '.join(PROFILES)}") - merged = asdict(Settings()) | PROFILES[chosen] | values | {"profile": chosen} - merged.update(overrides or {}) - defaults = asdict(Settings()) - unknown = merged.keys() - defaults.keys() - if unknown: - raise ValueError(f"Unknown setting(s): {', '.join(sorted(unknown))}") - for key, value in merged.items(): - expected = type(defaults[key]) - if expected is float: - if type(value) not in (int, float) or not math.isfinite(value): - raise ValueError(f"{key} must be a finite number") - elif type(value) is not expected: - raise ValueError(f"{key} must be {expected.__name__}") - if key in RANGES and not RANGES[key][0] <= value <= RANGES[key][1]: - raise ValueError(f"{key} must be between {RANGES[key][0]} and {RANGES[key][1]}") - for key, options in { - "denoiser": ("deepfilter", "fft", "none"), "device": ("auto", "cpu", "cuda"), - "channel": ("auto", "left", "right", "mix"), "hum_hz": (0, 50, 60), - "sample_rate": (44100, 48000), "mp3_bitrate": (128, 160, 192, 224, 256, 320), - }.items(): - if merged[key] not in options: - raise ValueError(f"{key} must be one of {options}") - if merged["lowpass"] and merged["lowpass_hz"] >= merged["sample_rate"] / 2: - raise ValueError("lowpass_hz must be below the output Nyquist frequency") - if merged["normalize"] and not merged["limiter"]: - raise ValueError("Normalization includes true-peak limiting; use --no-normalize to bypass it") - return Settings(**merged) - - -def toml(settings: Settings) -> str: - lines = ["# VoiceForge configuration. CLI flags override these values."] - for field in fields(settings): - value = getattr(settings, field.name) - text = str(value).lower() if isinstance(value, bool) else repr(value) - lines.append(f"{field.name} = {text}") - return "\n".join(lines) + "\n" diff --git a/lib/project/src/voiceforge/pipeline.py b/lib/project/src/voiceforge/pipeline.py deleted file mode 100644 index a143dcf..0000000 --- a/lib/project/src/voiceforge/pipeline.py +++ /dev/null @@ -1,199 +0,0 @@ -"""Offline mono voice mastering with float intermediates and atomic outputs.""" -from dataclasses import asdict -import fcntl -import json -import math -import os -from pathlib import Path -import tempfile - -from . import __version__ -from .audio import ffmpeg_run, loudness, measure -from .config import Settings -from .setup import ensure_ffmpeg - - -def filters(settings: Settings) -> list[str]: - s = settings - result = [] - if s.expansion: - result.append(f"agate=threshold={10 ** (s.expansion_threshold_db / 20)}:ratio={s.expansion_ratio}:range={10 ** (-s.expansion_range_db / 20)}:attack=10:release=250:detection=rms") - if s.eq: - result += [f"equalizer=f=140:t=q:w=0.7:g={s.warmth_db}", - f"equalizer=f=300:t=q:w=0.8:g={s.mud_db}", - f"equalizer=f=3500:t=q:w=0.7:g={s.presence_db}"] - if s.compression: - result.append(f"acompressor=threshold={10 ** (s.compressor_threshold_db / 20)}:ratio={s.compressor_ratio}:attack={s.compressor_attack_ms}:release={s.compressor_release_ms}:knee=4:makeup=1:detection=rms") - if s.deess: - result.append(f"deesser=i={s.deess_intensity}:m={s.deess_amount}:f=0.5") - if s.lowpass: - result.append(f"lowpass=f={s.lowpass_hz}:p=2") - return result or ["anull"] - - -def targets_for(destination: Path, settings: Settings) -> list[Path]: - """Every path a successful run publishes: master WAV, report, optional MP3.""" - targets = [destination, destination.with_suffix(".report.json")] - if settings.mp3: - targets.append(destination.with_suffix(".mp3")) - return targets - - -def check_target(source: Path, target: Path, overwrite: bool) -> None: - """Preflight one published path; the original recording is never touched.""" - if target.resolve() == source or (target.exists() and os.path.samefile(source, target)): - raise ValueError("Refusing to overwrite the original recording") - # lexists also rejects dangling symlinks, which publication would only hit late. - if os.path.lexists(target) and not overwrite: - raise FileExistsError(f"Output exists: {target}; use --overwrite to replace outputs") - if target.exists() and not target.is_file(): - raise ValueError(f"Output is not a regular file: {target}") - - -def publish(source: Path, target: Path, overwrite: bool): - if overwrite: - os.replace(source, target) - else: - # Link is atomic and fails if another process created the destination. - os.link(source, target) - source.unlink() - - -def process(source: Path, destination: Path, settings: Settings, progress, - overwrite=False, provenance: dict | None = None) -> dict: - source, destination = source.resolve(), destination.absolute() - if not source.is_file() or source.suffix.lower() != ".wav": - raise ValueError(f"Input must be an existing WAV file: {source}") - if destination.suffix.lower() != ".wav": - raise ValueError("Master output must have a .wav extension") - report_path = destination.with_suffix(".report.json") - for target in targets_for(destination, settings): - check_target(source, target, overwrite) - ffmpeg = ensure_ffmpeg() - original = measure(source, progress, "Analyzing original") - if original["sample_peak_dbfs"] < -100: - raise ValueError("Recording is silent or too quiet to master safely") - channel = settings.channel - if original["channels"] == 2 and channel == "auto": - raise ValueError("Stereo input: select --channel left, right, or mix explicitly. Auto downmix could cancel your voice.") - if original["channels"] == 1 and channel == "right": - raise ValueError("Cannot select the right channel of a mono recording") - duration = original["duration_seconds"] - warnings = [] - if original["near_full_scale_samples"]: - warnings.append("Input has near-full-scale samples: inspect for clipping; lost peaks cannot be reliably restored.") - if max(abs(x) for x in original["dc_offset"]) > 0.01: - warnings.append("Significant input DC offset detected; enable highpass filtering to remove it.") - destination.parent.mkdir(parents=True, exist_ok=True) - with tempfile.TemporaryDirectory(prefix=".voiceforge-", dir=destination.parent) as temporary: - work = Path(temporary) - prepared, cleaned, shaped = [work / name for name in ("prepared.wav", "cleaned.wav", "shaped.wav")] - input_filters = [] - if original["channels"] == 2: - input_filters.append({"left": "pan=mono|c0=c0", "right": "pan=mono|c0=c1", - "mix": "pan=mono|c0=0.5*c0+0.5*c1"}[channel]) - if settings.highpass: - input_filters.append(f"highpass=f={settings.highpass_hz}:p=2") - if settings.hum_hz: - input_filters += [f"equalizer=f={settings.hum_hz * n}:t=q:w=25:g=-18" for n in (1, 2, 3)] - ffmpeg_run(ffmpeg, ["-i", str(source), "-map", "0:a:0", "-af", - ",".join(input_filters or ["anull"]), "-ar", "48000", "-ac", "1", - "-c:a", "pcm_f32le", "-rf64", "auto", str(prepared)], "Preparing mono 48 kHz audio", duration, progress) - if settings.denoiser == "deepfilter" and settings.denoise_strength > 0: - from .ai import denoise - denoise(prepared, cleaned, settings.device, settings.denoise_strength, progress) - elif settings.denoiser == "fft" and settings.denoise_strength > 0: - ffmpeg_run(ffmpeg, ["-i", str(prepared), "-af", - f"afftdn=nr={max(0.01, settings.fft_reduction_db * settings.denoise_strength)}:tn=1", - "-c:a", "pcm_f32le", "-rf64", "auto", str(cleaned)], "FFT noise reduction", duration, progress) - else: - cleaned = prepared - clean_stats = measure(cleaned, progress, "Measuring cleaned voice") - if clean_stats["sample_peak_dbfs"] < -100: - raise ValueError("The selected audio is silent or too quiet to master safely; " - "check --channel, the mix, and cleanup settings") - gain = 0.0 - if settings.leveling: - speech_db = clean_stats["speech_level_estimate_dbfs"] - if speech_db <= -110: - warnings.append("Speech level estimate is unreliable; leveling skipped. " - "Inspect levels or rely on normalization.") - else: - desired = settings.level_target_db - speech_db - gain = max(-settings.max_gain_db, min(settings.max_gain_db, desired)) - if abs(desired) > settings.max_gain_db: - warnings.append("Input level correction reached its configured gain limit.") - chain = ([f"volume={gain}dB"] if gain else []) + filters(settings) - ffmpeg_run(ffmpeg, ["-i", str(cleaned), "-af", ",".join(chain), - "-c:a", "pcm_f32le", "-rf64", "auto", str(shaped)], "EQ, dynamics and de-essing", duration, progress) - before = loudness(ffmpeg, shaped, settings, duration, progress, "Loudness analysis (pass 1)") - if settings.normalize and any(before.get(k) is None for k in ("input_i", "input_tp", "input_lra", "input_thresh", "target_offset")): - raise ValueError("Cannot normalize: insufficient measurable audio after processing") - final_filters = [] - if settings.normalize: - final_filters.append(f"loudnorm=I={settings.target_lufs}:TP={settings.true_peak_db}:LRA={settings.loudness_range}:" - f"measured_I={before['input_i']}:measured_TP={before['input_tp']}:measured_LRA={before['input_lra']}:" - f"measured_thresh={before['input_thresh']}:offset={before['target_offset']}:linear=true:print_format=json") - elif settings.limiter: - final_filters += ["aresample=192000", f"alimiter=limit={10 ** (settings.true_peak_db / 20)}:level=false:latency=true"] - final_filters.append(f"aresample={settings.sample_rate}:output_sample_bits=24:dither_method=triangular") - master = work / "master.wav" - render_log = ffmpeg_run(ffmpeg, ["-i", str(shaped), "-af", ",".join(final_filters), - "-ar", str(settings.sample_rate), "-c:a", "pcm_s24le", "-rf64", "auto", str(master)], - "Mastering (pass 2)", duration, progress) - # Bounded disk use: drop consumed intermediates instead of waiting for - # the temporary directory teardown at the end of the run. - shaped.unlink(missing_ok=True) - for intermediate in {prepared, cleaned}: - intermediate.unlink(missing_ok=True) - normalization_mode = "disabled" - if settings.normalize: - start, end = render_log.rfind("{"), render_log.rfind("}") - if start < 0 or end < start: - raise RuntimeError("FFmpeg did not report the applied normalization mode") - normalization_mode = json.loads(render_log[start:end + 1])["normalization_type"] - final = loudness(ffmpeg, master, settings, duration, progress, "Verifying exported master") - final_stats = measure(master, progress, "Checking exported samples") - if abs(final_stats["duration_seconds"] - duration) > 0.02: - raise RuntimeError("Output duration differs from input by more than 20 ms") - if (final_stats["channels"] != 1 or final_stats["sample_rate"] != settings.sample_rate - or final_stats["subtype"] != "PCM_24"): - raise RuntimeError("Exported master is not the expected mono PCM_24 WAV") - if settings.normalize and (final["input_i"] is None or abs(final["input_i"] - settings.target_lufs) > 0.5): - warnings.append("Master misses the loudness target by more than 0.5 LU; inspect the report before publishing.") - if settings.limiter and final["input_tp"] is not None and final["input_tp"] > settings.true_peak_db + 0.1: - warnings.append("Master exceeds the true-peak target by more than 0.1 dB; inspect before publishing.") - if final_stats["near_full_scale_samples"]: - warnings.append("Export has near-full-scale samples. Enable limiting or lower gains.") - mp3_stats = None - if settings.mp3: - mp3 = work / "delivery.mp3" - ffmpeg_run(ffmpeg, ["-i", str(master), "-c:a", "libmp3lame", "-b:a", - f"{settings.mp3_bitrate}k", str(mp3)], "Encoding MP3", duration, progress) - mp3_stats = loudness(ffmpeg, mp3, settings, duration, progress, "Verifying decoded MP3") - if mp3_stats["input_tp"] is not None and mp3_stats["input_tp"] > settings.true_peak_db + 0.1: - warnings.append("MP3 encoding increased true peak beyond the configured ceiling; use more headroom.") - report = {"voiceforge_version": __version__, "source": str(source), - "output": str(destination), "settings": asdict(settings), "input": original, - "cleaned": clean_stats, "level_correction_db": gain, - "pre_master_loudness": before, "master_loudness": final, - "master": final_stats, "mp3_loudness": mp3_stats, - "normalization_mode": normalization_mode, - "warnings": warnings} - if provenance: - report["provenance"] = provenance - staged_report = work / "report.json" - staged_report.write_text(json.dumps(report, indent=2, allow_nan=False) + "\n") - # Publish only after all processing and verification succeeded. The - # directory lock serializes publication so concurrent jobs targeting - # the same output directory cannot interleave WAV/report/MP3 sets. - publication_fd = os.open(destination.parent, os.O_RDONLY) - try: - fcntl.flock(publication_fd, fcntl.LOCK_EX) - if settings.mp3: - publish(mp3, destination.with_suffix(".mp3"), overwrite) - publish(staged_report, report_path, overwrite) - publish(master, destination, overwrite) - finally: - os.close(publication_fd) - return report diff --git a/lib/project/src/voiceforge/progress.py b/lib/project/src/voiceforge/progress.py deleted file mode 100644 index a8c5476..0000000 --- a/lib/project/src/voiceforge/progress.py +++ /dev/null @@ -1,40 +0,0 @@ -"""A refresh thread keeps elapsed time moving even during native processing.""" -import time - -from rich.console import Console -from rich.progress import (BarColumn, Progress, SpinnerColumn, TaskProgressColumn, - TextColumn, TimeElapsedColumn, TimeRemainingColumn) - - -class Display: - def __init__(self): - self.console = Console(stderr=True) - self.progress = Progress(SpinnerColumn(), TextColumn("{task.description}"), - BarColumn(), TaskProgressColumn(), TimeElapsedColumn(), - TimeRemainingColumn(), console=self.console, - refresh_per_second=5, disable=not self.console.is_terminal) - self.task = None - self.stage = "" - self.started = time.monotonic() - self.last_print = 0.0 - - def __enter__(self): - self.progress.start() - return self - - def __exit__(self, *args): - self.progress.stop() - - def update(self, stage, completed=None, total=None): - changed = stage != self.stage - if changed: - if self.task is not None: - self.progress.remove_task(self.task) - self.task = self.progress.add_task(stage, total=total) - self.stage = stage - self.progress.update(self.task, completed=completed if total else 0, total=total) - now = time.monotonic() - if not self.console.is_terminal and (changed or now - self.last_print >= 5): - fraction = f" {min(100, 100 * (completed or 0) / total):.0f}%" if total else "" - self.console.print(f"[{now - self.started:7.1f}s] {stage}{fraction}", markup=False) - self.last_print = now diff --git a/lib/project/src/voiceforge/setup.py b/lib/project/src/voiceforge/setup.py deleted file mode 100644 index 9afeb44..0000000 --- a/lib/project/src/voiceforge/setup.py +++ /dev/null @@ -1,203 +0,0 @@ -"""User-local tools. No sudo and no changes to the application's interpreter.""" -from __future__ import annotations - -from collections import deque -from collections.abc import Callable -import json -import os -from pathlib import Path -import selectors -import shutil -import signal -import subprocess -import sys -import time - -Progress = Callable[[str, float | None, float | None], None] - - -def data_dir() -> Path: - """The VoiceForge data/prefix directory. - - VOICEFORGE_HOME wins (the single-directory launcher exports it), then - XDG_DATA_HOME/voiceforge, then ~/.local/share/voiceforge. A relative - VOICEFORGE_HOME resolves against the current directory. - """ - home = os.environ.get("VOICEFORGE_HOME") - if home: - return Path(home).absolute() - root = Path(os.environ.get("XDG_DATA_HOME", str(Path.home() / ".local/share"))) - if not root.is_absolute(): - root = Path.home() / ".local/share" - return root / "voiceforge" - - -def tool_env() -> dict[str, str]: - """Sanitized environment for bootstrap, setup, and worker subprocesses. - - Host Python settings are removed so a virtualenv or sitecustomize on the - machine cannot leak into managed environments. Installation-target, - index, and constraint overrides are dropped and package-manager - configuration files disabled, so packages can only come from the - explicitly passed trusted indexes and land inside the data directory; - every cache (pip, uv, managed Python, XDG, temporary files) stays there - too. - """ - env = os.environ.copy() - for name in ("PYTHONPATH", "PYTHONHOME", "VIRTUAL_ENV", - "PIP_TARGET", "PIP_PREFIX", "PIP_USER", "PIP_FIND_LINKS", - "PIP_INDEX_URL", "PIP_EXTRA_INDEX_URL", "PIP_CONSTRAINT", - "UV_INDEX_URL", "UV_DEFAULT_INDEX", "UV_EXTRA_INDEX_URL", - "UV_INDEX", "UV_FIND_LINKS", "UV_CONSTRAINT"): - env.pop(name, None) - env["PYTHONNOUSERSITE"] = "1" - env["PIP_CONFIG_FILE"] = os.devnull - env["UV_NO_CONFIG"] = "1" - cache = data_dir() / "cache" - env["XDG_CACHE_HOME"] = str(cache) - env["PIP_CACHE_DIR"] = str(cache / "pip") - env["UV_CACHE_DIR"] = str(cache / "uv") - env["UV_PYTHON_INSTALL_DIR"] = str(data_dir() / "uv/python") - tmp = data_dir() / "tmp" - try: - tmp.mkdir(parents=True, exist_ok=True) - env["TMPDIR"] = str(tmp) - except OSError: - pass # Best effort: never break subprocesses over temporary-file containment. - return env - - -def terminate_group(process: subprocess.Popen) -> None: - """SIGTERM the child's whole process group, escalate to SIGKILL, and reap. - - The group is signalled even when the leader has already exited, because - descendants can outlive it while still holding the output pipe open. - A race between exit and signalling is tolerated. - """ - for sig in (signal.SIGTERM, signal.SIGKILL): - try: - os.killpg(process.pid, sig) - except ProcessLookupError: - pass - try: - process.wait(timeout=5) - except subprocess.TimeoutExpired: - continue - try: - os.killpg(process.pid, 0) - except ProcessLookupError: - return - process.wait() - - -def run_process(args: list[str], stage: str, progress: Progress | None = None, - *, env: dict[str, str] | None = None) -> None: - """Drain output without blocking; send heartbeat callbacks every 0.25s. - - Worker JSON records carry frame/byte counts. Other processes report elapsed seconds - as completed with total=None. Only the last 64 KiB of diagnostics are retained. - Callback exceptions (including cancellation) terminate the process group. - """ - started = time.monotonic() - current_stage, completed, total = stage, None, None - diagnostics: deque[bytes] = deque(maxlen=16) - pending = b"" - if progress: - progress(stage, None, None) - with subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - env=env, start_new_session=True) as process: - assert process.stdout is not None - os.set_blocking(process.stdout.fileno(), False) - try: - with selectors.DefaultSelector() as selector: - selector.register(process.stdout, selectors.EVENT_READ) - while selector.get_map() or process.poll() is None: - for key, _ in selector.select(timeout=0.25): - chunk = os.read(key.fd, 4096) - if not chunk: - selector.unregister(key.fileobj) - continue - diagnostics.append(chunk) - pending += chunk - while b"\n" in pending: - line, pending = pending.split(b"\n", 1) - if line.startswith(b"VOICEFORGE_PROGRESS "): - event = json.loads(line[len(b"VOICEFORGE_PROGRESS "):]) - current_stage, completed, total = event - if len(pending) > 65536: - pending = pending[-4096:] - if progress: - progress(current_stage, completed if total is not None else - time.monotonic() - started, total) - code = process.wait() - if code: - detail = b"".join(diagnostics).decode("utf-8", errors="replace") - raise RuntimeError(f"{stage} failed (exit {code}):\n{detail}") - except BaseException: - terminate_group(process) - raise - - -def _uv_supports_relocatable(uv: str) -> bool: - """Whether this uv understands the venv flags AI setup relies on. - - A host uv can be older than the pinned one; capability is checked - directly instead of trusting the version string. - """ - try: - result = subprocess.run([uv, "venv", "--help"], capture_output=True, - text=True, check=True) - except (OSError, subprocess.SubprocessError): - return False - return "--relocatable" in result.stdout + result.stderr - - -def ensure_uv(progress: Progress | None = None) -> str: - """Return a uv that supports the flags AI setup relies on. - - The launcher's contained bootstrap uv is preferred; a host uv is trusted - only if it actually supports --relocatable. Otherwise uv==0.8.17 is - installed from PyPI (trust PyPI's distribution, not a downloaded shell - script). - """ - contained = data_dir() / "bootstrap/bin/uv" - if contained.is_file() and os.access(contained, os.X_OK): - return str(contained) - found = shutil.which("uv") - if found and _uv_supports_relocatable(found): - return found - root = contained.parent.parent - root.parent.mkdir(parents=True, exist_ok=True) - python = sys.executable or shutil.which("python3") - env = tool_env() - run_process([python, "-m", "venv", str(root)], "Creating uv bootstrap", - progress, env=env) - run_process([str(root / "bin/python"), "-m", "pip", "install", - "--index-url", "https://pypi.org/simple", "uv==0.8.17"], - "Installing uv from PyPI", progress, env=env) - if not (contained.is_file() and os.access(contained, os.X_OK)): - raise RuntimeError("Installing uv from PyPI did not produce a usable " - "uv; check for host pip settings (such as " - "PIP_TARGET) that redirect installations outside " - "the data directory.") - return str(contained) - - -def ensure_ffmpeg() -> str: - """Return the bundled FFmpeg, or a system one where the bundle is - absent, not ffprobe. - - The launcher installs imageio-ffmpeg via the bundled-ffmpeg extra; direct - package installs should declare it as a dependency. This helper never - runs pip in the current interpreter. - """ - try: - import imageio_ffmpeg - return imageio_ffmpeg.get_ffmpeg_exe() - except (ImportError, RuntimeError) as error: - found = shutil.which("ffmpeg") - if found: - return found - raise RuntimeError("FFmpeg is unavailable. Rebuild the contained " - "runtime with './producer.sh --rebuild', or " - "install a system FFmpeg.") from error -- cgit v1.2.3