diff options
| author | historia <historiavg@proton.me> | 2026-09-07 06:47:47 -0400 |
|---|---|---|
| committer | historia <historiavg@proton.me> | 2026-09-07 06:47:47 -0400 |
| commit | 84dd2d068317998f6fb59400c534ef5be6b51b53 (patch) | |
| tree | 025293e9d9229e02960374771ae522d9de2628ce /lib/project | |
| parent | 39b0f2bbed74f6487a41b82501ae3c6799e4b5c4 (diff) | |
| download | producer-main.tar.gz | |
Diffstat (limited to 'lib/project')
| -rw-r--r-- | lib/project/examples/voice.toml | 56 | ||||
| -rw-r--r-- | lib/project/pyproject.toml | 29 | ||||
| -rw-r--r-- | lib/project/src/voiceforge/__init__.py | 3 | ||||
| -rw-r--r-- | lib/project/src/voiceforge/__main__.py | 3 | ||||
| -rw-r--r-- | lib/project/src/voiceforge/ai.py | 157 | ||||
| -rw-r--r-- | lib/project/src/voiceforge/ai_worker.py | 161 | ||||
| -rw-r--r-- | lib/project/src/voiceforge/audio.py | 128 | ||||
| -rw-r--r-- | lib/project/src/voiceforge/cli.py | 193 | ||||
| -rw-r--r-- | lib/project/src/voiceforge/config.py | 118 | ||||
| -rw-r--r-- | lib/project/src/voiceforge/pipeline.py | 199 | ||||
| -rw-r--r-- | lib/project/src/voiceforge/progress.py | 40 | ||||
| -rw-r--r-- | lib/project/src/voiceforge/setup.py | 203 | ||||
| -rw-r--r-- | lib/project/tests/conftest.py | 43 | ||||
| -rw-r--r-- | lib/project/tests/test_ai.py | 366 | ||||
| -rw-r--r-- | lib/project/tests/test_cli.py | 287 | ||||
| -rw-r--r-- | lib/project/tests/test_config.py | 134 | ||||
| -rw-r--r-- | lib/project/tests/test_install.py | 225 | ||||
| -rw-r--r-- | lib/project/tests/test_pipeline.py | 273 |
18 files changed, 2618 insertions, 0 deletions
diff --git a/lib/project/examples/voice.toml b/lib/project/examples/voice.toml new file mode 100644 index 0000000..6a5251a --- /dev/null +++ b/lib/project/examples/voice.toml @@ -0,0 +1,56 @@ +# Complete natural-profile defaults. Load with --config lib/project/examples/voice.toml. +# Flat TOML: no section headers. Named CLI flags override these values; +# --set KEY=VALUE overrides named flags. Use --profile, not --set profile=... +# All explicit values below override preset defaults, even with --profile radio. +# Remove settings you want a different profile to supply. + +profile = "natural" # natural, narrator, radio, cleanup-only +denoiser = "deepfilter" # DeepFilterNet3 only; fft is explicit non-AI fallback; none +device = "auto" # auto, cpu, cuda; explicit cuda fails if unavailable +denoise_strength = 0.85 # 0..1; AI dry/wet blend; zero skips denoising and AI setup +fft_reduction_db = 8.0 # 0.01..30; FFT reduction is scaled by denoise_strength +channel = "auto" # auto accepts mono only; stereo requires left, right or mix + +highpass = true +highpass_hz = 70.0 # 20..300 +lowpass = false +lowpass_hz = 16000.0 # 4000..22000; must be below output Nyquist when enabled +hum_hz = 0 # 0 (off), 50 or 60; notches fundamental and next two harmonics + +# One static gain adjustment for the whole recording, not automatic gain riding. +leveling = true +level_target_db = -23.0 # -40..-12; heuristic speech level in dBFS, not LUFS +max_gain_db = 18.0 # 0..30; caps both gain and attenuation + +expansion = false +expansion_threshold_db = -50.0 # -90..-20 +expansion_ratio = 1.5 # 1..4 +expansion_range_db = 12.0 # 0..40 + +eq = true +warmth_db = 0.5 # -12..12; 140 Hz +mud_db = -1.5 # -12..12; 300 Hz +presence_db = 1.0 # -12..12; 3500 Hz + +compression = true +compressor_threshold_db = -21.0 # -50..-5 +compressor_ratio = 2.0 # 1..10 +compressor_attack_ms = 15.0 # 0.1..200 +compressor_release_ms = 150.0 # 10..2000 +deess = true +deess_intensity = 0.15 # 0..1 +deess_amount = 0.4 # 0..1 + +# Two-pass normalization plus export verification; target misses warn only. +# Turning normalize off leaves limiter on unless you also disable limiter. +# normalize=true requires limiter=true. Previews force both on, even cleanup-only. +normalize = true +target_lufs = -19.0 # -30..-12; mono delivery target, not ACX certification +loudness_range = 9.0 # 1..20 LU +limiter = true +true_peak_db = -1.5 # -9..-0.5 dBTP; MP3 can overshoot, inspect report warnings + +# Final exports are always mono, 24-bit PCM WAV, with an optional MP3 sidecar. +sample_rate = 48000 # 44100 or 48000; intermediate processing is 48 kHz float +mp3 = false +mp3_bitrate = 192 # 128, 160, 192, 224, 256 or 320 kb/s diff --git a/lib/project/pyproject.toml b/lib/project/pyproject.toml new file mode 100644 index 0000000..082b944 --- /dev/null +++ b/lib/project/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = ["hatchling==1.27.0"] +build-backend = "hatchling.build" + +[project] +name = "voiceforge" +version = "0.2.0" +description = "Local, configurable speech cleanup and podcast mastering" +license = "0BSD" +requires-python = ">=3.11" +dependencies = ["rich==14.1.0", "numpy>=1.26.4,<3", "soundfile==0.13.1"] + +[project.optional-dependencies] +test = ["pytest>=8,<10"] +# Preferred by producer.sh; falls back to a system FFmpeg where the wheel +# does not exist (for example musl systems). +bundled-ffmpeg = ["imageio-ffmpeg==0.6.0"] + +[project.scripts] +voiceforge = "voiceforge.cli:main" + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.hatch.build.targets.wheel] +packages = ["src/voiceforge"] + +[tool.hatch.build.targets.sdist] +include = ["/src", "/tests", "/examples", "/pyproject.toml"] diff --git a/lib/project/src/voiceforge/__init__.py b/lib/project/src/voiceforge/__init__.py new file mode 100644 index 0000000..1d8663d --- /dev/null +++ b/lib/project/src/voiceforge/__init__.py @@ -0,0 +1,3 @@ +"""VoiceForge: local speech cleanup and mastering.""" + +__version__ = "0.2.0" diff --git a/lib/project/src/voiceforge/__main__.py b/lib/project/src/voiceforge/__main__.py new file mode 100644 index 0000000..4e28416 --- /dev/null +++ b/lib/project/src/voiceforge/__main__.py @@ -0,0 +1,3 @@ +from .cli import main + +main() diff --git a/lib/project/src/voiceforge/ai.py b/lib/project/src/voiceforge/ai.py new file mode 100644 index 0000000..493be65 --- /dev/null +++ b/lib/project/src/voiceforge/ai.py @@ -0,0 +1,157 @@ +"""Automatic, isolated DeepFilterNet3 inference (Linux, Python 3.11 worker). + +Public progress callback: (stage: str, completed: float | None, total: float | None). +Known totals are audio frames or download bytes (identified by stage); +unknown totals use elapsed seconds as completed. +First use downloads Python, wheels, and upstream model weights into the +VoiceForge data directory (VOICEFORGE_HOME prefix or the XDG data location). +""" +from __future__ import annotations + +import ctypes +import fcntl +import math +import os +from pathlib import Path +import platform +import shutil +import time + +from .setup import Progress, data_dir, ensure_uv, run_process, tool_env + + +def _cuda_available() -> bool: + # Probe the driver without importing Torch or installing CUDA in the host env. + if os.environ.get("CUDA_VISIBLE_DEVICES") in {"", "-1"}: + return False + try: + driver = ctypes.CDLL("libcuda.so.1") + count = ctypes.c_int() + return driver.cuInit(0) == 0 and driver.cuDeviceGetCount(ctypes.byref(count)) == 0 and count.value > 0 + except (OSError, AttributeError): + return False + + +def _flavor(device: str) -> str: + if device not in {"auto", "cpu", "cuda"}: + raise ValueError("device must be 'auto', 'cpu', or 'cuda'") + if device == "cuda" or (device == "auto" and _cuda_available()): + if platform.machine() not in {"x86_64", "AMD64"}: + raise RuntimeError("The pinned CUDA 12.1 backend requires Linux x86_64.") + return "cu121" + return "cpu" + + +def _worker_env() -> dict[str, str]: + # Same isolation as tool_env(); kept as a named seam for worker launches. + return tool_env() + + +def _interpreter_is_local(python: Path, root: Path) -> bool: + """Whether the environment's interpreter belongs to this prefix. + + An installation duplicated by copying (while the original remained) keeps + interpreter links that resolve into the original directory. Such an + environment must be rebuilt, or the copy would silently depend on files + outside this prefix and break when the original is removed. + """ + try: + return python.resolve().is_relative_to(root) + except OSError: + return False + + +def ensure_ai(device: str = "auto", progress: Progress | None = None, + verify: bool = False) -> Path: + """Install/validate the backend and model; return its Python executable. + + CPU and CUDA environments are separate. Explicit CUDA never falls back; + auto uses Torch's availability check in the selected worker. Downloads trust + PyPI, download.pytorch.org, Astral's Python distribution, and upstream DF. + verify forces a real self-test even when the environment is marked ready + ('setup' uses this); ordinary processing trusts the marker so a ready + environment does not pay for a second model load per run. + """ + flavor = _flavor(device) + try: + libc = os.confstr("CS_GNU_LIBC_VERSION") or "" + except (ValueError, OSError): + libc = "" + if platform.system() != "Linux" or not libc.startswith("glibc ") or tuple( + int(part) for part in libc.split()[1].split(".")[:2] + ) < (2, 28): + raise RuntimeError("The pinned DeepFilterNet3 backend requires Linux with glibc >= 2.28 " + "(for example Ubuntu 22.04+ or Debian 12+). " + "Alpine/musl is not supported by its prebuilt wheels.") + root = data_dir() + root.mkdir(parents=True, exist_ok=True) + environment = root / f"ai-df-0.5.6-torch-2.5.1-{flavor}-py311-v1" + python = environment / "bin/python" + marker = environment / ".voiceforge-ready" + started = time.monotonic() + # Serialize bootstrap/model downloads, including calls from separate CLIs. + with (root / "ai.lock").open("a") as lock: + while True: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + if progress: + progress("Waiting for AI setup", time.monotonic() - started, None) + time.sleep(0.25) + ready = (marker.is_file() and python.is_file() + and _interpreter_is_local(python, root)) + if not ready: + # A moved, copied, or half-installed environment cannot be + # repaired in place: interpreter links reference absolute paths. + # Rebuild it; the shared wheel caches make this fast, usually + # offline. + if environment.exists(): + shutil.rmtree(environment) + uv = ensure_uv(progress) + env = _worker_env() + run_process([uv, "venv", "--python", "3.11", "--managed-python", + "--relocatable", "--allow-existing", str(environment)], + "Preparing AI Python 3.11", progress, env=env) + run_process([uv, "pip", "install", "--python", str(python), + "--index-url", f"https://download.pytorch.org/whl/{flavor}", + "torch==2.5.1", "torchaudio==2.5.1"], + f"Installing PyTorch ({flavor})", progress, env=env) + run_process([uv, "pip", "install", "--python", str(python), + "--only-binary", ":all:", + "--index-url", "https://pypi.org/simple", "deepfilternet==0.5.6", + "numpy==1.26.4", "soundfile==0.12.1"], + "Installing DeepFilterNet3", progress, env=env) + # Check a freshly built environment (and any explicit setup) with a + # real model load and inference; this also downloads the weights. + if verify or not ready: + run_process([str(python), "-I", str(Path(__file__).with_name("ai_worker.py")), + "--device", device, "--model-dir", str(root / "models/df3-0.5.6"), + "--check"], "Checking DeepFilterNet3", progress, + env=_worker_env()) + marker.touch() + return python + + +def denoise(source: Path, target: Path, device: str = "auto", strength: float = 1.0, + progress: Progress | None = None) -> None: + """Denoise mono 48 kHz WAV to float WAV, preserving the exact sample count. + + strength is a dry/wet mix in [0, 1]. The worker publishes output atomically; + errors/cancellation leave an existing target untouched. Chunked inference is + bounded-memory, not bit-identical to inference over an entire recording. + """ + if not math.isfinite(strength) or not 0 <= strength <= 1: + raise ValueError("strength must be finite and between 0 and 1") + source, target = Path(source).resolve(), Path(target).resolve() + if source == target: + raise ValueError("source and target must be different files") + if not source.is_file(): + raise FileNotFoundError(source) + if not target.parent.is_dir(): + raise FileNotFoundError(target.parent) + python = ensure_ai(device, progress) + run_process([str(python), "-I", str(Path(__file__).with_name("ai_worker.py")), + "--device", device, "--source", str(source), "--target", str(target), + "--model-dir", str(data_dir() / "models/df3-0.5.6"), + "--strength", str(strength)], "Denoising", progress, env=_worker_env()) diff --git a/lib/project/src/voiceforge/ai_worker.py b/lib/project/src/voiceforge/ai_worker.py new file mode 100644 index 0000000..99ec9f5 --- /dev/null +++ b/lib/project/src/voiceforge/ai_worker.py @@ -0,0 +1,161 @@ +"""Private script executed only by the isolated backend Python.""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import signal +import tempfile +import urllib.request +import zipfile + + +def report(stage: str, completed: float | None = None, total: float | None = None) -> None: + print("VOICEFORGE_PROGRESS " + json.dumps([stage, completed, total]), flush=True) + + +def ensure_model(directory: Path) -> Path: + """Install a release-pinned, hash-checked archive atomically under setup lock.""" + model = directory / "DeepFilterNet3" + if (model / "config.ini").is_file() and (model / "checkpoints/model_120.ckpt.best").is_file(): + return model + if directory.exists(): + raise RuntimeError(f"Incomplete model directory: {directory}. Remove it and retry AI setup.") + directory.parent.mkdir(parents=True, exist_ok=True) + url = "https://raw.githubusercontent.com/Rikorose/DeepFilterNet/v0.5.6/models/DeepFilterNet3.zip" + expected = "49c52edc8947ae1f9bf50d81530beaf3a2c3245aeaf34b6f31ff535cd22284d2" + # Release archive size is fixed alongside its hash, even without Content-Length. + expected_bytes = 7986207 + downloaded = 0 + report("Downloading DeepFilterNet3 model (bytes)", 0, expected_bytes) + with tempfile.TemporaryDirectory(prefix=".df3-", dir=directory.parent) as staging: + archive = Path(staging) / "model.zip" + digest = hashlib.sha256() + with urllib.request.urlopen(url, timeout=60) as response, archive.open("wb") as output: + while chunk := response.read(256 * 1024): + downloaded += len(chunk) + if downloaded > expected_bytes: + raise RuntimeError("DeepFilterNet3 model download exceeds the expected size") + digest.update(chunk) + output.write(chunk) + report("Downloading DeepFilterNet3 model (bytes)", downloaded, expected_bytes) + if downloaded != expected_bytes or digest.hexdigest() != expected: + raise RuntimeError("DeepFilterNet3 model checksum mismatch; refusing to load weights") + report("Verifying and extracting DeepFilterNet3 model") + unpacked = Path(staging) / "unpacked" + with zipfile.ZipFile(archive) as bundle: + for member in bundle.infolist(): + destination = (unpacked / member.filename).resolve() + if not destination.is_relative_to(unpacked.resolve()): + raise RuntimeError("Unsafe path in model archive") + bundle.extractall(unpacked) + os.replace(unpacked, directory) + return model + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--device", choices=("auto", "cpu", "cuda"), required=True) + parser.add_argument("--check", action="store_true") + parser.add_argument("--model-dir", type=Path, required=True) + parser.add_argument("--source", type=Path) + parser.add_argument("--target", type=Path) + parser.add_argument("--strength", type=float, default=1.0) + args = parser.parse_args() + + # SIGTERM from the parent takes the same cleanup path as Ctrl-C. + def terminate(signum, frame): + raise KeyboardInterrupt + + signal.signal(signal.SIGTERM, terminate) + + import numpy as np + import soundfile as sf + import torch + + torch.set_num_threads(min(4, os.cpu_count() or 1)) + available = torch.cuda.is_available() + if args.device == "cuda" and not available: + raise RuntimeError("CUDA was explicitly requested, but PyTorch cannot use it. " + "Check the NVIDIA driver and CUDA_VISIBLE_DEVICES, or select cpu.") + device = "cuda" if args.device != "cpu" and available else "cpu" + # df.utils.get_device() consults DEVICE on every call, including model creation. + os.environ["DEVICE"] = device + from df.enhance import enhance, init_df + from df.model import ModelParams + from libdf import DF + + model_path = ensure_model(args.model_dir) + report(f"Loading DeepFilterNet3 ({device})") + model, state, _ = init_df(str(model_path), log_level="ERROR", log_file=None) + if state.sr() != 48000: + raise RuntimeError("DeepFilterNet3 model must use 48 kHz") + if args.check: + # Exercise both model execution and the compiled libdf/NumPy ABI. + sample = enhance(model, state, torch.zeros(1, 4800), pad=True) + if sample.shape != (1, 4800) or not torch.isfinite(sample).all(): + raise RuntimeError("DeepFilterNet3 self-test failed") + report(f"DeepFilterNet3 ready ({device})", 1, 1) + return + if args.source is None or args.target is None: + parser.error("--source and --target are required unless --check is used") + if not np.isfinite(args.strength) or not 0 <= args.strength <= 1: + parser.error("--strength must be in [0, 1]") + + temporary = None + try: + with sf.SoundFile(args.source) as source: + if source.samplerate != 48000 or source.channels != 1 or source.format not in {"WAV", "WAVEX", "RF64"}: + raise ValueError("AI input must be a mono 48 kHz WAV; convert it before denoising") + frames = len(source) + fd, name = tempfile.mkstemp(prefix=".voiceforge-ai-", suffix=".wav", dir=args.target.parent) + os.close(fd) + temporary = Path(name) + # RF64 avoids RIFF's 4 GiB limit on very long recordings. + output_format = "RF64" if frames * 4 > 0xFFFFFFFF - 4096 else "WAV" + with sf.SoundFile(temporary, "w", samplerate=48000, channels=1, + format=output_format, subtype="FLOAT") as target: + block, context, overlap = 480000, 96000, 2400 + previous = None + params = ModelParams() + report(f"Denoising ({device})", 0, frames) + for start in range(0, frames, block): + end = min(start + block, frames) + left, right = max(0, start - context), min(frames, end + overlap + context) + source.seek(left) + audio = source.read(right - left, dtype="float32") + if not np.isfinite(audio).all(): + raise ValueError("Input contains non-finite audio samples") + if args.strength: + # Fresh STFT state per window; context warms normalization and + # recurrent layers. Pad to a whole hop before delay compensation. + state = DF(sr=params.sr, fft_size=params.fft_size, + hop_size=params.hop_size, nb_bands=params.nb_erb, + min_nb_erb_freqs=params.min_nb_freqs) + padded = np.pad(audio, (0, (-len(audio)) % params.hop_size)) + wet = enhance(model, state, torch.from_numpy(padded).unsqueeze(0), + pad=True).squeeze(0).numpy()[:len(audio)] + if len(wet) != len(audio) or not np.isfinite(wet).all(): + raise RuntimeError("DeepFilterNet3 returned invalid audio") + audio = audio * (1 - args.strength) + wet * args.strength + kept = audio[start - left:min(frames, end + overlap) - left].copy() + if previous is not None and args.strength: + n = len(previous) + fade = np.linspace(0, 1, n, dtype=np.float32) + kept[:n] = previous * (1 - fade) + kept[:n] * fade + target.write(kept[:end - start]) + previous = kept[end - start:].copy() + report(f"Denoising ({device})", end, frames) + if target.tell() != frames: + raise RuntimeError("Output frame count does not match input") + os.replace(temporary, args.target) + temporary = None + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + + +if __name__ == "__main__": + main() diff --git a/lib/project/src/voiceforge/audio.py b/lib/project/src/voiceforge/audio.py new file mode 100644 index 0000000..7fee0c2 --- /dev/null +++ b/lib/project/src/voiceforge/audio.py @@ -0,0 +1,128 @@ +"""Streaming measurement and FFmpeg execution. No shell interpolation.""" +from collections import deque +import json +import math +import os +from pathlib import Path +import selectors +import signal +import subprocess +import time + +import numpy as np +import soundfile as sf + +from .setup import terminate_group + + +def ffmpeg_run(ffmpeg: str, args: list[str], stage: str, duration: float, + progress) -> str: + command = [ffmpeg, "-hide_banner", "-nostdin", "-y", "-nostats", + "-progress", "pipe:1"] + args + progress(stage, 0, duration or None) + diagnostics = deque(maxlen=128) + pending = b"" + completed = 0.0 + last_advance = time.monotonic() + with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + start_new_session=True) as process: + try: + with selectors.DefaultSelector() as selector: + for stream in (process.stdout, process.stderr): + os.set_blocking(stream.fileno(), False) + selector.register(stream, selectors.EVENT_READ) + while selector.get_map() or process.poll() is None: + for key, _ in selector.select(timeout=0.2): + data = os.read(key.fd, 8192) + if not data: + selector.unregister(key.fileobj) + continue + if key.fileobj is process.stderr: + diagnostics.append(data) + else: + pending += data + while b"\n" in pending: + line, pending = pending.split(b"\n", 1) + if line.startswith(b"out_time_us="): + try: + value = max(0.0, int(line.split(b"=", 1)[1]) / 1e6) + except ValueError: + continue + if value > completed: + last_advance = time.monotonic() + completed = value + label = stage + if time.monotonic() - last_advance > 60: + label += " (no new audio progress for 60s; still waiting)" + progress(label, min(completed, duration), duration or None) + code = process.wait() + output = b"".join(diagnostics).decode(errors="replace") + if code: + raise RuntimeError(f"{stage} failed (exit {code}):\n{output[-8000:]}") + progress(stage, duration, duration or None) + return output + except BaseException: + terminate_group(process) + raise + + +def measure(path: Path, progress, stage="Analyzing audio") -> dict: + """100ms RMS histogram is bounded in memory, including multi-hour input.""" + with sf.SoundFile(path) as source: + if source.channels not in (1, 2): + raise ValueError("Only mono or stereo voice recordings are supported") + if not len(source): + raise ValueError("The recording is empty") + peaks = np.zeros(source.channels) + sums = np.zeros(source.channels) + squares = np.zeros(source.channels) + clipped = 0 + histogram = np.zeros(121, dtype=np.int64) + frames = 0 + progress(stage, 0, len(source)) + for audio in source.blocks(blocksize=max(1, source.samplerate // 10), + dtype="float64", always_2d=True): + if not np.isfinite(audio).all(): + raise ValueError("Recording contains non-finite samples") + peaks = np.maximum(peaks, np.max(np.abs(audio), axis=0)) + sums += audio.sum(axis=0) + squares += (audio * audio).sum(axis=0) + clipped += int(np.count_nonzero(np.abs(audio) >= 0.9999)) + rms = float(np.sqrt(np.mean(audio * audio))) + db = 20 * math.log10(max(rms, 1e-6)) + histogram[int(np.clip(round(db) + 120, 0, 120))] += 1 + frames += len(audio) + progress(stage, frames, len(source)) + cumulative = histogram.cumsum() + def percentile(p): + return int(np.searchsorted(cumulative, max(1, math.ceil(cumulative[-1] * p)))) - 120 + quiet, speech = percentile(0.1), percentile(0.8) + if speech - quiet < 15: + # Pause-dominated recordings: the 80th percentile of all blocks + # sits in room tone, so estimate the level from the loud tail + # instead of mistaking sparse active speech for silence. + speech = max(speech, percentile(0.98)) + noise_available = speech - quiet >= 15 and quiet > -110 + return {"frames": frames, "duration_seconds": frames / source.samplerate, + "sample_rate": source.samplerate, "channels": source.channels, + "subtype": source.subtype, + "sample_peak_dbfs": float(20 * np.log10(max(float(peaks.max()), 1e-12))), + "channel_rms_dbfs": (20 * np.log10(np.maximum(np.sqrt(squares / frames), 1e-12))).tolist(), + "dc_offset": (sums / frames).tolist(), "near_full_scale_samples": clipped, + "speech_level_estimate_dbfs": speech, + "quiet_blocks_dbfs": quiet, + "noise_floor_estimate_dbfs": quiet if noise_available else None, + "noise_floor_confidence": "low (quiet-block heuristic, not speech recognition)" if noise_available else "insufficient room tone"} + + +def loudness(ffmpeg, source, settings, duration, progress, stage="Measuring loudness"): + output = ffmpeg_run(ffmpeg, ["-i", str(source), "-map", "0:a:0", "-af", + f"loudnorm=I={settings.target_lufs}:TP={settings.true_peak_db}:LRA={settings.loudness_range}:print_format=json", + "-f", "null", "-"], stage, duration, progress) + start, end = output.rfind("{"), output.rfind("}") + if start < 0 or end < start: + raise RuntimeError("FFmpeg did not return loudness measurements") + raw = json.loads(output[start:end + 1]) + return {key: (float(value) if math.isfinite(float(value)) else None) + for key, value in raw.items() if key != "normalization_type"} | { + "normalization_type": raw.get("normalization_type")} diff --git a/lib/project/src/voiceforge/cli.py b/lib/project/src/voiceforge/cli.py new file mode 100644 index 0000000..52cd1bd --- /dev/null +++ b/lib/project/src/voiceforge/cli.py @@ -0,0 +1,193 @@ +"""Command-line entry point; configuration errors fail before model downloads.""" +import argparse +from dataclasses import asdict, fields +import json +import math +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import tomllib + +from rich.console import Console + +from . import __version__ +from .audio import ffmpeg_run, loudness, measure +from .config import PROFILES, Settings, resolve, toml +from .pipeline import check_target, process, targets_for +from .progress import Display +from .setup import data_dir, ensure_ffmpeg + + +def parser(): + app = argparse.ArgumentParser(prog="producer", description="Local speech cleanup and podcast mastering. Originals are never overwritten.") + app.add_argument("--version", action="version", version=__version__) + sub = app.add_subparsers(dest="command", required=True) + for name in ("process", "preview", "config", "analyze"): + cmd = sub.add_parser(name) + cmd.add_argument("--config", type=Path, help="TOML configuration file") + cmd.add_argument("--profile", choices=PROFILES, default=None) + cmd.add_argument("--set", action="append", default=[], metavar="KEY=VALUE", help="Override any setting using TOML value syntax") + for field in fields(Settings): + if field.name == "profile": + continue + default = getattr(Settings(), field.name) + option = "--" + field.name.replace("_", "-") + if isinstance(default, bool): + cmd.add_argument(option, action=argparse.BooleanOptionalAction, default=None) + else: + cmd.add_argument(option, type=type(default), default=None) + cmd.add_argument("--no-denoise", dest="denoiser", action="store_const", const="none") + if name == "config": + cmd.add_argument("--json", action="store_true") + else: + cmd.add_argument("inputs", type=Path, nargs="+" if name == "process" else 1) + if name in ("process", "preview"): + cmd.add_argument("--output-dir", type=Path, default=None, + help="Directory for generated outputs (default: beside each input)") + cmd.add_argument("--overwrite", action="store_true", help="Replace generated outputs, never input files") + if name == "process": + cmd.add_argument("-o", "--output", type=Path, help="Explicit WAV destination (one input only)") + if name == "preview": + cmd.add_argument("--start", type=float, default=0) + cmd.add_argument("--duration", type=float, default=30) + cmd.add_argument("--profiles", nargs="+", choices=PROFILES, default=["natural", "narrator", "radio"]) + sub.add_parser("doctor", help="Check tools and driver without downloading AI") + setup = sub.add_parser("setup", help="Download and self-test the isolated AI backend") + setup.add_argument("--device", choices=("auto", "cpu", "cuda"), default="auto") + return app + + +def settings_for(args, profile=None): + overrides = {field.name: getattr(args, field.name) for field in fields(Settings) + if field.name != "profile" and getattr(args, field.name, None) is not None} + for item in args.set: + key, sep, value = item.partition("=") + if not sep: + raise ValueError("--set expects KEY=VALUE") + # Strings can be supplied without TOML quotes for CLI convenience. + try: + parsed = tomllib.loads("value = " + value)["value"] + except tomllib.TOMLDecodeError: + parsed = value + overrides[key.strip()] = parsed + return resolve(args.config, profile or args.profile, overrides) + + +def require_input(source: Path) -> None: + if not source.is_file() or source.suffix.lower() != ".wav": + raise ValueError(f"Input must be an existing WAV file: {source}") + + +def main(): + app = parser() + args = app.parse_args() + console = Console(stderr=True) + try: + if args.command == "doctor": + from .ai import _cuda_available + ffmpeg = ensure_ffmpeg() + version = subprocess.run([ffmpeg, "-version"], capture_output=True, text=True, check=True).stdout.splitlines()[0] + console.print(version, markup=False) + console.print(f"FFmpeg: {ffmpeg}\nData: {data_dir()}\nCUDA driver available: {_cuda_available()}\nAI is installed and self-tested by './producer.sh setup'.", markup=False) + return + if args.command == "setup": + from .ai import ensure_ai + with Display() as display: + ensure_ffmpeg() + python = ensure_ai(args.device, display.update, verify=True) + console.print(f"AI ready: {python}", markup=False) + return + settings = settings_for(args) + if args.command == "config": + print(json.dumps(asdict(settings), indent=2) if args.json else toml(settings), end="\n" if args.json else "") + return + with Display() as display: + if args.command == "analyze": + stats = measure(args.inputs[0], display.update) + stats["loudness"] = loudness(ensure_ffmpeg(), args.inputs[0], settings, stats["duration_seconds"], display.update) + print(json.dumps(stats, indent=2, allow_nan=False)) + return + if args.command == "process": + if args.output and len(args.inputs) != 1: + raise ValueError("--output requires exactly one input") + if args.output and args.output_dir: + raise ValueError("--output and --output-dir cannot be combined") + # Without an explicit destination, outputs are written beside + # each input; --output-dir keeps every result in one directory. + outputs = [args.output or (args.output_dir or p.parent) / f"{p.stem}.{settings.profile}.wav" + for p in args.inputs] + # Preflight the whole plan so a predictable conflict cannot + # fail a batch after earlier inputs have already been published. + sources = {p.resolve() for p in args.inputs} + claimed: set[Path] = set() + for source, output in zip(args.inputs, outputs): + require_input(source) + targets = targets_for(output.absolute(), settings) + resolved = {target.resolve() for target in targets} + if len(resolved) < len(targets) or resolved & claimed: + raise ValueError("Inputs have duplicate output names; process them separately with --output") + claimed |= resolved + if resolved & sources or any( + target.exists() and os.path.samefile(source, target) for target in targets): + raise ValueError("An output would overwrite an input recording") + for target in targets: + check_target(source, target, args.overwrite) + for source, output in zip(args.inputs, outputs): + report = process(source, output, settings, display.update, args.overwrite) + console.print(f"Written: {output} | {report['master_loudness']['input_i']} LUFS | {report['master_loudness']['input_tp']} dBTP", markup=False) + for warning in report["warnings"]: + console.print(f"Warning: {warning}", markup=False) + return + if not math.isfinite(args.start) or args.start < 0 or not 0 < args.duration <= 300: + raise ValueError("Preview start must be >= 0 and duration must be in (0, 300] seconds") + if len(set(args.profiles)) != len(args.profiles): + raise ValueError("Preview profiles must be unique") + if any(item.partition("=")[0].strip() == "profile" for item in args.set): + raise ValueError("Preview profiles come from --profiles; remove the profile override from --set") + source = args.inputs[0] + require_input(source) + original = source.resolve() + output_dir = args.output_dir or source.parent + # Preflight every preview destination before extracting the excerpt. + claimed: set[Path] = set() + for profile in args.profiles: + output = output_dir / f"{source.stem}.{profile}.preview.wav" + if output.resolve() == original: + raise ValueError("Preview output would overwrite the original") + targets = [output, output.with_suffix(".report.json")] + resolved = {target.resolve() for target in targets} + if len(resolved) < len(targets) or resolved & claimed: + raise ValueError("Preview outputs have duplicate names") + claimed |= resolved + for target in targets: + check_target(original, target, args.overwrite) + output_dir.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=".preview-", dir=output_dir) as tmp: + excerpt = Path(tmp) / "excerpt.wav" + ffmpeg_run(ensure_ffmpeg(), ["-ss", str(args.start), "-i", str(original), "-t", str(args.duration), + "-map", "0:a:0", "-c:a", "pcm_f32le", str(excerpt)], "Extracting preview", args.duration, display.update) + for profile in args.profiles: + configured = settings_for(args, profile) + # Equal loudness prevents a louder preset winning an unfair A/B. + configured.normalize = True + configured.limiter = True + output = output_dir / f"{source.stem}.{profile}.preview.wav" + report = process(excerpt, output, configured, display.update, args.overwrite, + provenance={"original": str(original), "preview_start_seconds": args.start, + "requested_duration_seconds": args.duration, + "normalization_forced": True}) + console.print(f"Preview: {output} | {report['master_loudness']['input_i']} LUFS | {report['master_loudness']['input_tp']} dBTP", markup=False) + for warning in report["warnings"]: + console.print(f"Warning: {warning}", markup=False) + except KeyboardInterrupt: + console.print("Cancelled; temporary files removed. Completed outputs may remain.") + raise SystemExit(130) + except (ValueError, OSError, RuntimeError, subprocess.SubprocessError) as error: + console.print(f"Error: {error}", markup=False) + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/lib/project/src/voiceforge/config.py b/lib/project/src/voiceforge/config.py new file mode 100644 index 0000000..9cbd59d --- /dev/null +++ b/lib/project/src/voiceforge/config.py @@ -0,0 +1,118 @@ +"""Flat, strict TOML configuration; CLI overrides are applied last.""" +from dataclasses import asdict, dataclass, fields +import math +from pathlib import Path +import tomllib + + +@dataclass +class Settings: + profile: str = "natural" + denoiser: str = "deepfilter" + device: str = "auto" + denoise_strength: float = 0.85 + fft_reduction_db: float = 8.0 + channel: str = "auto" + highpass: bool = True + highpass_hz: float = 70.0 + lowpass: bool = False + lowpass_hz: float = 16000.0 + hum_hz: int = 0 + leveling: bool = True + level_target_db: float = -23.0 + max_gain_db: float = 18.0 + expansion: bool = False + expansion_threshold_db: float = -50.0 + expansion_ratio: float = 1.5 + expansion_range_db: float = 12.0 + eq: bool = True + warmth_db: float = 0.5 + mud_db: float = -1.5 + presence_db: float = 1.0 + compression: bool = True + compressor_threshold_db: float = -21.0 + compressor_ratio: float = 2.0 + compressor_attack_ms: float = 15.0 + compressor_release_ms: float = 150.0 + deess: bool = True + deess_intensity: float = 0.15 + deess_amount: float = 0.4 + normalize: bool = True + target_lufs: float = -19.0 + loudness_range: float = 9.0 + limiter: bool = True + true_peak_db: float = -1.5 + sample_rate: int = 48000 + mp3: bool = False + mp3_bitrate: int = 192 + + +PROFILES = { + "natural": {}, + "narrator": {"compressor_ratio": 1.6, "compressor_attack_ms": 25.0, + "presence_db": 0.5, "warmth_db": 1.0, "loudness_range": 11.0}, + "radio": {"compressor_ratio": 3.5, "compressor_threshold_db": -24.0, + "compressor_attack_ms": 8.0, "compressor_release_ms": 100.0, + "warmth_db": 2.0, "mud_db": -2.0, "presence_db": 2.0, + "deess_intensity": 0.25, "loudness_range": 6.0}, + "cleanup-only": {"leveling": False, "eq": False, "compression": False, + "deess": False, "normalize": False, "limiter": False}, +} + +RANGES = { + "denoise_strength": (0, 1), "fft_reduction_db": (0.01, 30), + "highpass_hz": (20, 300), "lowpass_hz": (4000, 22000), + "level_target_db": (-40, -12), "max_gain_db": (0, 30), + "expansion_threshold_db": (-90, -20), "expansion_ratio": (1, 4), + "expansion_range_db": (0, 40), "warmth_db": (-12, 12), + "mud_db": (-12, 12), "presence_db": (-12, 12), + "compressor_threshold_db": (-50, -5), "compressor_ratio": (1, 10), + "compressor_attack_ms": (0.1, 200), "compressor_release_ms": (10, 2000), + "deess_intensity": (0, 1), "deess_amount": (0, 1), + "target_lufs": (-30, -12), "loudness_range": (1, 20), + "true_peak_db": (-9, -0.5), +} + + +def resolve(path: Path | None = None, profile: str | None = None, + overrides: dict | None = None) -> Settings: + values = tomllib.loads(path.read_text()) if path else {} + chosen = (overrides or {}).get("profile", profile or values.get("profile", "natural")) + if not isinstance(chosen, str) or chosen not in PROFILES: + raise ValueError(f"Unknown profile {chosen!r}; choose {', '.join(PROFILES)}") + merged = asdict(Settings()) | PROFILES[chosen] | values | {"profile": chosen} + merged.update(overrides or {}) + defaults = asdict(Settings()) + unknown = merged.keys() - defaults.keys() + if unknown: + raise ValueError(f"Unknown setting(s): {', '.join(sorted(unknown))}") + for key, value in merged.items(): + expected = type(defaults[key]) + if expected is float: + if type(value) not in (int, float) or not math.isfinite(value): + raise ValueError(f"{key} must be a finite number") + elif type(value) is not expected: + raise ValueError(f"{key} must be {expected.__name__}") + if key in RANGES and not RANGES[key][0] <= value <= RANGES[key][1]: + raise ValueError(f"{key} must be between {RANGES[key][0]} and {RANGES[key][1]}") + for key, options in { + "denoiser": ("deepfilter", "fft", "none"), "device": ("auto", "cpu", "cuda"), + "channel": ("auto", "left", "right", "mix"), "hum_hz": (0, 50, 60), + "sample_rate": (44100, 48000), "mp3_bitrate": (128, 160, 192, 224, 256, 320), + }.items(): + if merged[key] not in options: + raise ValueError(f"{key} must be one of {options}") + if merged["lowpass"] and merged["lowpass_hz"] >= merged["sample_rate"] / 2: + raise ValueError("lowpass_hz must be below the output Nyquist frequency") + if merged["normalize"] and not merged["limiter"]: + raise ValueError("Normalization includes true-peak limiting; use --no-normalize to bypass it") + return Settings(**merged) + + +def toml(settings: Settings) -> str: + lines = ["# VoiceForge configuration. CLI flags override these values."] + for field in fields(settings): + value = getattr(settings, field.name) + text = str(value).lower() if isinstance(value, bool) else repr(value) + lines.append(f"{field.name} = {text}") + return "\n".join(lines) + "\n" diff --git a/lib/project/src/voiceforge/pipeline.py b/lib/project/src/voiceforge/pipeline.py new file mode 100644 index 0000000..a143dcf --- /dev/null +++ b/lib/project/src/voiceforge/pipeline.py @@ -0,0 +1,199 @@ +"""Offline mono voice mastering with float intermediates and atomic outputs.""" +from dataclasses import asdict +import fcntl +import json +import math +import os +from pathlib import Path +import tempfile + +from . import __version__ +from .audio import ffmpeg_run, loudness, measure +from .config import Settings +from .setup import ensure_ffmpeg + + +def filters(settings: Settings) -> list[str]: + s = settings + result = [] + if s.expansion: + result.append(f"agate=threshold={10 ** (s.expansion_threshold_db / 20)}:ratio={s.expansion_ratio}:range={10 ** (-s.expansion_range_db / 20)}:attack=10:release=250:detection=rms") + if s.eq: + result += [f"equalizer=f=140:t=q:w=0.7:g={s.warmth_db}", + f"equalizer=f=300:t=q:w=0.8:g={s.mud_db}", + f"equalizer=f=3500:t=q:w=0.7:g={s.presence_db}"] + if s.compression: + result.append(f"acompressor=threshold={10 ** (s.compressor_threshold_db / 20)}:ratio={s.compressor_ratio}:attack={s.compressor_attack_ms}:release={s.compressor_release_ms}:knee=4:makeup=1:detection=rms") + if s.deess: + result.append(f"deesser=i={s.deess_intensity}:m={s.deess_amount}:f=0.5") + if s.lowpass: + result.append(f"lowpass=f={s.lowpass_hz}:p=2") + return result or ["anull"] + + +def targets_for(destination: Path, settings: Settings) -> list[Path]: + """Every path a successful run publishes: master WAV, report, optional MP3.""" + targets = [destination, destination.with_suffix(".report.json")] + if settings.mp3: + targets.append(destination.with_suffix(".mp3")) + return targets + + +def check_target(source: Path, target: Path, overwrite: bool) -> None: + """Preflight one published path; the original recording is never touched.""" + if target.resolve() == source or (target.exists() and os.path.samefile(source, target)): + raise ValueError("Refusing to overwrite the original recording") + # lexists also rejects dangling symlinks, which publication would only hit late. + if os.path.lexists(target) and not overwrite: + raise FileExistsError(f"Output exists: {target}; use --overwrite to replace outputs") + if target.exists() and not target.is_file(): + raise ValueError(f"Output is not a regular file: {target}") + + +def publish(source: Path, target: Path, overwrite: bool): + if overwrite: + os.replace(source, target) + else: + # Link is atomic and fails if another process created the destination. + os.link(source, target) + source.unlink() + + +def process(source: Path, destination: Path, settings: Settings, progress, + overwrite=False, provenance: dict | None = None) -> dict: + source, destination = source.resolve(), destination.absolute() + if not source.is_file() or source.suffix.lower() != ".wav": + raise ValueError(f"Input must be an existing WAV file: {source}") + if destination.suffix.lower() != ".wav": + raise ValueError("Master output must have a .wav extension") + report_path = destination.with_suffix(".report.json") + for target in targets_for(destination, settings): + check_target(source, target, overwrite) + ffmpeg = ensure_ffmpeg() + original = measure(source, progress, "Analyzing original") + if original["sample_peak_dbfs"] < -100: + raise ValueError("Recording is silent or too quiet to master safely") + channel = settings.channel + if original["channels"] == 2 and channel == "auto": + raise ValueError("Stereo input: select --channel left, right, or mix explicitly. Auto downmix could cancel your voice.") + if original["channels"] == 1 and channel == "right": + raise ValueError("Cannot select the right channel of a mono recording") + duration = original["duration_seconds"] + warnings = [] + if original["near_full_scale_samples"]: + warnings.append("Input has near-full-scale samples: inspect for clipping; lost peaks cannot be reliably restored.") + if max(abs(x) for x in original["dc_offset"]) > 0.01: + warnings.append("Significant input DC offset detected; enable highpass filtering to remove it.") + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=".voiceforge-", dir=destination.parent) as temporary: + work = Path(temporary) + prepared, cleaned, shaped = [work / name for name in ("prepared.wav", "cleaned.wav", "shaped.wav")] + input_filters = [] + if original["channels"] == 2: + input_filters.append({"left": "pan=mono|c0=c0", "right": "pan=mono|c0=c1", + "mix": "pan=mono|c0=0.5*c0+0.5*c1"}[channel]) + if settings.highpass: + input_filters.append(f"highpass=f={settings.highpass_hz}:p=2") + if settings.hum_hz: + input_filters += [f"equalizer=f={settings.hum_hz * n}:t=q:w=25:g=-18" for n in (1, 2, 3)] + ffmpeg_run(ffmpeg, ["-i", str(source), "-map", "0:a:0", "-af", + ",".join(input_filters or ["anull"]), "-ar", "48000", "-ac", "1", + "-c:a", "pcm_f32le", "-rf64", "auto", str(prepared)], "Preparing mono 48 kHz audio", duration, progress) + if settings.denoiser == "deepfilter" and settings.denoise_strength > 0: + from .ai import denoise + denoise(prepared, cleaned, settings.device, settings.denoise_strength, progress) + elif settings.denoiser == "fft" and settings.denoise_strength > 0: + ffmpeg_run(ffmpeg, ["-i", str(prepared), "-af", + f"afftdn=nr={max(0.01, settings.fft_reduction_db * settings.denoise_strength)}:tn=1", + "-c:a", "pcm_f32le", "-rf64", "auto", str(cleaned)], "FFT noise reduction", duration, progress) + else: + cleaned = prepared + clean_stats = measure(cleaned, progress, "Measuring cleaned voice") + if clean_stats["sample_peak_dbfs"] < -100: + raise ValueError("The selected audio is silent or too quiet to master safely; " + "check --channel, the mix, and cleanup settings") + gain = 0.0 + if settings.leveling: + speech_db = clean_stats["speech_level_estimate_dbfs"] + if speech_db <= -110: + warnings.append("Speech level estimate is unreliable; leveling skipped. " + "Inspect levels or rely on normalization.") + else: + desired = settings.level_target_db - speech_db + gain = max(-settings.max_gain_db, min(settings.max_gain_db, desired)) + if abs(desired) > settings.max_gain_db: + warnings.append("Input level correction reached its configured gain limit.") + chain = ([f"volume={gain}dB"] if gain else []) + filters(settings) + ffmpeg_run(ffmpeg, ["-i", str(cleaned), "-af", ",".join(chain), + "-c:a", "pcm_f32le", "-rf64", "auto", str(shaped)], "EQ, dynamics and de-essing", duration, progress) + before = loudness(ffmpeg, shaped, settings, duration, progress, "Loudness analysis (pass 1)") + if settings.normalize and any(before.get(k) is None for k in ("input_i", "input_tp", "input_lra", "input_thresh", "target_offset")): + raise ValueError("Cannot normalize: insufficient measurable audio after processing") + final_filters = [] + if settings.normalize: + final_filters.append(f"loudnorm=I={settings.target_lufs}:TP={settings.true_peak_db}:LRA={settings.loudness_range}:" + f"measured_I={before['input_i']}:measured_TP={before['input_tp']}:measured_LRA={before['input_lra']}:" + f"measured_thresh={before['input_thresh']}:offset={before['target_offset']}:linear=true:print_format=json") + elif settings.limiter: + final_filters += ["aresample=192000", f"alimiter=limit={10 ** (settings.true_peak_db / 20)}:level=false:latency=true"] + final_filters.append(f"aresample={settings.sample_rate}:output_sample_bits=24:dither_method=triangular") + master = work / "master.wav" + render_log = ffmpeg_run(ffmpeg, ["-i", str(shaped), "-af", ",".join(final_filters), + "-ar", str(settings.sample_rate), "-c:a", "pcm_s24le", "-rf64", "auto", str(master)], + "Mastering (pass 2)", duration, progress) + # Bounded disk use: drop consumed intermediates instead of waiting for + # the temporary directory teardown at the end of the run. + shaped.unlink(missing_ok=True) + for intermediate in {prepared, cleaned}: + intermediate.unlink(missing_ok=True) + normalization_mode = "disabled" + if settings.normalize: + start, end = render_log.rfind("{"), render_log.rfind("}") + if start < 0 or end < start: + raise RuntimeError("FFmpeg did not report the applied normalization mode") + normalization_mode = json.loads(render_log[start:end + 1])["normalization_type"] + final = loudness(ffmpeg, master, settings, duration, progress, "Verifying exported master") + final_stats = measure(master, progress, "Checking exported samples") + if abs(final_stats["duration_seconds"] - duration) > 0.02: + raise RuntimeError("Output duration differs from input by more than 20 ms") + if (final_stats["channels"] != 1 or final_stats["sample_rate"] != settings.sample_rate + or final_stats["subtype"] != "PCM_24"): + raise RuntimeError("Exported master is not the expected mono PCM_24 WAV") + if settings.normalize and (final["input_i"] is None or abs(final["input_i"] - settings.target_lufs) > 0.5): + warnings.append("Master misses the loudness target by more than 0.5 LU; inspect the report before publishing.") + if settings.limiter and final["input_tp"] is not None and final["input_tp"] > settings.true_peak_db + 0.1: + warnings.append("Master exceeds the true-peak target by more than 0.1 dB; inspect before publishing.") + if final_stats["near_full_scale_samples"]: + warnings.append("Export has near-full-scale samples. Enable limiting or lower gains.") + mp3_stats = None + if settings.mp3: + mp3 = work / "delivery.mp3" + ffmpeg_run(ffmpeg, ["-i", str(master), "-c:a", "libmp3lame", "-b:a", + f"{settings.mp3_bitrate}k", str(mp3)], "Encoding MP3", duration, progress) + mp3_stats = loudness(ffmpeg, mp3, settings, duration, progress, "Verifying decoded MP3") + if mp3_stats["input_tp"] is not None and mp3_stats["input_tp"] > settings.true_peak_db + 0.1: + warnings.append("MP3 encoding increased true peak beyond the configured ceiling; use more headroom.") + report = {"voiceforge_version": __version__, "source": str(source), + "output": str(destination), "settings": asdict(settings), "input": original, + "cleaned": clean_stats, "level_correction_db": gain, + "pre_master_loudness": before, "master_loudness": final, + "master": final_stats, "mp3_loudness": mp3_stats, + "normalization_mode": normalization_mode, + "warnings": warnings} + if provenance: + report["provenance"] = provenance + staged_report = work / "report.json" + staged_report.write_text(json.dumps(report, indent=2, allow_nan=False) + "\n") + # Publish only after all processing and verification succeeded. The + # directory lock serializes publication so concurrent jobs targeting + # the same output directory cannot interleave WAV/report/MP3 sets. + publication_fd = os.open(destination.parent, os.O_RDONLY) + try: + fcntl.flock(publication_fd, fcntl.LOCK_EX) + if settings.mp3: + publish(mp3, destination.with_suffix(".mp3"), overwrite) + publish(staged_report, report_path, overwrite) + publish(master, destination, overwrite) + finally: + os.close(publication_fd) + return report diff --git a/lib/project/src/voiceforge/progress.py b/lib/project/src/voiceforge/progress.py new file mode 100644 index 0000000..a8c5476 --- /dev/null +++ b/lib/project/src/voiceforge/progress.py @@ -0,0 +1,40 @@ +"""A refresh thread keeps elapsed time moving even during native processing.""" +import time + +from rich.console import Console +from rich.progress import (BarColumn, Progress, SpinnerColumn, TaskProgressColumn, + TextColumn, TimeElapsedColumn, TimeRemainingColumn) + + +class Display: + def __init__(self): + self.console = Console(stderr=True) + self.progress = Progress(SpinnerColumn(), TextColumn("{task.description}"), + BarColumn(), TaskProgressColumn(), TimeElapsedColumn(), + TimeRemainingColumn(), console=self.console, + refresh_per_second=5, disable=not self.console.is_terminal) + self.task = None + self.stage = "" + self.started = time.monotonic() + self.last_print = 0.0 + + def __enter__(self): + self.progress.start() + return self + + def __exit__(self, *args): + self.progress.stop() + + def update(self, stage, completed=None, total=None): + changed = stage != self.stage + if changed: + if self.task is not None: + self.progress.remove_task(self.task) + self.task = self.progress.add_task(stage, total=total) + self.stage = stage + self.progress.update(self.task, completed=completed if total else 0, total=total) + now = time.monotonic() + if not self.console.is_terminal and (changed or now - self.last_print >= 5): + fraction = f" {min(100, 100 * (completed or 0) / total):.0f}%" if total else "" + self.console.print(f"[{now - self.started:7.1f}s] {stage}{fraction}", markup=False) + self.last_print = now diff --git a/lib/project/src/voiceforge/setup.py b/lib/project/src/voiceforge/setup.py new file mode 100644 index 0000000..9afeb44 --- /dev/null +++ b/lib/project/src/voiceforge/setup.py @@ -0,0 +1,203 @@ +"""User-local tools. No sudo and no changes to the application's interpreter.""" +from __future__ import annotations + +from collections import deque +from collections.abc import Callable +import json +import os +from pathlib import Path +import selectors +import shutil +import signal +import subprocess +import sys +import time + +Progress = Callable[[str, float | None, float | None], None] + + +def data_dir() -> Path: + """The VoiceForge data/prefix directory. + + VOICEFORGE_HOME wins (the single-directory launcher exports it), then + XDG_DATA_HOME/voiceforge, then ~/.local/share/voiceforge. A relative + VOICEFORGE_HOME resolves against the current directory. + """ + home = os.environ.get("VOICEFORGE_HOME") + if home: + return Path(home).absolute() + root = Path(os.environ.get("XDG_DATA_HOME", str(Path.home() / ".local/share"))) + if not root.is_absolute(): + root = Path.home() / ".local/share" + return root / "voiceforge" + + +def tool_env() -> dict[str, str]: + """Sanitized environment for bootstrap, setup, and worker subprocesses. + + Host Python settings are removed so a virtualenv or sitecustomize on the + machine cannot leak into managed environments. Installation-target, + index, and constraint overrides are dropped and package-manager + configuration files disabled, so packages can only come from the + explicitly passed trusted indexes and land inside the data directory; + every cache (pip, uv, managed Python, XDG, temporary files) stays there + too. + """ + env = os.environ.copy() + for name in ("PYTHONPATH", "PYTHONHOME", "VIRTUAL_ENV", + "PIP_TARGET", "PIP_PREFIX", "PIP_USER", "PIP_FIND_LINKS", + "PIP_INDEX_URL", "PIP_EXTRA_INDEX_URL", "PIP_CONSTRAINT", + "UV_INDEX_URL", "UV_DEFAULT_INDEX", "UV_EXTRA_INDEX_URL", + "UV_INDEX", "UV_FIND_LINKS", "UV_CONSTRAINT"): + env.pop(name, None) + env["PYTHONNOUSERSITE"] = "1" + env["PIP_CONFIG_FILE"] = os.devnull + env["UV_NO_CONFIG"] = "1" + cache = data_dir() / "cache" + env["XDG_CACHE_HOME"] = str(cache) + env["PIP_CACHE_DIR"] = str(cache / "pip") + env["UV_CACHE_DIR"] = str(cache / "uv") + env["UV_PYTHON_INSTALL_DIR"] = str(data_dir() / "uv/python") + tmp = data_dir() / "tmp" + try: + tmp.mkdir(parents=True, exist_ok=True) + env["TMPDIR"] = str(tmp) + except OSError: + pass # Best effort: never break subprocesses over temporary-file containment. + return env + + +def terminate_group(process: subprocess.Popen) -> None: + """SIGTERM the child's whole process group, escalate to SIGKILL, and reap. + + The group is signalled even when the leader has already exited, because + descendants can outlive it while still holding the output pipe open. + A race between exit and signalling is tolerated. + """ + for sig in (signal.SIGTERM, signal.SIGKILL): + try: + os.killpg(process.pid, sig) + except ProcessLookupError: + pass + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + continue + try: + os.killpg(process.pid, 0) + except ProcessLookupError: + return + process.wait() + + +def run_process(args: list[str], stage: str, progress: Progress | None = None, + *, env: dict[str, str] | None = None) -> None: + """Drain output without blocking; send heartbeat callbacks every 0.25s. + + Worker JSON records carry frame/byte counts. Other processes report elapsed seconds + as completed with total=None. Only the last 64 KiB of diagnostics are retained. + Callback exceptions (including cancellation) terminate the process group. + """ + started = time.monotonic() + current_stage, completed, total = stage, None, None + diagnostics: deque[bytes] = deque(maxlen=16) + pending = b"" + if progress: + progress(stage, None, None) + with subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + env=env, start_new_session=True) as process: + assert process.stdout is not None + os.set_blocking(process.stdout.fileno(), False) + try: + with selectors.DefaultSelector() as selector: + selector.register(process.stdout, selectors.EVENT_READ) + while selector.get_map() or process.poll() is None: + for key, _ in selector.select(timeout=0.25): + chunk = os.read(key.fd, 4096) + if not chunk: + selector.unregister(key.fileobj) + continue + diagnostics.append(chunk) + pending += chunk + while b"\n" in pending: + line, pending = pending.split(b"\n", 1) + if line.startswith(b"VOICEFORGE_PROGRESS "): + event = json.loads(line[len(b"VOICEFORGE_PROGRESS "):]) + current_stage, completed, total = event + if len(pending) > 65536: + pending = pending[-4096:] + if progress: + progress(current_stage, completed if total is not None else + time.monotonic() - started, total) + code = process.wait() + if code: + detail = b"".join(diagnostics).decode("utf-8", errors="replace") + raise RuntimeError(f"{stage} failed (exit {code}):\n{detail}") + except BaseException: + terminate_group(process) + raise + + +def _uv_supports_relocatable(uv: str) -> bool: + """Whether this uv understands the venv flags AI setup relies on. + + A host uv can be older than the pinned one; capability is checked + directly instead of trusting the version string. + """ + try: + result = subprocess.run([uv, "venv", "--help"], capture_output=True, + text=True, check=True) + except (OSError, subprocess.SubprocessError): + return False + return "--relocatable" in result.stdout + result.stderr + + +def ensure_uv(progress: Progress | None = None) -> str: + """Return a uv that supports the flags AI setup relies on. + + The launcher's contained bootstrap uv is preferred; a host uv is trusted + only if it actually supports --relocatable. Otherwise uv==0.8.17 is + installed from PyPI (trust PyPI's distribution, not a downloaded shell + script). + """ + contained = data_dir() / "bootstrap/bin/uv" + if contained.is_file() and os.access(contained, os.X_OK): + return str(contained) + found = shutil.which("uv") + if found and _uv_supports_relocatable(found): + return found + root = contained.parent.parent + root.parent.mkdir(parents=True, exist_ok=True) + python = sys.executable or shutil.which("python3") + env = tool_env() + run_process([python, "-m", "venv", str(root)], "Creating uv bootstrap", + progress, env=env) + run_process([str(root / "bin/python"), "-m", "pip", "install", + "--index-url", "https://pypi.org/simple", "uv==0.8.17"], + "Installing uv from PyPI", progress, env=env) + if not (contained.is_file() and os.access(contained, os.X_OK)): + raise RuntimeError("Installing uv from PyPI did not produce a usable " + "uv; check for host pip settings (such as " + "PIP_TARGET) that redirect installations outside " + "the data directory.") + return str(contained) + + +def ensure_ffmpeg() -> str: + """Return the bundled FFmpeg, or a system one where the bundle is + absent, not ffprobe. + + The launcher installs imageio-ffmpeg via the bundled-ffmpeg extra; direct + package installs should declare it as a dependency. This helper never + runs pip in the current interpreter. + """ + try: + import imageio_ffmpeg + return imageio_ffmpeg.get_ffmpeg_exe() + except (ImportError, RuntimeError) as error: + found = shutil.which("ffmpeg") + if found: + return found + raise RuntimeError("FFmpeg is unavailable. Rebuild the contained " + "runtime with './producer.sh --rebuild', or " + "install a system FFmpeg.") from error diff --git a/lib/project/tests/conftest.py b/lib/project/tests/conftest.py new file mode 100644 index 0000000..7c75358 --- /dev/null +++ b/lib/project/tests/conftest.py @@ -0,0 +1,43 @@ +from pathlib import Path +import sys + +import numpy as np +import pytest +import soundfile as sf + +# Support running the suite before the project is installed. +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from voiceforge.config import resolve +from voiceforge.setup import ensure_ffmpeg + + +@pytest.fixture +def progress(): + return lambda *_: None + + +@pytest.fixture +def wav(tmp_path): + def write(samples=None, *, rate=48000, name="input.wav"): + if samples is None: + time = np.arange(rate * 3) / rate + samples = 0.15 * np.sin(2 * np.pi * 440 * time) + path = tmp_path / name + sf.write(path, np.asarray(samples), rate, format="WAV", subtype="FLOAT") + return path + + return write + + +@pytest.fixture +def bypass(): + return resolve(profile="cleanup-only", overrides={"denoiser": "none", "highpass": False}) + + +@pytest.fixture +def ffmpeg(): + try: + return ensure_ffmpeg() + except RuntimeError as error: + pytest.skip(str(error)) diff --git a/lib/project/tests/test_ai.py b/lib/project/tests/test_ai.py new file mode 100644 index 0000000..e8c5872 --- /dev/null +++ b/lib/project/tests/test_ai.py @@ -0,0 +1,366 @@ +"""Offline AI contract tests; no Torch installation, model download, or GPU needed.""" +import hashlib +import io +import json +import os +from pathlib import Path +import sys +import time +from types import SimpleNamespace +from unittest.mock import Mock +import zipfile + +import numpy as np +import pytest +import soundfile as sf + +from voiceforge import ai, ai_worker, setup + + +MODEL_BYTES = 7986207 +MODEL_HASH = "49c52edc8947ae1f9bf50d81530beaf3a2c3245aeaf34b6f31ff535cd22284d2" + + +@pytest.mark.parametrize("size,message", [ + (10, "checksum mismatch"), + (MODEL_BYTES, "checksum mismatch"), + (MODEL_BYTES + 1, "exceeds the expected size"), +]) +def test_model_rejects_truncated_corrupt_and_oversized_downloads(tmp_path, monkeypatch, size, message): + monkeypatch.setattr(ai_worker.urllib.request, "urlopen", Mock(return_value=io.BytesIO(b"x" * size))) + progress = Mock() + monkeypatch.setattr(ai_worker, "report", progress) + with pytest.raises(RuntimeError, match=message): + ai_worker.ensure_model(tmp_path / "model") + assert list(tmp_path.iterdir()) == [] + events = [call.args for call in progress.call_args_list] + assert events[0] == ("Downloading DeepFilterNet3 model (bytes)", 0, MODEL_BYTES) + counts = [completed for _, completed, total in events if total == MODEL_BYTES] + assert counts == sorted(counts) + assert all(0 <= count <= MODEL_BYTES for count in counts) + + +@pytest.mark.parametrize("unsafe", [False, True]) +def test_mocked_model_archive_progress_extraction_and_reuse(tmp_path, monkeypatch, unsafe): + # Mock only the release digest for a synthetic archive; corruption tests above + # exercise the real SHA-256 gate. Keep its real byte count and ZIP extraction. + def archive(padding): + stream = io.BytesIO() + with zipfile.ZipFile(stream, "w") as bundle: + bundle.writestr("DeepFilterNet3/config.ini", "[df]\nsr=48000\n") + bundle.writestr("DeepFilterNet3/checkpoints/model_120.ckpt.best", "checkpoint") + bundle.writestr("../escape" if unsafe else "padding", b"x" * padding) + return stream.getvalue() + + payload = archive(MODEL_BYTES - len(archive(0))) + assert len(payload) == MODEL_BYTES + digest = Mock(wraps=hashlib.sha256()) + digest.hexdigest.return_value = MODEL_HASH + monkeypatch.setattr(ai_worker.hashlib, "sha256", lambda: digest) + download = Mock(return_value=io.BytesIO(payload)) + monkeypatch.setattr(ai_worker.urllib.request, "urlopen", download) + progress = Mock() + monkeypatch.setattr(ai_worker, "report", progress) + directory = tmp_path / "model" + if unsafe: + with pytest.raises(RuntimeError, match="Unsafe path"): + ai_worker.ensure_model(directory) + assert not (tmp_path / "escape").exists() + assert list(tmp_path.iterdir()) == [] + else: + model = ai_worker.ensure_model(directory) + assert (model / "config.ini").is_file() + assert (model / "checkpoints/model_120.ckpt.best").is_file() + assert ai_worker.ensure_model(directory) == model + download.assert_called_once() + assert b"".join(call.args[0] for call in digest.update.call_args_list) == payload + events = [call.args for call in progress.call_args_list] + byte_events = [event for event in events if "(bytes)" in event[0]] + assert byte_events[0][1:] == (0, MODEL_BYTES) + assert byte_events[-1][1:] == (MODEL_BYTES, MODEL_BYTES) + assert len(byte_events) > 2 + assert events[-1] == ("Verifying and extracting DeepFilterNet3 model",) + + +def test_incomplete_model_fails_without_network(tmp_path, monkeypatch): + directory = tmp_path / "model" + directory.mkdir() + download = Mock(side_effect=AssertionError("Unexpected network call")) + monkeypatch.setattr(ai_worker.urllib.request, "urlopen", download) + with pytest.raises(RuntimeError, match="Incomplete model directory"): + ai_worker.ensure_model(directory) + download.assert_not_called() + + +@pytest.mark.parametrize("close_output", [False, True]) +def test_subprocess_heartbeat_does_not_block_on_silence_or_closed_output(close_output): + events = [] + code = "import os,time; " + if close_output: + code += "os.close(1); os.close(2); " + code += "time.sleep(0.8)" + setup.run_process([sys.executable, "-c", code], "Waiting", lambda *event: events.append(event)) + assert events[0] == ("Waiting", None, None) + elapsed = [completed for _, completed, total in events[1:] if total is None] + assert len(elapsed) >= 2 + assert elapsed == sorted(elapsed) + + +def test_subprocess_parses_split_byte_progress_records(): + events = [] + record = "VOICEFORGE_PROGRESS " + json.dumps(["Download (bytes)", 123, 456]) + "\n" + code = f"import os,time; os.write(1,{record[:12].encode()!r}); time.sleep(.3); os.write(1,{record[12:].encode()!r})" + setup.run_process([sys.executable, "-c", code], "Download", lambda *event: events.append(event)) + assert events[-1] == ("Download (bytes)", 123, 456) + + +@pytest.mark.parametrize("exception", [KeyboardInterrupt, RuntimeError]) +def test_callback_cancellation_terminates_and_reaps_child(monkeypatch, exception): + popen = setup.subprocess.Popen + children = [] + + def launch(*args, **kwargs): + child = popen(*args, **kwargs) + children.append(child) + return child + + def cancel(stage, completed, total): + if completed is not None: + raise exception("cancelled") + + monkeypatch.setattr(setup.subprocess, "Popen", launch) + started = time.monotonic() + with pytest.raises(exception, match="cancelled"): + setup.run_process([sys.executable, "-c", "import time; time.sleep(30)"], "Waiting", cancel) + assert len(children) == 1 and children[0].poll() is not None + assert time.monotonic() - started < 10 + + +def test_cancellation_kills_descendants_that_outlive_the_leader(tmp_path, monkeypatch): + sentinel = tmp_path / "grandchild.pid" + monkeypatch.setenv("GRANDCHILD_PID_FILE", str(sentinel)) + child_code = ( + "import os, subprocess, sys, time\n" + "subprocess.Popen([sys.executable, '-c', " + "\"import os, time; open(os.environ['GRANDCHILD_PID_FILE'], 'w')" + ".write(str(os.getpid())); time.sleep(60)\"])\n" + "while not os.path.exists(os.environ['GRANDCHILD_PID_FILE']):\n" + " time.sleep(0.02)\n" + ) + + def cancel(stage, completed, total): + if completed is not None and children and children[0].poll() is not None: + raise KeyboardInterrupt("cancelled") + + popen = setup.subprocess.Popen + children = [] + + def launch(*args, **kwargs): + child = popen(*args, **kwargs) + children.append(child) + return child + + monkeypatch.setattr(setup.subprocess, "Popen", launch) + with pytest.raises(KeyboardInterrupt, match="cancelled"): + setup.run_process([sys.executable, "-c", child_code], "Waiting", cancel) + + def alive(pid): + try: + for line in Path(f"/proc/{pid}/status").read_text().splitlines(): + if line.startswith("State:"): + return "Z" not in line + return True + except FileNotFoundError: + return False + + grandchild = int(sentinel.read_text()) + for _ in range(50): + if not alive(grandchild): + break + time.sleep(0.1) + else: + pytest.fail("A descendant survived the cancellation of its process group") + + +def test_subprocess_failure_keeps_bounded_diagnostics(): + code = "import sys; print('x'*100000); print('specific failure'); sys.exit(7)" + with pytest.raises(RuntimeError, match="exit 7") as error: + setup.run_process([sys.executable, "-c", code], "Worker") + assert "specific failure" in str(error.value) + assert len(str(error.value)) < 66000 + + +@pytest.mark.parametrize("strength", [-1, 2, float("nan"), float("inf")]) +def test_invalid_strength_fails_before_setup(tmp_path, monkeypatch, strength): + install = Mock(side_effect=AssertionError("Unexpected setup")) + monkeypatch.setattr(ai, "ensure_ai", install) + with pytest.raises(ValueError, match="strength"): + ai.denoise(tmp_path / "input.wav", tmp_path / "output.wav", strength=strength) + install.assert_not_called() + + +def _ready_environment(root, flavor="cpu"): + environment = root / f"ai-df-0.5.6-torch-2.5.1-{flavor}-py311-v1" + (environment / "bin").mkdir(parents=True) + (environment / "bin/python").write_text("#!/bin/sh\n") + (environment / ".voiceforge-ready").touch() + return environment + + +def test_ready_environment_skips_the_check_worker(tmp_path, monkeypatch): + root = tmp_path / "prefix" + environment = _ready_environment(root) + monkeypatch.setenv("VOICEFORGE_HOME", str(root)) + monkeypatch.setattr(ai.os, "confstr", lambda *_: "glibc 2.39") + executed = [] + monkeypatch.setattr(ai, "run_process", lambda args, *rest, **kwargs: executed.append(args)) + python = ai.ensure_ai("cpu") + assert executed == [] + assert python == environment / "bin/python" + + +def test_setup_verifies_even_a_ready_environment(tmp_path, monkeypatch): + root = tmp_path / "prefix" + _ready_environment(root) + monkeypatch.setenv("VOICEFORGE_HOME", str(root)) + monkeypatch.setattr(ai.os, "confstr", lambda *_: "glibc 2.39") + executed = [] + monkeypatch.setattr(ai, "run_process", lambda args, *rest, **kwargs: executed.append(args)) + ai.ensure_ai("cpu", verify=True) + assert len(executed) == 1 + assert executed[0][-1] == "--check" + + +def test_broken_environment_is_rebuilt_and_self_tested(tmp_path, monkeypatch): + root = tmp_path / "prefix" + environment = _ready_environment(root) + (environment / "bin/python").unlink() + (environment / "bin/python").symlink_to("/nonexistent/python3.11") + leftover = environment / "leftover.txt" + leftover.write_text("from the previous location") + monkeypatch.setenv("VOICEFORGE_HOME", str(root)) + monkeypatch.setattr(ai.os, "confstr", lambda *_: "glibc 2.39") + stages, commands = [], [] + + def fake_run(args, stage, progress=None, **kwargs): + commands.append(args) + stages.append(stage) + if stage == "Checking DeepFilterNet3": + environment.mkdir(parents=True, exist_ok=True) + (environment / ".voiceforge-ready").touch() + + monkeypatch.setattr(ai, "run_process", fake_run) + monkeypatch.setattr(ai, "ensure_uv", lambda progress=None: "uv") + ai.ensure_ai("cpu") + assert not leftover.exists() + assert stages == ["Preparing AI Python 3.11", "Installing PyTorch (cpu)", + "Installing DeepFilterNet3", "Checking DeepFilterNet3"] + assert commands[0][-1] == str(environment) + assert (environment / ".voiceforge-ready").is_file() + + +def test_copied_environment_is_rebuilt_not_trusted(tmp_path, monkeypatch): + # Copying an installation while the original remains leaves the copy's + # interpreter links resolving into the original prefix; that must not + # count as ready, or the copy breaks when the original is removed. + root = tmp_path / "prefix" + environment = _ready_environment(root) + original = _ready_environment(tmp_path / "other") + (environment / "bin/python").unlink() + (environment / "bin/python").symlink_to(original / "bin/python") + monkeypatch.setenv("VOICEFORGE_HOME", str(root)) + monkeypatch.setattr(ai.os, "confstr", lambda *_: "glibc 2.39") + stages = [] + + def fake_run(args, stage, progress=None, **kwargs): + stages.append(stage) + if stage == "Checking DeepFilterNet3": + environment.mkdir(parents=True, exist_ok=True) + (environment / ".voiceforge-ready").touch() + + monkeypatch.setattr(ai, "run_process", fake_run) + monkeypatch.setattr(ai, "ensure_uv", lambda progress=None: "uv") + ai.ensure_ai("cpu") + assert stages[0] == "Preparing AI Python 3.11" + + +@pytest.fixture +def mocked_worker(tmp_path, monkeypatch): + class Tensor: + def __init__(self, array): + self.array = array + + def unsqueeze(self, axis): + return Tensor(np.expand_dims(self.array, axis)) + + def squeeze(self, axis): + return Tensor(np.squeeze(self.array, axis)) + + def numpy(self): + return self.array + + calls = [] + + def enhance(model, state, tensor, pad): + assert pad is True + assert tensor.array.shape[1] % 480 == 0 + calls.append(tensor.array.shape[1]) + return Tensor(tensor.array * 0.5) + + state = SimpleNamespace(sr=lambda: 48000) + backend = SimpleNamespace(enhance=enhance, init_df=lambda *a, **kw: (None, state, None)) + for name, module in { + "torch": SimpleNamespace(set_num_threads=lambda n: None, from_numpy=Tensor, + cuda=SimpleNamespace(is_available=lambda: False)), + "df": SimpleNamespace(), + "df.enhance": backend, + "df.model": SimpleNamespace(ModelParams=lambda: SimpleNamespace( + sr=48000, fft_size=960, hop_size=480, nb_erb=32, min_nb_freqs=2)), + "libdf": SimpleNamespace(DF=lambda **kw: state), + }.items(): + monkeypatch.setitem(sys.modules, name, module) + monkeypatch.setattr(ai_worker, "ensure_model", lambda path: path) + monkeypatch.setattr(ai_worker.signal, "signal", lambda *args: None) + monkeypatch.setenv("DEVICE", "cpu") + progress = Mock() + monkeypatch.setattr(ai_worker, "report", progress) + + def run(source, target, strength): + monkeypatch.setattr(sys, "argv", ["worker", "--device", "cpu", "--model-dir", str(tmp_path), + "--source", str(source), "--target", str(target), + "--strength", str(strength)]) + ai_worker.main() + + return run, calls, backend, progress + + +@pytest.mark.parametrize("frames", [0, 1, 479, 481, 480001, 960017]) +@pytest.mark.parametrize("strength", [0, 0.85, 1]) +def test_mocked_worker_exact_length_mix_and_bounded_windows(tmp_path, wav, mocked_worker, frames, strength): + run, calls, _, progress = mocked_worker + audio = np.random.default_rng(42).normal(0, 0.1, frames).astype("float32") + source = wav(audio) + target = tmp_path / "clean.wav" + run(source, target, strength) + result, rate = sf.read(target, dtype="float32") + assert len(result) == frames and rate == 48000 + assert sf.info(target).subtype == "FLOAT" + np.testing.assert_allclose(result, audio * (1 - strength * 0.5), rtol=2e-6, atol=1e-8) + if strength == 0: + np.testing.assert_array_equal(result, audio) + assert calls == [] + assert max(calls, default=0) <= 674400 + assert progress.call_args.args == ("Denoising (cpu)", frames, frames) + assert not list(tmp_path.glob(".voiceforge-ai-*")) + + +def test_worker_failure_preserves_existing_target(tmp_path, wav, mocked_worker): + run, _, backend, _ = mocked_worker + backend.enhance = Mock(side_effect=RuntimeError("inference failed")) + source = wav() + target = tmp_path / "existing.wav" + target.write_bytes(b"existing output") + with pytest.raises(RuntimeError, match="inference failed"): + run(source, target, 0.85) + assert target.read_bytes() == b"existing output" + assert not list(tmp_path.glob(".voiceforge-ai-*")) diff --git a/lib/project/tests/test_cli.py b/lib/project/tests/test_cli.py new file mode 100644 index 0000000..a5437b7 --- /dev/null +++ b/lib/project/tests/test_cli.py @@ -0,0 +1,287 @@ +import json +from pathlib import Path +import sys + +import numpy as np +import pytest +import soundfile as sf + +from voiceforge import cli, pipeline + + +@pytest.fixture +def invoke(monkeypatch): + def run(*args): + monkeypatch.setattr(sys, "argv", ["voiceforge", *map(str, args)]) + return cli.main() + + return run + + +def test_preview_real_audio_preserves_original_and_reports_provenance( + wav, tmp_path, ffmpeg, invoke, capsys, monkeypatch +): + time = np.arange(48000 * 7) / 48000 + source = wav(0.15 * np.sin(2 * np.pi * 440 * time)) + original = source.read_bytes() + output_dir = tmp_path / "previews" + modes = [] + real_run = pipeline.ffmpeg_run + + def record_render(*args, **kwargs): + log = real_run(*args, **kwargs) + if args[2] == "Mastering (pass 2)": + modes.append(json.loads(log[log.rfind("{"):log.rfind("}") + 1])["normalization_type"]) + return log + + monkeypatch.setattr(pipeline, "ffmpeg_run", record_render) + invoke("preview", source, "--no-denoise", "--start", "1", "--duration", "5", + "--profiles", "natural", "radio", "--no-normalize", "--no-limiter", + "--output-dir", output_dir) + + captured = capsys.readouterr() + assert captured.out == "" + assert "Preview:" in captured.err + assert source.read_bytes() == original + assert len(modes) == 2 + assert {path.name for path in output_dir.iterdir()} == { + f"input.{profile}.preview{suffix}" + for profile in ("natural", "radio") for suffix in (".wav", ".report.json") + } + for profile, mode in zip(("natural", "radio"), modes): + target = output_dir / f"input.{profile}.preview.wav" + report = json.loads(target.with_suffix(".report.json").read_text()) + info = sf.info(target) + assert info.duration == pytest.approx(5, abs=0.02) + assert info.channels == 1 + assert info.subtype == "PCM_24" + assert report["output"] == str(target) + assert report["provenance"] == { + "original": str(source.resolve()), "preview_start_seconds": 1, + "requested_duration_seconds": 5, "normalization_forced": True, + } + assert not Path(report["source"]).exists() + assert report["settings"]["profile"] == profile + assert report["settings"]["denoiser"] == "none" + assert report["settings"]["normalize"] is True + assert report["settings"]["limiter"] is True + assert report["settings"]["compressor_ratio"] == {"natural": 2.0, "radio": 3.5}[profile] + assert report["normalization_mode"] == mode + assert report["master_loudness"]["input_i"] == pytest.approx(-19, abs=0.5) + assert report["master_loudness"]["input_tp"] <= -1.4 + + +def test_config_stdout_json_and_set_precedence(tmp_path, invoke, capsys): + config = tmp_path / "settings.toml" + config.write_text('profile = "cleanup-only"\ntarget_lufs = -24.0\n') + invoke("config", "--json", "--config", config, "--profile", "narrator", + "--target-lufs", "-21", "--set", "target_lufs=-18", + "--set", "profile=radio", "--no-denoise") + captured = capsys.readouterr() + settings = json.loads(captured.out) + assert captured.err == "" + assert settings["profile"] == "radio" + assert settings["compressor_ratio"] == 3.5 + assert settings["compressor_attack_ms"] == 8 + assert settings["normalize"] is True + assert settings["target_lufs"] == -18 + assert settings["denoiser"] == "none" + + +def test_analyze_stdout_is_json(wav, ffmpeg, invoke, capsys): + source = wav() + original = source.read_bytes() + invoke("analyze", source, "--no-denoise") + stats = json.loads(capsys.readouterr().out) + assert stats["duration_seconds"] == pytest.approx(3) + assert stats["channels"] == 1 + assert stats["frames"] == 144000 + assert np.isfinite(stats["loudness"]["input_i"]) + assert source.read_bytes() == original + assert list(source.parent.iterdir()) == [source] + + +@pytest.mark.parametrize("start", ["nan", "inf", "-inf"]) +def test_preview_rejects_nonfinite_start_before_tool_setup( + tmp_path, invoke, capsys, monkeypatch, start +): + monkeypatch.setattr(cli, "ensure_ffmpeg", lambda: pytest.fail("Invalid start reached FFmpeg")) + output_dir = tmp_path / "previews" + with pytest.raises(SystemExit) as error: + invoke("preview", tmp_path / "missing.wav", f"--start={start}", + "--output-dir", output_dir) + assert error.value.code == 1 + captured = capsys.readouterr() + assert captured.out == "" + assert "Preview start" in captured.err + assert not output_dir.exists() + + +def test_unknown_setting_is_a_cli_error(invoke, capsys): + with pytest.raises(SystemExit) as error: + invoke("config", "--json", "--set", "unknown_setting=1") + assert error.value.code == 1 + captured = capsys.readouterr() + assert captured.out == "" + assert "Unknown setting(s): unknown_setting" in captured.err + + +def test_missing_input_fails_before_tool_setup(tmp_path, invoke, capsys, monkeypatch): + monkeypatch.setattr(pipeline, "ensure_ffmpeg", lambda: pytest.fail("Missing input reached FFmpeg")) + target = tmp_path / "master.wav" + with pytest.raises(SystemExit) as error: + invoke("process", tmp_path / "missing.wav", "--no-denoise", "-o", target) + assert error.value.code == 1 + assert "existing WAV file" in capsys.readouterr().err + assert not target.exists() + + +def test_existing_output_is_not_overwritten(wav, tmp_path, invoke, capsys, monkeypatch): + source = wav() + original = source.read_bytes() + target = tmp_path / "master.wav" + target.write_bytes(b"previous master") + monkeypatch.setattr(pipeline, "ensure_ffmpeg", lambda: pytest.fail("Existing output reached FFmpeg")) + with pytest.raises(SystemExit) as error: + invoke("process", source, "--no-denoise", "-o", target) + assert error.value.code == 1 + assert "Output exists" in capsys.readouterr().err + assert target.read_bytes() == b"previous master" + assert source.read_bytes() == original + assert not target.with_suffix(".report.json").exists() + + +def test_overwrite_cannot_replace_input(wav, invoke, capsys, monkeypatch): + source = wav() + original = source.read_bytes() + monkeypatch.setattr(cli, "process", lambda *a, **kw: pytest.fail("Input collision reached processing")) + with pytest.raises(SystemExit) as error: + invoke("process", source, "-o", source, "--overwrite") + assert error.value.code == 1 + assert "overwrite an input recording" in capsys.readouterr().err + assert source.read_bytes() == original + + +def test_duplicate_batch_names_fail_before_processing(wav, tmp_path, invoke, capsys, monkeypatch): + (tmp_path / "other").mkdir() + sources = [wav(), wav(name="other/input.wav")] + originals = [source.read_bytes() for source in sources] + output_dir = tmp_path / "masters" + monkeypatch.setattr(cli, "process", lambda *a, **kw: pytest.fail("Duplicate names reached processing")) + with pytest.raises(SystemExit) as error: + invoke("process", *sources, "--output-dir", output_dir, "--overwrite") + assert error.value.code == 1 + assert "duplicate output names" in capsys.readouterr().err + assert not output_dir.exists() + assert [source.read_bytes() for source in sources] == originals + + +def test_default_outputs_are_written_beside_each_input(wav, tmp_path, ffmpeg, invoke, capsys): + (tmp_path / "a").mkdir() + (tmp_path / "b").mkdir() + sources = [wav(name="a/take.wav"), wav(name="b/take.wav")] + invoke("process", *sources, "--no-denoise") + for source in sources: + assert {path.name for path in source.parent.iterdir()} == { + "take.wav", "take.natural.wav", "take.natural.report.json"} + captured = capsys.readouterr() + assert captured.out == "" + assert captured.err.count("Written:") == 2 + + +def test_output_and_output_dir_cannot_be_combined(wav, tmp_path, invoke, capsys): + with pytest.raises(SystemExit) as error: + invoke("process", wav(), "-o", tmp_path / "x.wav", "--output-dir", tmp_path / "outs") + assert error.value.code == 1 + assert "--output-dir cannot be combined" in capsys.readouterr().err + + +def test_missing_later_input_fails_before_any_processing(wav, tmp_path, invoke, capsys, monkeypatch): + source = wav() + monkeypatch.setattr(cli, "process", lambda *a, **kw: pytest.fail("Preflight must reject the batch")) + with pytest.raises(SystemExit) as error: + invoke("process", source, tmp_path / "missing.wav", "--no-denoise") + assert error.value.code == 1 + assert "existing WAV file" in capsys.readouterr().err + assert not (tmp_path / "input.natural.wav").exists() + + +def test_existing_output_in_batch_fails_before_any_processing(wav, tmp_path, invoke, capsys, monkeypatch): + (tmp_path / "a").mkdir() + (tmp_path / "b").mkdir() + sources = [wav(name="a/take.wav"), wav(name="b/take.wav")] + (tmp_path / "b" / "take.natural.wav").write_bytes(b"previous result") + monkeypatch.setattr(cli, "process", lambda *a, **kw: pytest.fail("Preflight must reject the batch")) + with pytest.raises(SystemExit) as error: + invoke("process", *sources, "--no-denoise") + assert error.value.code == 1 + assert "Output exists" in capsys.readouterr().err + assert not (tmp_path / "a" / "take.natural.wav").exists() + + +def test_preview_rejects_duplicate_profiles_and_profile_override(wav, invoke, capsys, monkeypatch): + source = wav() + monkeypatch.setattr(cli, "ensure_ffmpeg", lambda: pytest.fail("Invalid preview plan reached FFmpeg")) + with pytest.raises(SystemExit) as error: + invoke("preview", source, "--no-denoise", "--profiles", "natural", "natural") + assert error.value.code == 1 + assert "unique" in capsys.readouterr().err + with pytest.raises(SystemExit) as error: + invoke("preview", source, "--no-denoise", "--profiles", "natural", + "--set", "profile=radio") + assert error.value.code == 1 + assert "remove the profile override" in capsys.readouterr().err + assert list(source.parent.iterdir()) == [source] + + +def test_preview_default_output_is_written_beside_input(wav, tmp_path, ffmpeg, invoke, capsys): + source = wav() + original = source.read_bytes() + invoke("preview", source, "--no-denoise", "--start", "1", "--duration", "2", + "--profiles", "natural") + assert {path.name for path in tmp_path.iterdir()} == { + "input.wav", "input.natural.preview.wav", "input.natural.preview.report.json"} + report = json.loads((tmp_path / "input.natural.preview.report.json").read_text()) + assert report["settings"]["profile"] == "natural" + assert source.read_bytes() == original + + +def test_preview_cancellation_cleans_excerpt(wav, tmp_path, ffmpeg, invoke, capsys, monkeypatch): + source = wav() + original = source.read_bytes() + output_dir = tmp_path / "previews" + excerpts = [] + + def cancel(excerpt, *args, **kwargs): + assert excerpt.is_file() + excerpts.append(excerpt) + raise KeyboardInterrupt + + monkeypatch.setattr(cli, "process", cancel) + with pytest.raises(SystemExit) as error: + invoke("preview", source, "--no-denoise", "--output-dir", output_dir) + assert error.value.code == 130 + assert "Cancelled" in capsys.readouterr().err + assert len(excerpts) == 1 + assert not excerpts[0].exists() + assert list(output_dir.iterdir()) == [] + assert source.read_bytes() == original + + +@pytest.mark.parametrize("suffix", [".wav", ".report.json", ".mp3"]) +def test_nonfile_output_with_overwrite_fails_before_tool_setup( + wav, tmp_path, bypass, progress, monkeypatch, suffix +): + source = wav() + original = source.read_bytes() + target = tmp_path / "master.wav" + nonfile = target.with_suffix(suffix) + nonfile.mkdir() + bypass.mp3 = True + monkeypatch.setattr(pipeline, "ensure_ffmpeg", lambda: pytest.fail("Non-file output reached FFmpeg")) + with pytest.raises(ValueError, match="Output is not a regular file"): + pipeline.process(source, target, bypass, progress, overwrite=True) + assert nonfile.is_dir() + assert list(nonfile.iterdir()) == [] + assert set(tmp_path.iterdir()) == {source, nonfile} + assert source.read_bytes() == original diff --git a/lib/project/tests/test_config.py b/lib/project/tests/test_config.py new file mode 100644 index 0000000..8144fc8 --- /dev/null +++ b/lib/project/tests/test_config.py @@ -0,0 +1,134 @@ +from dataclasses import asdict, fields + +import pytest + +from voiceforge.cli import main, parser, settings_for +from voiceforge.config import PROFILES, RANGES, Settings, resolve, toml +from voiceforge.pipeline import filters + + +@pytest.mark.parametrize("profile", PROFILES) +def test_profiles_and_toml_round_trip(profile, tmp_path): + settings = resolve(profile=profile) + assert asdict(settings) == asdict(Settings()) | PROFILES[profile] | {"profile": profile} + path = tmp_path / "settings.toml" + path.write_text(toml(settings)) + assert resolve(path) == settings + + +def test_precedence_defaults_profile_file_flags_then_set(tmp_path): + path = tmp_path / "settings.toml" + path.write_text('profile = "narrator"\ncompressor_ratio = 2.2\nwarmth_db = 3\n') + args = parser().parse_args([ + "config", "--config", str(path), "--profile", "radio", + "--warmth-db", "4", "--set", "warmth_db=5", "--set", "warmth_db=6", + ]) + settings = settings_for(args) + assert settings.profile == "radio" + assert settings.compressor_attack_ms == 8 # Selected profile beats file profile. + assert settings.compressor_ratio == 2.2 # File values beat profile defaults. + assert settings.warmth_db == 6 # Last --set beats named flags. + assert settings.sample_rate == 48000 + + +@pytest.mark.parametrize("field", [f.name for f in fields(Settings) if type(getattr(Settings(), f.name)) is bool]) +def test_boolean_flags_preserve_unspecified_and_accept_both_forms(field): + assert getattr(parser().parse_args(["config"]), field) is None + for enabled in (True, False): + flag = "--" + ("" if enabled else "no-") + field.replace("_", "-") + extra = ["--limiter"] if field == "normalize" and enabled else [] + args = parser().parse_args(["config", "--profile", "cleanup-only", flag, *extra]) + assert getattr(settings_for(args), field) is enabled + + +def test_false_cli_toggle_overrides_true_file_value(tmp_path): + path = tmp_path / "settings.toml" + path.write_text("eq = true\n") + args = parser().parse_args(["config", "--config", str(path), "--no-eq"]) + assert settings_for(args).eq is False + + +def test_no_denoise_and_unquoted_set_strings(): + assert settings_for(parser().parse_args(["config", "--no-denoise"])).denoiser == "none" + args = parser().parse_args(["config", "--no-denoise", "--set", "denoiser=fft"]) + assert settings_for(args).denoiser == "fft" + + +@pytest.mark.parametrize("key", RANGES) +def test_numeric_ranges_are_inclusive_and_reject_outside(key): + low, high = RANGES[key] + for value in (low, high): + assert getattr(resolve(overrides={key: value}), key) == value + for value in (low - 0.01, high + 0.01): + with pytest.raises(ValueError, match=key): + resolve(overrides={key: value}) + + +@pytest.mark.parametrize("overrides,match", [ + ({"typo": True}, "Unknown setting"), + ({"highpass": 1}, "highpass must be bool"), + ({"sample_rate": 48000.0}, "sample_rate must be int"), + ({"target_lufs": True}, "finite number"), + ({"target_lufs": float("nan")}, "finite number"), + ({"target_lufs": float("inf")}, "finite number"), + ({"denoiser": "unknown"}, "denoiser must be one of"), + ({"device": "gpu"}, "device must be one of"), + ({"channel": "stereo"}, "channel must be one of"), + ({"hum_hz": 55}, "hum_hz must be one of"), + ({"sample_rate": 96000}, "sample_rate must be one of"), + ({"mp3_bitrate": 64}, "mp3_bitrate must be one of"), + ({"limiter": False}, "Normalization includes true-peak limiting"), +]) +def test_invalid_configuration(overrides, match): + with pytest.raises(ValueError, match=match): + resolve(overrides=overrides) + + +def test_unknown_profile_and_malformed_toml(tmp_path): + with pytest.raises(ValueError, match="Unknown profile"): + resolve(profile="missing") + path = tmp_path / "bad.toml" + path.write_text("normalize = [") + with pytest.raises(ValueError): + resolve(path) + + +def test_invalid_cli_configuration_fails_before_processing(monkeypatch, capsys): + def unexpected(*args, **kwargs): + pytest.fail("Invalid configuration must not reach processing or tool setup") + + monkeypatch.setattr("voiceforge.cli.process", unexpected) + monkeypatch.setattr("voiceforge.cli.ensure_ffmpeg", unexpected) + monkeypatch.setattr("sys.argv", ["voiceforge", "process", "missing.wav", "--target-lufs", "0"]) + with pytest.raises(SystemExit) as error: + main() + assert error.value.code == 1 + assert "target_lufs must be between" in capsys.readouterr().err + + +def test_set_requires_assignment(): + with pytest.raises(ValueError, match="KEY=VALUE"): + settings_for(parser().parse_args(["config", "--set", "normalize"])) + + +@pytest.mark.parametrize("profile", ["radio", "not-a-profile"]) +def test_set_profile_uses_normal_profile_resolution(profile): + args = parser().parse_args(["config", "--set", f"profile={profile}"]) + if profile not in PROFILES: + with pytest.raises(ValueError, match="Unknown profile"): + settings_for(args) + else: + assert settings_for(args) == resolve(profile=profile) + + +@pytest.mark.parametrize("toggle,prefix,count", [ + ("expansion", "agate=", 1), ("eq", "equalizer=", 3), + ("compression", "acompressor=", 1), ("deess", "deesser=", 1), + ("lowpass", "lowpass=", 1), +]) +def test_shaping_filter_toggles(bypass, toggle, prefix, count): + assert filters(bypass) == ["anull"] + setattr(bypass, toggle, True) + chain = filters(bypass) + assert len(chain) == count + assert all(item.startswith(prefix) for item in chain) diff --git a/lib/project/tests/test_install.py b/lib/project/tests/test_install.py new file mode 100644 index 0000000..f1dfcf2 --- /dev/null +++ b/lib/project/tests/test_install.py @@ -0,0 +1,225 @@ +"""Data-dir resolution, contained tool environment, and launcher script checks.""" +import os +import subprocess +from pathlib import Path +import sys +import types + +import pytest + +from voiceforge import setup + +ROOT = Path(__file__).resolve().parents[3] +LAUNCHER = ROOT / "producer.sh" + + +@pytest.fixture +def shell_env(tmp_path): + env = os.environ.copy() + env.pop("VOICEFORGE_HOME", None) + env.pop("XDG_DATA_HOME", None) + env["HOME"] = str(tmp_path / "home") + return env + + +# --- data_dir resolution --------------------------------------------------- + +def test_data_dir_defaults_and_precedence(tmp_path, monkeypatch): + monkeypatch.delenv("VOICEFORGE_HOME", raising=False) + monkeypatch.delenv("XDG_DATA_HOME", raising=False) + monkeypatch.setenv("HOME", str(tmp_path)) + assert setup.data_dir() == tmp_path / ".local/share/voiceforge" + monkeypatch.setenv("XDG_DATA_HOME", str(tmp_path / "xdg")) + assert setup.data_dir() == tmp_path / "xdg/voiceforge" + monkeypatch.setenv("VOICEFORGE_HOME", str(tmp_path / "prefix")) + assert setup.data_dir() == tmp_path / "prefix" + + +def test_relative_voiceforge_home_resolves_against_cwd(tmp_path, monkeypatch): + monkeypatch.setenv("VOICEFORGE_HOME", "voiceforge-home") + monkeypatch.chdir(tmp_path) + assert setup.data_dir() == tmp_path / "voiceforge-home" + + +def test_relative_xdg_data_home_falls_back_to_home(tmp_path, monkeypatch): + monkeypatch.delenv("VOICEFORGE_HOME", raising=False) + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("XDG_DATA_HOME", "relative/path") + assert setup.data_dir() == tmp_path / ".local/share/voiceforge" + + +# --- contained tool environment -------------------------------------------- + +def test_tool_env_is_sanitized_and_contained(tmp_path, monkeypatch): + monkeypatch.setenv("VOICEFORGE_HOME", str(tmp_path / "prefix")) + monkeypatch.setenv("PYTHONPATH", "/host/site-packages") + monkeypatch.setenv("PYTHONHOME", "/host/python") + monkeypatch.setenv("VIRTUAL_ENV", "/host/venv") + monkeypatch.setenv("PIP_TARGET", "/host/target") + monkeypatch.setenv("PIP_PREFIX", "/host/prefix") + monkeypatch.setenv("PIP_USER", "1") + monkeypatch.setenv("PIP_FIND_LINKS", "/host/wheels") + monkeypatch.setenv("PIP_INDEX_URL", "https://host.example/simple") + monkeypatch.setenv("PIP_EXTRA_INDEX_URL", "https://host.example/extra") + monkeypatch.setenv("PIP_CONSTRAINT", "/host/constraints.txt") + monkeypatch.setenv("UV_INDEX_URL", "https://host.example/simple") + monkeypatch.setenv("UV_DEFAULT_INDEX", "https://host.example/simple") + monkeypatch.setenv("UV_EXTRA_INDEX_URL", "https://host.example/extra") + monkeypatch.setenv("UV_INDEX", "https://host.example/extra") + monkeypatch.setenv("UV_FIND_LINKS", "/host/wheels") + monkeypatch.setenv("UV_CONSTRAINT", "/host/constraints.txt") + monkeypatch.delenv("PIP_CACHE_DIR", raising=False) + env = setup.tool_env() + prefix = str(tmp_path / "prefix") + assert "PYTHONPATH" not in env and "PYTHONHOME" not in env and "VIRTUAL_ENV" not in env + for name in ("PIP_TARGET", "PIP_PREFIX", "PIP_USER", "PIP_FIND_LINKS", + "PIP_INDEX_URL", "PIP_EXTRA_INDEX_URL", "PIP_CONSTRAINT", + "UV_INDEX_URL", "UV_DEFAULT_INDEX", "UV_EXTRA_INDEX_URL", + "UV_INDEX", "UV_FIND_LINKS", "UV_CONSTRAINT"): + assert name not in env, name + assert env["PYTHONNOUSERSITE"] == "1" + assert env["PIP_CONFIG_FILE"] == os.devnull + assert env["UV_NO_CONFIG"] == "1" + assert env["XDG_CACHE_HOME"] == f"{prefix}/cache" + assert env["PIP_CACHE_DIR"] == f"{prefix}/cache/pip" + assert env["UV_CACHE_DIR"] == f"{prefix}/cache/uv" + assert env["UV_PYTHON_INSTALL_DIR"] == f"{prefix}/uv/python" + assert env["TMPDIR"] == f"{prefix}/tmp" + assert (tmp_path / "prefix/tmp").is_dir() + + +def test_tool_env_tolerates_an_uncreatable_data_dir(monkeypatch): + monkeypatch.setenv("VOICEFORGE_HOME", "/vf-prefix-unwritable") + env = setup.tool_env() + assert env["PIP_CACHE_DIR"] == "/vf-prefix-unwritable/cache/pip" + assert "TMPDIR" not in env + + +def test_tool_env_leaves_process_environment_untouched(tmp_path, monkeypatch): + monkeypatch.setenv("VOICEFORGE_HOME", str(tmp_path / "prefix")) + monkeypatch.setenv("PYTHONPATH", "/host/site-packages") + setup.tool_env() + assert os.environ["PYTHONPATH"] == "/host/site-packages" + + +def test_ensure_uv_bootstraps_into_data_dir_with_contained_pip_cache(tmp_path, monkeypatch): + monkeypatch.setenv("VOICEFORGE_HOME", str(tmp_path / "prefix")) + monkeypatch.setattr(setup.shutil, "which", lambda name: None) + calls = [] + uv_binary = tmp_path / "prefix/bootstrap/bin/uv" + + def fake_run(args, stage, progress=None, *, env=None): + calls.append((args, env)) + # Simulate pip actually producing the pinned uv. + if stage == "Installing uv from PyPI": + uv_binary.parent.mkdir(parents=True, exist_ok=True) + uv_binary.write_text("#!/bin/sh\n") + uv_binary.chmod(0o755) + + monkeypatch.setattr(setup, "run_process", fake_run) + uv = setup.ensure_uv() + assert uv == str(uv_binary) + venv_args, venv_env = calls[0] + pip_args, pip_env = calls[1] + assert venv_args[-1] == str(tmp_path / "prefix/bootstrap") + assert pip_env["PIP_CACHE_DIR"] == str(tmp_path / "prefix/cache/pip") + assert pip_env["UV_CACHE_DIR"] == str(tmp_path / "prefix/cache/uv") + assert pip_env["PIP_CONFIG_FILE"] == os.devnull + assert "PYTHONPATH" not in pip_env + assert "uv==0.8.17" in pip_args + + +def test_ensure_uv_reuses_contained_bootstrap(tmp_path, monkeypatch): + monkeypatch.setenv("VOICEFORGE_HOME", str(tmp_path / "prefix")) + uv_binary = tmp_path / "prefix/bootstrap/bin/uv" + uv_binary.parent.mkdir(parents=True) + uv_binary.write_text("#!/bin/sh\n") + uv_binary.chmod(0o755) + monkeypatch.setattr(setup.shutil, "which", lambda name: None) + assert setup.ensure_uv() == str(uv_binary) + + +def test_ensure_uv_prefers_the_contained_bootstrap_over_a_host_uv(tmp_path, monkeypatch): + monkeypatch.setenv("VOICEFORGE_HOME", str(tmp_path / "prefix")) + uv_binary = tmp_path / "prefix/bootstrap/bin/uv" + uv_binary.parent.mkdir(parents=True) + uv_binary.write_text("#!/bin/sh\n") + uv_binary.chmod(0o755) + monkeypatch.setattr(setup.shutil, "which", lambda name: "/host/bin/uv") + assert setup.ensure_uv() == str(uv_binary) + + +def test_ensure_uv_ignores_a_host_uv_without_required_capability(tmp_path, monkeypatch): + monkeypatch.setenv("VOICEFORGE_HOME", str(tmp_path / "prefix")) + monkeypatch.setattr(setup.shutil, "which", lambda name: "/host/bin/uv") + monkeypatch.setattr(setup, "_uv_supports_relocatable", lambda uv: False) + stages = [] + uv_binary = tmp_path / "prefix/bootstrap/bin/uv" + + def fake_run(args, stage, progress=None, *, env=None): + stages.append(stage) + if stage == "Installing uv from PyPI": + uv_binary.parent.mkdir(parents=True, exist_ok=True) + uv_binary.write_text("#!/bin/sh\n") + uv_binary.chmod(0o755) + + monkeypatch.setattr(setup, "run_process", fake_run) + assert setup.ensure_uv() == str(uv_binary) + assert stages == ["Creating uv bootstrap", "Installing uv from PyPI"] + + +def test_ensure_uv_fails_when_bootstrap_does_not_produce_uv(tmp_path, monkeypatch): + monkeypatch.setenv("VOICEFORGE_HOME", str(tmp_path / "prefix")) + monkeypatch.setattr(setup.shutil, "which", lambda name: None) + monkeypatch.setattr(setup, "run_process", lambda *args, **kwargs: None) + with pytest.raises(RuntimeError, match="did not produce a usable uv"): + setup.ensure_uv() + + +# --- FFmpeg selection --------------------------------------------------------- + +def test_ensure_ffmpeg_prefers_the_bundled_binary_over_host(tmp_path, monkeypatch): + bundled = tmp_path / "bundled/ffmpeg" + bundled.parent.mkdir(parents=True) + bundled.write_text("#!/bin/sh\n") + monkeypatch.setitem(sys.modules, "imageio_ffmpeg", + types.SimpleNamespace(get_ffmpeg_exe=lambda: str(bundled))) + monkeypatch.setattr(setup.shutil, "which", lambda name: "/usr/bin/ffmpeg") + assert setup.ensure_ffmpeg() == str(bundled) + + +def test_ensure_ffmpeg_falls_back_to_system_without_the_bundle(monkeypatch): + monkeypatch.setitem(sys.modules, "imageio_ffmpeg", None) + monkeypatch.setattr(setup.shutil, "which", lambda name: "/usr/bin/ffmpeg") + assert setup.ensure_ffmpeg() == "/usr/bin/ffmpeg" + + +def test_ensure_ffmpeg_error_points_at_the_launcher(monkeypatch): + monkeypatch.setitem(sys.modules, "imageio_ffmpeg", None) + monkeypatch.setattr(setup.shutil, "which", lambda name: None) + with pytest.raises(RuntimeError, match=r"\./producer\.sh --rebuild") as error: + setup.ensure_ffmpeg() + assert "install.sh" not in str(error.value) + + +# --- producer.sh launcher ---------------------------------------------------- + +@pytest.mark.parametrize("argv", [["--help"], ["--rebuild", "--help"]]) +def test_launcher_forwards_help_without_bootstrapping(argv, shell_env, tmp_path, monkeypatch): + # A missing lib/ must not be created by a pure help request. + staging = tmp_path / "checkout" + staging.mkdir() + (staging / "producer.sh").write_text(LAUNCHER.read_text()) + (staging / "producer.sh").chmod(0o755) + monkeypatch.setenv("PATH", "/usr/bin:/bin") + result = subprocess.run(["./producer.sh", *argv], cwd=staging, env=shell_env, + capture_output=True, text=True, timeout=30) + assert result.returncode == 0 + assert "process" in result.stdout + assert not (staging / "lib").exists() + + +def test_launcher_script_parses(): + result = subprocess.run(["bash", "-n", str(LAUNCHER)], + capture_output=True, text=True, timeout=30) + assert result.returncode == 0, result.stderr diff --git a/lib/project/tests/test_pipeline.py b/lib/project/tests/test_pipeline.py new file mode 100644 index 0000000..0fa1c52 --- /dev/null +++ b/lib/project/tests/test_pipeline.py @@ -0,0 +1,273 @@ +import json + +import numpy as np +import pytest +import soundfile as sf + +from voiceforge import pipeline +from voiceforge.audio import loudness, measure +from voiceforge.config import resolve + + +@pytest.mark.parametrize("kind", ["missing", "input-extension", "output-extension"]) +def test_invalid_paths_fail_before_tool_setup(wav, tmp_path, bypass, progress, monkeypatch, kind): + source = wav(name="input.txt" if kind == "input-extension" else "input.wav") + if kind == "missing": + source = tmp_path / "missing.wav" + target = tmp_path / ("master.mp3" if kind == "output-extension" else "master.wav") + monkeypatch.setattr(pipeline, "ensure_ffmpeg", lambda: pytest.fail("Invalid paths must fail preflight")) + with pytest.raises(ValueError, match="WAV file|.wav extension"): + pipeline.process(source, target, bypass, progress) + assert not target.exists() + + +@pytest.mark.parametrize("alias", ["same", "symlink", "hardlink"]) +@pytest.mark.parametrize("overwrite", [False, True]) +def test_original_is_never_overwritten(wav, tmp_path, bypass, progress, monkeypatch, alias, overwrite): + source = wav() + original = source.read_bytes() + target = source if alias == "same" else tmp_path / "alias.wav" + if alias == "symlink": + target.symlink_to(source) + elif alias == "hardlink": + target.hardlink_to(source) + monkeypatch.setattr(pipeline, "ensure_ffmpeg", lambda: pytest.fail("Preflight must reject the input")) + with pytest.raises(ValueError, match="original recording"): + pipeline.process(source, target, bypass, progress, overwrite=overwrite) + assert source.read_bytes() == original + + +@pytest.mark.parametrize("suffix", [".wav", ".report.json", ".mp3"]) +def test_existing_output_or_sidecar_rejected_before_processing(wav, tmp_path, bypass, progress, monkeypatch, suffix): + source = wav() + target = tmp_path / "master.wav" + existing = target.with_suffix(suffix) + existing.write_bytes(b"existing output") + bypass.mp3 = True + monkeypatch.setattr(pipeline, "ensure_ffmpeg", lambda: pytest.fail("Preflight must reject existing outputs")) + with pytest.raises(FileExistsError, match="Output exists"): + pipeline.process(source, target, bypass, progress) + assert existing.read_bytes() == b"existing output" + + +@pytest.mark.parametrize("kind,message", [ + ("empty", "empty"), ("silence", "silent or too quiet"), + ("quiet", "silent or too quiet"), ("surround", "Only mono or stereo"), + ("nonfinite", "non-finite"), ("stereo", "select --channel"), + ("mono-right", "right channel of a mono"), +]) +def test_unusable_inputs_leave_no_outputs(wav, tmp_path, bypass, progress, ffmpeg, kind, message): + samples = { + "empty": np.empty(0), "silence": np.zeros(48000), + "quiet": np.full(48000, 1e-6), "surround": np.ones((4800, 3)) * 0.1, + "nonfinite": np.array([0.1, np.nan]), "stereo": np.ones((4800, 2)) * 0.1, + "mono-right": np.ones(4800) * 0.1, + }[kind] + source = wav(samples) + target = tmp_path / "outputs" / "master.wav" + if kind == "mono-right": + bypass.channel = "right" + with pytest.raises(ValueError, match=message): + pipeline.process(source, target, bypass, progress) + assert not target.parent.exists() + + +def test_measure_reports_channels_peaks_clipping_and_progress(wav): + samples = np.tile([1.0, -0.25], (4800, 1)) + events = [] + result = measure(wav(samples), lambda *event: events.append(event)) + assert result["duration_seconds"] == 0.1 + assert result["channels"] == 2 + assert result["sample_peak_dbfs"] == 0 + assert result["channel_rms_dbfs"] == pytest.approx([0, 20 * np.log10(0.25)]) + assert result["dc_offset"] == pytest.approx([1, -0.25]) + assert result["near_full_scale_samples"] == 4800 + assert events[0][1:] == (0, 4800) + assert events[-1][1:] == (4800, 4800) + + +@pytest.mark.parametrize("channel", ["left", "right", "mix"]) +def test_explicit_stereo_selection_preserves_signal_and_duration(wav, tmp_path, bypass, progress, ffmpeg, channel): + time = np.arange(48000 * 3) / 48000 + left = 0.2 * np.sin(2 * np.pi * 440 * time) + right = 0.1 * np.sin(2 * np.pi * 880 * time) + source = wav(np.column_stack([left, right])) + bypass.channel = channel + target = tmp_path / "master.wav" + report = pipeline.process(source, target, bypass, progress) + actual, rate = sf.read(target) + expected = {"left": left, "right": right, "mix": (left + right) / 2}[channel] + assert actual.ndim == 1 + assert rate == 48000 + assert len(actual) == len(expected) + np.testing.assert_allclose(actual, expected, atol=5e-7, rtol=0) + assert report["normalization_mode"] == "disabled" + assert report["level_correction_db"] == 0 + assert sf.info(target).subtype == "PCM_24" + + +@pytest.mark.parametrize("rate", [44100, 48000]) +def test_normalization_peak_ceiling_and_report(wav, tmp_path, progress, ffmpeg, rate): + source = wav() + original = source.read_bytes() + target = tmp_path / "nested" / "master.wav" + settings = resolve(overrides={"denoiser": "none", "sample_rate": rate}) + report = pipeline.process(source, target, settings, progress) + stats = sf.info(target) + assert stats.channels == 1 + assert stats.samplerate == rate + assert stats.subtype == "PCM_24" + assert abs(stats.duration - 3) <= 0.02 + verified = loudness(ffmpeg, target, settings, stats.duration, progress) + assert verified["input_i"] == pytest.approx(settings.target_lufs, abs=0.5) + assert verified["input_tp"] <= settings.true_peak_db + 0.1 + assert report["master"]["near_full_scale_samples"] == 0 + assert report["master"]["sample_peak_dbfs"] <= settings.true_peak_db + 0.1 + assert json.loads(target.with_suffix(".report.json").read_text()) == report + assert source.read_bytes() == original + assert not list(target.parent.glob(".voiceforge-*")) + + +def test_limiter_without_normalization(wav, tmp_path, bypass, progress, ffmpeg): + time = np.arange(48000 * 3) / 48000 + source = wav(0.99 * np.sin(2 * np.pi * 440 * time)) + bypass.limiter = True + bypass.true_peak_db = -6 + report = pipeline.process(source, tmp_path / "master.wav", bypass, progress) + assert report["normalization_mode"] == "disabled" + assert report["master_loudness"]["input_tp"] <= -5.9 + assert report["master"]["sample_peak_dbfs"] > -7 + + +def test_speech_estimate_survives_pause_dominated_recordings(wav): + time = np.arange(48000 * 3) / 48000 + audio = np.concatenate([np.zeros(48000 * 27), 0.02 * np.sin(2 * np.pi * 440 * time)]) + stats = measure(wav(audio), lambda *_: None) + assert stats["quiet_blocks_dbfs"] <= -110 + assert stats["speech_level_estimate_dbfs"] == pytest.approx(-37, abs=2) + + +def test_leveling_uses_active_speech_not_pauses(wav, tmp_path, progress, ffmpeg): + time = np.arange(48000 * 3) / 48000 + audio = np.concatenate([np.zeros(48000 * 27), 0.02 * np.sin(2 * np.pi * 440 * time)]) + source = wav(audio) + settings = resolve(profile="cleanup-only", overrides={"denoiser": "none", "leveling": True}) + report = pipeline.process(source, tmp_path / "master.wav", settings, progress) + assert 10 < report["level_correction_db"] < settings.max_gain_db + assert not any("gain limit" in warning for warning in report["warnings"]) + + +def test_leveling_skipped_when_speech_estimate_is_unreliable(wav, tmp_path, bypass, progress, ffmpeg, monkeypatch): + real_measure = pipeline.measure + + def measured(path, callback, stage="Analyzing audio"): + stats = real_measure(path, callback, stage) + if stage == "Measuring cleaned voice": + stats["speech_level_estimate_dbfs"] = -120 + return stats + + monkeypatch.setattr(pipeline, "measure", measured) + bypass.leveling = True + report = pipeline.process(wav(), tmp_path / "master.wav", bypass, progress) + assert report["level_correction_db"] == 0 + assert any("leveling skipped" in warning for warning in report["warnings"]) + + +def test_antiphase_mix_is_rejected_after_selection(wav, tmp_path, bypass, progress, ffmpeg): + time = np.arange(48000 * 3) / 48000 + tone = 0.2 * np.sin(2 * np.pi * 440 * time) + source = wav(np.column_stack([tone, -tone])) + bypass.channel = "mix" + target = tmp_path / "master.wav" + with pytest.raises(ValueError, match="selected audio is silent or too quiet"): + pipeline.process(source, target, bypass, progress) + assert not target.exists() + assert not target.with_suffix(".report.json").exists() + assert not list(tmp_path.glob(".voiceforge-*")) + + +def test_silent_selected_channel_is_rejected_after_selection(wav, tmp_path, bypass, progress, ffmpeg): + time = np.arange(48000 * 3) / 48000 + tone = 0.2 * np.sin(2 * np.pi * 440 * time) + source = wav(np.column_stack([tone, np.zeros_like(tone)])) + bypass.channel = "right" + target = tmp_path / "master.wav" + with pytest.raises(ValueError, match="selected audio is silent or too quiet"): + pipeline.process(source, target, bypass, progress) + assert not target.exists() + + +def test_zero_denoise_strength_skips_ai(wav, tmp_path, bypass, progress, ffmpeg, monkeypatch): + def unexpected(*args, **kwargs): + pytest.fail("Zero denoise strength must not invoke AI") + + monkeypatch.setattr("voiceforge.ai.denoise", unexpected) + bypass.denoiser = "deepfilter" + bypass.denoise_strength = 0 + report = pipeline.process(wav(), tmp_path / "master.wav", bypass, progress) + assert report["cleaned"]["frames"] == report["input"]["frames"] + + +def test_fft_and_mp3_export_can_replace_generated_outputs(wav, tmp_path, bypass, progress, ffmpeg): + source = wav() + original = source.read_bytes() + bypass.denoiser = "fft" + bypass.mp3 = True + target = tmp_path / "master.wav" + for suffix in (".wav", ".report.json", ".mp3"): + target.with_suffix(suffix).write_bytes(b"old output") + events = [] + report = pipeline.process(source, target, bypass, lambda *event: events.append(event), overwrite=True) + assert any(event[0] == "FFT noise reduction" for event in events) + assert target.with_suffix(".mp3").stat().st_size > 1000 + assert report["mp3_loudness"]["input_i"] is not None + assert json.loads(target.with_suffix(".report.json").read_text()) == report + assert sf.info(target).channels == 1 + assert source.read_bytes() == original + + +def test_render_failure_preserves_existing_outputs_and_removes_temporary_files(wav, tmp_path, bypass, progress, ffmpeg, monkeypatch): + source = wav() + target = tmp_path / "master.wav" + bypass.mp3 = True + outputs = [target.with_suffix(suffix) for suffix in (".wav", ".report.json", ".mp3")] + for path in outputs: + path.write_bytes(b"previous publication") + real_run = pipeline.ffmpeg_run + + def fail_encoding(executable, args, stage, duration, callback): + if stage == "Encoding MP3": + raise RuntimeError("injected encoder failure") + return real_run(executable, args, stage, duration, callback) + + monkeypatch.setattr(pipeline, "ffmpeg_run", fail_encoding) + with pytest.raises(RuntimeError, match="injected encoder failure"): + pipeline.process(source, target, bypass, progress, overwrite=True) + assert all(path.read_bytes() == b"previous publication" for path in outputs) + assert not list(tmp_path.glob(".voiceforge-*")) + + +@pytest.mark.parametrize("overwrite", [False, True]) +def test_publish_collision_respects_overwrite(tmp_path, overwrite): + staged = tmp_path / "staged" + target = tmp_path / "target" + staged.write_bytes(b"new") + target.write_bytes(b"concurrent output") + if overwrite: + pipeline.publish(staged, target, overwrite=True) + assert target.read_bytes() == b"new" + assert not staged.exists() + else: + with pytest.raises(FileExistsError): + pipeline.publish(staged, target, overwrite=False) + assert target.read_bytes() == b"concurrent output" + assert staged.read_bytes() == b"new" + + +def test_publish_new_output_moves_staged_file(tmp_path): + staged = tmp_path / "staged" + target = tmp_path / "target" + staged.write_bytes(b"finished") + pipeline.publish(staged, target, overwrite=False) + assert target.read_bytes() == b"finished" + assert not staged.exists() |
