From 1bbe59f78be9a8355bb0bee2bc8ab14ca42ac71c Mon Sep 17 00:00:00 2001 From: historia Date: Fri, 11 Sep 2026 22:03:43 -0400 Subject: Add ClearVoice MossFormer2 denoiser backend and static gap trimming --- README.md | 29 ++++-- config.toml | 13 ++- lib/project/src/producer/ai.py | 160 +++++++++++++++++++++++++++------- lib/project/src/producer/audio.py | 81 ++++++++++++++++- lib/project/src/producer/cli.py | 23 +++-- lib/project/src/producer/config.py | 9 +- lib/project/src/producer/cv_worker.py | 142 ++++++++++++++++++++++++++++++ lib/project/src/producer/pipeline.py | 44 +++++++--- lib/project/tests/test_config.py | 14 +++ lib/project/tests/test_pipeline.py | 59 ++++++++++++- producer.sh | 8 +- 11 files changed, 517 insertions(+), 65 deletions(-) create mode 100644 lib/project/src/producer/cv_worker.py diff --git a/README.md b/README.md index 63f5944..1d43285 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ constant bitrate at 192 kb/s by default. Loudness is normalized to **-19 LUFS with a -1.5 dBTP true-peak ceiling**. ### Examples - +``` # Optional MP3 alongside the WAV ./producer.sh recording.wav --mp3 @@ -47,18 +47,21 @@ LUFS with a -1.5 dBTP true-peak ceiling**. ./producer.sh recording.wav --denoiser fft ./producer.sh recording.wav --no-denoise +# Use the ClearVoice MossFormer2_SE_48K denoiser instead of DeepFilterNet3 +# (larger download; preserves word attacks noticeably better at high strength) +./producer.sh recording.wav --denoiser clearvoice +./producer.sh setup --denoiser clearvoice --device cpu + +# Shorten long gaps between takes (speech is never attenuated) +./producer.sh recording.wav --trim-silence --max-gap-s 1.5 --gap-pad-s 0.6 + # LibriVox-style delivery file: 44.1 kHz, 128 kbps CBR, -19 LUFS, mono ./producer.sh recording.wav \ --sample-rate 44100 --mp3 --mp3-bitrate 128 \ --target-lufs -19 --true-peak-db -1.5 ``` -All of these settings are configurable; run `./producer.sh config` to see every -setting and its default, or `--help` on any command for the flags. A -`config.toml` in the same directory as `producer.sh` is loaded automatically: -`./producer.sh config` then shows those resolved values, and an explicit -`--config FILE` or any CLI flag still overrides it. The shipped `config.toml` -documents every setting. +Edit default settings in `config.toml` ## Profiles @@ -69,6 +72,14 @@ documents every setting. | `radio` | Stronger compression, more warmth/presence, tighter range | | `cleanup-only` | Denoise + highpass only; no EQ, dynamics or normalization | +AI denoisers soften word attacks to some degree; `clearvoice` +(MossFormer2_SE_48K) was measured to leave onsets essentially untouched, +while `deepfilter` ducks them a couple of percent even at reduced strength. +Denoisers also never reach true digital silence: a noise floor remains in +blank sections. `--trim-silence` addresses both properly - long gaps are +shortened to `--gap-pad-s` of untouched room tone with a sample-exact cut, +so speech is never gated or attenuated. + Test the difference on your own audio before committing: ```bash @@ -83,6 +94,10 @@ Test the difference on your own audio before committing: driver, then retry `./producer.sh setup --device cuda`. Or use `--device cpu`. - **Sounds watery or thin:** lower the denoise strength (`--denoise-strength 0.65`) or compare profiles with `preview`. +- **Word attacks sound soft or ramp up:** AI denoisers can soften consonant + onsets. Try `--denoiser clearvoice` (larger non-causal model that sees more + attack context), lower `--denoise-strength`, or compare against + `--no-denoise` to locate the culprit. ## License diff --git a/config.toml b/config.toml index a574717..75e0e3b 100644 --- a/config.toml +++ b/config.toml @@ -5,9 +5,9 @@ # with --profile radio. Remove settings you want a different profile to supply. profile = "narrator" # natural, narrator, radio, cleanup-only -denoiser = "deepfilter" # DeepFilterNet3 only; fft is explicit non-AI fallback; none +denoiser = "clearvoice" # clearvoice (MossFormer2_SE_48K, best attack preservation), deepfilter; fft is explicit non-AI fallback; none device = "auto" # auto, cpu, cuda; explicit cuda fails if unavailable -denoise_strength = 0.85 # 0..1; AI dry/wet blend; zero skips denoising and AI setup +denoise_strength = 1.0 # 0..1; AI dry/wet blend; zero skips denoising and AI setup fft_reduction_db = 8.0 # 0.01..30; FFT reduction is scaled by denoise_strength highpass = true @@ -26,6 +26,15 @@ expansion_threshold_db = -50.0 # -90..-20 expansion_ratio = 1.5 # 1..4 expansion_range_db = 12.0 # 0..40 +# Static gap trimming: gaps longer than max_gap_s are shortened to gap_pad_s +# (total silence kept, split at the gap edges). Speech is copied through +# unattenuated - no gate, so word attacks are never touched. Use this instead +# of relying on the denoiser to erase room tone in blank sections. +trim_silence = true +max_gap_s = 1.5 # 0.5..10; gaps longer than this are shortened +gap_pad_s = 0.6 # 0.1..3; total silence kept around each trimmed gap (< max_gap_s) +gap_threshold_db = -50.0 # -90..-20; blocks below this count as silence + eq = true warmth_db = 0.5 # -12..12; 140 Hz mud_db = -1.5 # -12..12; 300 Hz diff --git a/lib/project/src/producer/ai.py b/lib/project/src/producer/ai.py index 7598222..76d38a6 100644 --- a/lib/project/src/producer/ai.py +++ b/lib/project/src/producer/ai.py @@ -1,4 +1,8 @@ -"""Automatic, isolated DeepFilterNet3 inference (Linux, Python 3.11 worker). +"""Automatic, isolated AI denoising (Linux, Python 3.11 workers). + +Two interchangeable backends run in separate environments: +DeepFilterNet3 (default, release-pinned weights) and ClearVoice's +MossFormer2_SE_48K (weights fetched from Hugging Face). Public progress callback: (stage: str, completed: float | None, total: float | None). Known totals are audio frames or download bytes (identified by stage); @@ -8,6 +12,7 @@ Producer data directory (PRODUCER_HOME prefix or the XDG data location). """ from __future__ import annotations +from contextlib import contextmanager import ctypes import fcntl import math @@ -19,6 +24,9 @@ import time from .setup import Progress, data_dir, ensure_uv, run_process, tool_env +CLEARVOICE_VERSION = "0.1.2" +CLEARVOICE_MODEL = "MossFormer2_SE_48K" + def _cuda_available() -> bool: # Probe the driver without importing Torch or installing CUDA in the host env. @@ -32,6 +40,19 @@ def _cuda_available() -> bool: return False +def _require_supported_platform(what: str) -> None: + try: + libc = os.confstr("CS_GNU_LIBC_VERSION") or "" + except (ValueError, OSError): + libc = "" + if platform.system() != "Linux" or not libc.startswith("glibc ") or tuple( + int(part) for part in libc.split()[1].split(".")[:2] + ) < (2, 28): + raise RuntimeError(f"The pinned {what} backend requires Linux with glibc >= 2.28 " + "(for example Ubuntu 22.04+ or Debian 12+). " + "Alpine/musl is not supported by its prebuilt wheels.") + + def _flavor(device: str) -> str: if device not in {"auto", "cpu", "cuda"}: raise ValueError("device must be 'auto', 'cpu', or 'cuda'") @@ -61,9 +82,24 @@ def _interpreter_is_local(python: Path, root: Path) -> bool: return False +@contextmanager +def _boot_lock(root: Path, progress: Progress | None, started: float): + """Serialize bootstrap/model downloads, including calls from separate CLIs.""" + with (root / "ai.lock").open("a") as lock: + while True: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + if progress: + progress("Waiting for AI setup", time.monotonic() - started, None) + time.sleep(0.25) + yield + + def ensure_ai(device: str = "auto", progress: Progress | None = None, verify: bool = False) -> Path: - """Install/validate the backend and model; return its Python executable. + """Install/validate the DeepFilterNet3 backend and model; return its Python. CPU and CUDA environments are separate. Explicit CUDA never falls back; auto uses Torch's availability check in the selected worker. Downloads trust @@ -73,32 +109,14 @@ def ensure_ai(device: str = "auto", progress: Progress | None = None, environment does not pay for a second model load per run. """ flavor = _flavor(device) - try: - libc = os.confstr("CS_GNU_LIBC_VERSION") or "" - except (ValueError, OSError): - libc = "" - if platform.system() != "Linux" or not libc.startswith("glibc ") or tuple( - int(part) for part in libc.split()[1].split(".")[:2] - ) < (2, 28): - raise RuntimeError("The pinned DeepFilterNet3 backend requires Linux with glibc >= 2.28 " - "(for example Ubuntu 22.04+ or Debian 12+). " - "Alpine/musl is not supported by its prebuilt wheels.") + _require_supported_platform("DeepFilterNet3") root = data_dir() root.mkdir(parents=True, exist_ok=True) environment = root / f"ai-df-0.5.6-torch-2.5.1-{flavor}-py311-v1" python = environment / "bin/python" marker = environment / ".producer-ready" started = time.monotonic() - # Serialize bootstrap/model downloads, including calls from separate CLIs. - with (root / "ai.lock").open("a") as lock: - while True: - try: - fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) - break - except BlockingIOError: - if progress: - progress("Waiting for AI setup", time.monotonic() - started, None) - time.sleep(0.25) + with _boot_lock(root, progress, started): ready = (marker.is_file() and python.is_file() and _interpreter_is_local(python, root)) if not ready: @@ -133,13 +151,81 @@ def ensure_ai(device: str = "auto", progress: Progress | None = None, return python +def ensure_cv(device: str = "auto", progress: Progress | None = None, + verify: bool = False) -> Path: + """Install/validate the ClearVoice MossFormer2_SE_48K backend. + + Mirrors ensure_ai() (separate CPU/CUDA environments, ready marker, real + self-test that also fetches the weights) but installs clearvoice and its + pinned scientific stack from PyPI. Weight downloads happen inside the + worker and trust Hugging Face; they land under /models/checkpoints. + """ + flavor = _flavor(device) + _require_supported_platform("ClearVoice") + root = data_dir() + root.mkdir(parents=True, exist_ok=True) + environment = root / f"ai-cv-{CLEARVOICE_VERSION}-torch-2.5.1-{flavor}-py311-v1" + python = environment / "bin/python" + marker = environment / ".producer-ready" + started = time.monotonic() + with _boot_lock(root, progress, started): + ready = (marker.is_file() and python.is_file() + and _interpreter_is_local(python, root)) + if not ready: + # Same policy as ensure_ai(): a moved, copied, or half-installed + # environment is rebuilt from the shared wheel caches. + if environment.exists(): + shutil.rmtree(environment) + uv = ensure_uv(progress) + env = _worker_env() + run_process([uv, "venv", "--python", "3.11", "--managed-python", + "--relocatable", "--allow-existing", str(environment)], + "Preparing AI Python 3.11", progress, env=env) + run_process([uv, "pip", "install", "--python", str(python), + "--index-url", f"https://download.pytorch.org/whl/{flavor}", + "torch==2.5.1", "torchaudio==2.5.1", "torchvision==0.20.1"], + f"Installing PyTorch ({flavor})", progress, env=env) + run_process([uv, "pip", "install", "--python", str(python), + "--only-binary", ":all:", + "--index-url", "https://pypi.org/simple", "numpy==1.26.4"], + "Installing ClearVoice", progress, env=env) + # python_speech_features publishes no wheel on PyPI; it is the one + # deliberately built from source, isolated in its own install step. + run_process([uv, "pip", "install", "--python", str(python), + "--index-url", "https://pypi.org/simple", + "python_speech_features==0.6"], + "Installing ClearVoice", progress, env=env) + # Pinned torch stack keeps uv's resolver on the PyTorch index + # instead of upgrading to an unrelated CUDA build from PyPI. + run_process([uv, "pip", "install", "--python", str(python), + "--no-build", + "--index-url", "https://pypi.org/simple", + "--extra-index-url", f"https://download.pytorch.org/whl/{flavor}", + f"torch==2.5.1+{flavor}", f"torchaudio==2.5.1+{flavor}", + f"torchvision==0.20.1+{flavor}", + f"clearvoice=={CLEARVOICE_VERSION}"], + "Installing ClearVoice", progress, env=env) + # Check a freshly built environment (and any explicit setup) with a + # real model load and inference; this also downloads the weights. + if verify or not ready: + env = _worker_env() + env["HF_HOME"] = str(root / "cache/huggingface") + run_process([str(python), "-I", str(Path(__file__).with_name("cv_worker.py")), + "--device", device, "--model-dir", str(root / "models"), + "--check"], f"Checking {CLEARVOICE_MODEL}", progress, env=env) + marker.touch() + return python + + def denoise(source: Path, target: Path, device: str = "auto", strength: float = 1.0, - progress: Progress | None = None) -> None: + progress: Progress | None = None, backend: str = "deepfilter") -> None: """Denoise mono 48 kHz WAV to float WAV, preserving the exact sample count. - strength is a dry/wet mix in [0, 1]. The worker publishes output atomically; - errors/cancellation leave an existing target untouched. Chunked inference is - bounded-memory, not bit-identical to inference over an entire recording. + strength is a dry/wet mix in [0, 1]. backend selects the model: + 'deepfilter' (DeepFilterNet3) or 'clearvoice' (MossFormer2_SE_48K). The + worker publishes output atomically; errors/cancellation leave an existing + target untouched. Chunked inference is bounded-memory, not bit-identical + to inference over an entire recording. """ if not math.isfinite(strength) or not 0 <= strength <= 1: raise ValueError("strength must be finite and between 0 and 1") @@ -150,8 +236,20 @@ def denoise(source: Path, target: Path, device: str = "auto", strength: float = raise FileNotFoundError(source) if not target.parent.is_dir(): raise FileNotFoundError(target.parent) - python = ensure_ai(device, progress) - run_process([str(python), "-I", str(Path(__file__).with_name("ai_worker.py")), - "--device", device, "--source", str(source), "--target", str(target), - "--model-dir", str(data_dir() / "models/df3-0.5.6"), - "--strength", str(strength)], "Denoising", progress, env=_worker_env()) + if backend == "deepfilter": + python = ensure_ai(device, progress) + args = ["--device", device, "--source", str(source), "--target", str(target), + "--model-dir", str(data_dir() / "models/df3-0.5.6"), + "--strength", str(strength)] + worker = "ai_worker.py" + elif backend == "clearvoice": + python = ensure_cv(device, progress) + args = ["--device", device, "--source", str(source), "--target", str(target), + "--model-dir", str(data_dir() / "models"), + "--strength", str(strength)] + worker = "cv_worker.py" + else: + raise ValueError(f"Unknown denoiser backend {backend!r}; " + "choose 'deepfilter' or 'clearvoice'") + run_process([str(python), "-I", str(Path(__file__).with_name(worker)), *args], + "Denoising", progress, env=_worker_env()) diff --git a/lib/project/src/producer/audio.py b/lib/project/src/producer/audio.py index 7fee0c2..59fe1ff 100644 --- a/lib/project/src/producer/audio.py +++ b/lib/project/src/producer/audio.py @@ -1,4 +1,4 @@ -"""Streaming measurement and FFmpeg execution. No shell interpolation.""" +"""Streaming measurement, gap trimming and FFmpeg execution. No shell interpolation.""" from collections import deque import json import math @@ -7,6 +7,7 @@ from pathlib import Path import selectors import signal import subprocess +import tempfile import time import numpy as np @@ -115,6 +116,84 @@ def measure(path: Path, progress, stage="Analyzing audio") -> dict: "noise_floor_confidence": "low (quiet-block heuristic, not speech recognition)" if noise_available else "insufficient room tone"} +def trim_gaps(source: Path, target: Path, threshold_db: float, max_gap_s: float, + pad_s: float, progress, + stage="Trimming long silences") -> tuple[int, float]: + """Shorten silences longer than max_gap_s, keeping pad_s of untouched + room tone (split evenly at the gap edges) and dropping the middle. + + Static, sample-exact trimming: speech is copied through unattenuated and + never dynamically gated. Mono 48 kHz input required. Returns the number + of trimmed gaps and the seconds removed. Streaming and bounded in memory. + """ + sr, block = 48000, 2400 # 50 ms RMS blocks + with sf.SoundFile(source) as src: + if src.samplerate != sr or src.channels != 1: + raise ValueError("Gap trimming requires mono 48 kHz audio") + frames = len(src) + total_blocks = (frames + block - 1) // block + quiet = np.zeros(total_blocks, dtype=bool) + done = 0 + progress(stage, 0, frames) + while True: + audio = src.read(block * 32, dtype="float32") + if not len(audio): + break + if not np.isfinite(audio).all(): + raise ValueError("Input contains non-finite samples") + count = len(audio) // block + rms = np.sqrt((audio[:count * block].reshape(count, block) ** 2).mean(axis=1)) + quiet[done:done + count] = (20 * np.log10(np.maximum(rms, 1e-12)) < threshold_db) + done += count + progress(stage, min(done * block, frames), frames) + # Collect quiet runs, then walk the timeline keeping the pad edges. + keep: list[tuple[int, int]] = [] + cursor, trimmed, removed = 0, 0, 0 + run_start = None + for index in range(total_blocks + 1): + at_gap = index < total_blocks and quiet[index] + if at_gap and run_start is None: + run_start = index + elif not at_gap and run_start is not None: + a, b = run_start * block, min(index * block, frames) + run_start = None + if b - a >= int(max_gap_s * sr): + half = min(int(pad_s * sr / 2), (b - a) // 2) + keep.append((cursor, a + half)) + removed += b - a - 2 * half + cursor = b - half + trimmed += 1 + keep.append((cursor, frames)) + # Pass 2: copy the kept ranges; output size drives the RF64 choice. + temporary = None + try: + fd, name = tempfile.mkstemp(prefix=".producer-trim-", suffix=".wav", + dir=target.parent) + os.close(fd) + temporary = Path(name) + output_format = "RF64" if frames * 4 > 0xFFFFFFFF - 4096 else "WAV" + written = 0 + with sf.SoundFile(temporary, "w", samplerate=sr, channels=1, + format=output_format, subtype="FLOAT") as dst: + for a, b in keep: + src.seek(a) + remaining = b - a + while remaining: + audio = src.read(min(block * 32, remaining), dtype="float32") + if not len(audio): + raise RuntimeError("Unexpected end of input while trimming") + dst.write(audio) + remaining -= len(audio) + written += len(audio) + progress(stage, min(written, frames), frames) + os.replace(temporary, target) + temporary = None + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + return trimmed, removed / sr + + def loudness(ffmpeg, source, settings, duration, progress, stage="Measuring loudness"): output = ffmpeg_run(ffmpeg, ["-i", str(source), "-map", "0:a:0", "-af", f"loudnorm=I={settings.target_lufs}:TP={settings.true_peak_db}:LRA={settings.loudness_range}:print_format=json", diff --git a/lib/project/src/producer/cli.py b/lib/project/src/producer/cli.py index 374455a..8b34e7c 100644 --- a/lib/project/src/producer/cli.py +++ b/lib/project/src/producer/cli.py @@ -46,7 +46,7 @@ def parser(): if name in ("process", "preview"): cmd.add_argument("--output-dir", type=Path, default=None, help="Directory for generated outputs (default: beside each input)") - cmd.add_argument("--overwrite", action="store_true", help="Replace generated outputs, never input files") + cmd.add_argument("--overwrite", action="store_true", help="Replace existing audio outputs, never input files (report logs are always replaced)") if name == "process": cmd.add_argument("-o", "--output", type=Path, help="Explicit WAV destination (one input only)") if name == "preview": @@ -56,6 +56,8 @@ def parser(): sub.add_parser("doctor", help="Check tools and driver without downloading AI") setup = sub.add_parser("setup", help="Download and self-test the isolated AI backend") setup.add_argument("--device", choices=("auto", "cpu", "cuda"), default="auto") + setup.add_argument("--denoiser", choices=("deepfilter", "clearvoice"), default="deepfilter", + help="Which backend to install and self-test") return app @@ -101,10 +103,13 @@ def main(): console.print(f"FFmpeg: {ffmpeg}\nData: {data_dir()}\nCUDA driver available: {_cuda_available()}\nAI is installed and self-tested by './producer.sh setup'.", markup=False) return if args.command == "setup": - from .ai import ensure_ai + from .ai import ensure_ai, ensure_cv with Display() as display: ensure_ffmpeg() - python = ensure_ai(args.device, display.update, verify=True) + if args.denoiser == "clearvoice": + python = ensure_cv(args.device, display.update, verify=True) + else: + python = ensure_ai(args.device, display.update, verify=True) console.print(f"AI ready: {python}", markup=False) return settings = settings_for(args) @@ -135,8 +140,11 @@ def main(): if resolved & sources or any( target.exists() and os.path.samefile(source, target) for target in targets): raise ValueError("An output would overwrite an input recording") + report = report_for(output) for target in targets: - check_target(source, target, args.overwrite) + # Report logs are replaced silently; only audio + # outputs require --overwrite. + check_target(source, target, args.overwrite or target == report) for source, output in zip(args.inputs, outputs): report = process(source, output, settings, display.update, args.overwrite) console.print(f"Written: {output} | {report['master_loudness']['input_i']} LUFS | {report['master_loudness']['input_tp']} dBTP", markup=False) @@ -159,13 +167,16 @@ def main(): output = output_dir / f"{source.stem}.{profile}.preview.wav" if output.resolve() == original: raise ValueError("Preview output would overwrite the original") - targets = [output, report_for(output)] + report = report_for(output) + targets = [output, report] resolved = {target.resolve() for target in targets} if len(resolved) < len(targets) or resolved & claimed: raise ValueError("Preview outputs have duplicate names") claimed |= resolved for target in targets: - check_target(original, target, args.overwrite) + # Report logs are replaced silently; only audio outputs + # require --overwrite. + check_target(original, target, args.overwrite or target == report) output_dir.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory(prefix=".preview-", dir=output_dir) as tmp: excerpt = Path(tmp) / "excerpt.wav" diff --git a/lib/project/src/producer/config.py b/lib/project/src/producer/config.py index b1d84cc..347504a 100644 --- a/lib/project/src/producer/config.py +++ b/lib/project/src/producer/config.py @@ -24,6 +24,10 @@ class Settings: expansion_threshold_db: float = -50.0 expansion_ratio: float = 1.5 expansion_range_db: float = 12.0 + trim_silence: bool = False + max_gap_s: float = 1.5 + gap_pad_s: float = 0.6 + gap_threshold_db: float = -50.0 eq: bool = True warmth_db: float = 0.5 mud_db: float = -1.5 @@ -64,6 +68,7 @@ RANGES = { "level_target_db": (-40, -12), "max_gain_db": (0, 30), "expansion_threshold_db": (-90, -20), "expansion_ratio": (1, 4), "expansion_range_db": (0, 40), "warmth_db": (-12, 12), + "max_gap_s": (0.5, 10), "gap_pad_s": (0.1, 3), "gap_threshold_db": (-90, -20), "mud_db": (-12, 12), "presence_db": (-12, 12), "compressor_threshold_db": (-50, -5), "compressor_ratio": (1, 10), "compressor_attack_ms": (0.1, 200), "compressor_release_ms": (10, 2000), @@ -95,7 +100,7 @@ def resolve(path: Path | None = None, profile: str | None = None, if key in RANGES and not RANGES[key][0] <= value <= RANGES[key][1]: raise ValueError(f"{key} must be between {RANGES[key][0]} and {RANGES[key][1]}") for key, options in { - "denoiser": ("deepfilter", "fft", "none"), "device": ("auto", "cpu", "cuda"), + "denoiser": ("deepfilter", "clearvoice", "fft", "none"), "device": ("auto", "cpu", "cuda"), "hum_hz": (0, 50, 60), "sample_rate": (44100, 48000), "mp3_bitrate": (128, 160, 192, 224, 256, 320), }.items(): @@ -103,6 +108,8 @@ def resolve(path: Path | None = None, profile: str | None = None, raise ValueError(f"{key} must be one of {options}") if merged["lowpass"] and merged["lowpass_hz"] >= merged["sample_rate"] / 2: raise ValueError("lowpass_hz must be below the output Nyquist frequency") + if merged["trim_silence"] and merged["gap_pad_s"] >= merged["max_gap_s"]: + raise ValueError("gap_pad_s must be smaller than max_gap_s") if merged["normalize"] and not merged["limiter"]: raise ValueError("Normalization includes true-peak limiting; use --no-normalize to bypass it") return Settings(**merged) diff --git a/lib/project/src/producer/cv_worker.py b/lib/project/src/producer/cv_worker.py new file mode 100644 index 0000000..268dfa9 --- /dev/null +++ b/lib/project/src/producer/cv_worker.py @@ -0,0 +1,142 @@ +"""Private MossFormer2_SE_48K (ClearVoice) inference, run by the isolated backend Python. + +The public contract matches ai_worker.py: identical CLI, mono 48 kHz float WAV +in and out with an unchanged sample count, atomic publication, and +PRODUCER_PROGRESS records on stdout. Weights are fetched by ClearVoice from +Hugging Face into /checkpoints on first use; the model directory is +also the working directory because ClearVoice resolves its relative checkpoint +path against the process cwd. +""" +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import signal +import tempfile + + +def report(stage: str, completed: float | None = None, total: float | None = None) -> None: + print("PRODUCER_PROGRESS " + json.dumps([stage, completed, total]), flush=True) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--device", choices=("auto", "cpu", "cuda"), required=True) + parser.add_argument("--check", action="store_true") + parser.add_argument("--model-dir", type=Path, required=True) + parser.add_argument("--source", type=Path) + parser.add_argument("--target", type=Path) + parser.add_argument("--strength", type=float, default=1.0) + args = parser.parse_args() + + # SIGTERM from the parent takes the same cleanup path as Ctrl-C. + def terminate(signum, frame): + raise KeyboardInterrupt + + signal.signal(signal.SIGTERM, terminate) + + # SpeechModel hides MPS and auto-selects CUDA whenever it is visible, so an + # explicit CPU run hides the driver before torch is imported. + if args.device == "cpu": + os.environ["CUDA_VISIBLE_DEVICES"] = "" + + import numpy as np + import soundfile as sf + import torch + + torch.set_num_threads(min(4, os.cpu_count() or 1)) + available = torch.cuda.is_available() + if args.device == "cuda" and not available: + raise RuntimeError("CUDA was explicitly requested, but PyTorch cannot use it. " + "Check the NVIDIA driver and CUDA_VISIBLE_DEVICES, or select cpu.") + device = "cuda" if args.device != "cpu" and available else "cpu" + + if not np.isfinite(args.strength) or not 0 <= args.strength <= 1: + parser.error("--strength must be in [0, 1]") + # Relative IO paths are resolved before chdir(model-dir) re-bases the cwd. + source = args.source.resolve() if args.source is not None else None + target = args.target.resolve() if args.target is not None else None + args.model_dir.mkdir(parents=True, exist_ok=True) + # ClearVoice resolves its relative 'checkpoints/...' configuration path + # against the process working directory; keep weights inside the data dir. + os.chdir(args.model_dir) + + report("Loading MossFormer2_SE_48K") + from clearvoice import ClearVoice + + voice = ClearVoice(task="speech_enhancement", model_names=["MossFormer2_SE_48K"]) + checkpoint = Path("checkpoints/MossFormer2_SE_48K/last_best_checkpoint") + if not checkpoint.is_file(): + raise RuntimeError("MossFormer2_SE_48K weights are missing; ClearVoice downloads " + "them from Hugging Face, so check network access and retry setup.") + + def enhance(audio: np.ndarray) -> np.ndarray: + """Mono float32 samples in, same-length float32 samples out.""" + wet = voice(np.ascontiguousarray(audio, dtype="float32")[None, :], False) + wet = np.asarray(wet)[0][:len(audio)] + if wet.shape[-1] != len(audio) or not np.isfinite(wet).all(): + raise RuntimeError("MossFormer2_SE_48K returned invalid audio") + return wet.astype("float32") + + if args.check: + # A tone, not silence: silent input can degenerate the model's own + # level bookkeeping and would not exercise speech anyway. + seconds = 48000 // 2 + tone = (0.1 * np.sin(2 * np.pi * 440 * np.arange(seconds) / 48000)).astype("float32") + if not np.isfinite(enhance(tone)).all(): + raise RuntimeError("MossFormer2_SE_48K self-test failed") + report(f"MossFormer2_SE_48K ready ({device})", 1, 1) + return + if source is None or target is None: + parser.error("--source and --target are required unless --check is used") + + temporary = None + try: + with sf.SoundFile(source) as opened: + if opened.samplerate != 48000 or opened.channels != 1 or opened.format not in {"WAV", "WAVEX", "RF64"}: + raise ValueError("AI input must be a mono 48 kHz WAV; convert it before denoising") + frames = len(opened) + fd, name = tempfile.mkstemp(prefix=".producer-ai-", suffix=".wav", dir=target.parent) + os.close(fd) + temporary = Path(name) + # RF64 avoids RIFF's 4 GiB limit on very long recordings. + output_format = "RF64" if frames * 4 > 0xFFFFFFFF - 4096 else "WAV" + with sf.SoundFile(temporary, "w", samplerate=48000, channels=1, + format=output_format, subtype="FLOAT") as destination: + # ClearVoice decodes each call in one pass (its segmented path + # never triggers for single-channel input), so blocks stay + # short; context warms both edges and the overlap crossfades. + block, context, overlap = 720000, 96000, 48000 + previous = None + report(f"Denoising ({device})", 0, frames) + for start in range(0, frames, block): + end = min(start + block, frames) + left, right = max(0, start - context), min(frames, end + overlap + context) + opened.seek(left) + audio = opened.read(right - left, dtype="float32") + if not np.isfinite(audio).all(): + raise ValueError("Input contains non-finite audio samples") + if args.strength: + wet = enhance(audio) + audio = audio * (1 - args.strength) + wet * args.strength + kept = audio[start - left:min(frames, end + overlap) - left].copy() + if previous is not None and args.strength: + n = len(previous) + fade = np.linspace(0, 1, n, dtype="float32") + kept[:n] = previous * (1 - fade) + kept[:n] * fade + destination.write(kept[:end - start]) + previous = kept[end - start:].copy() + report(f"Denoising ({device})", end, frames) + if destination.tell() != frames: + raise RuntimeError("Output frame count does not match input") + os.replace(temporary, target) + temporary = None + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + + +if __name__ == "__main__": + main() diff --git a/lib/project/src/producer/pipeline.py b/lib/project/src/producer/pipeline.py index d7c2ced..d6c0cf1 100644 --- a/lib/project/src/producer/pipeline.py +++ b/lib/project/src/producer/pipeline.py @@ -73,7 +73,9 @@ def process(source: Path, destination: Path, settings: Settings, progress, raise ValueError("Master output must have a .wav extension") report_path = report_for(destination) for target in targets_for(destination, settings): - check_target(source, target, overwrite) + # Report logs are diagnostic: a rerun replaces them silently. Only + # audio outputs require --overwrite. + check_target(source, target, overwrite or target == report_path) ffmpeg = ensure_ffmpeg() original = measure(source, progress, "Analyzing original") if original["sample_peak_dbfs"] < -100: @@ -101,15 +103,31 @@ def process(source: Path, destination: Path, settings: Settings, progress, ffmpeg_run(ffmpeg, ["-i", str(source), "-map", "0:a:0", "-af", ",".join(input_filters or ["anull"]), "-ar", "48000", "-ac", "1", "-c:a", "pcm_f32le", "-rf64", "auto", str(prepared)], "Preparing mono 48 kHz audio", duration, progress) - if settings.denoiser == "deepfilter" and settings.denoise_strength > 0: + if settings.denoiser in {"deepfilter", "clearvoice"} and settings.denoise_strength > 0: from .ai import denoise - denoise(prepared, cleaned, settings.device, settings.denoise_strength, progress) + denoise(prepared, cleaned, settings.device, settings.denoise_strength, progress, + backend=settings.denoiser) elif settings.denoiser == "fft" and settings.denoise_strength > 0: ffmpeg_run(ffmpeg, ["-i", str(prepared), "-af", f"afftdn=nr={max(0.01, settings.fft_reduction_db * settings.denoise_strength)}:tn=1", "-c:a", "pcm_f32le", "-rf64", "auto", str(cleaned)], "FFT noise reduction", duration, progress) else: cleaned = prepared + # Static gap trimming shortens long silences after cleanup; speech is + # copied through, so it stays independent of the denoiser backend. + trimmed = work / "trimmed.wav" + gaps_trimmed, silence_removed = 0, 0.0 + if settings.trim_silence: + from .audio import trim_gaps + gaps_trimmed, silence_removed = trim_gaps(cleaned, trimmed, + settings.gap_threshold_db, settings.max_gap_s, settings.gap_pad_s, + progress) + cleaned.unlink(missing_ok=True) + cleaned = trimmed + if settings.trim_silence and gaps_trimmed == 0: + warnings.append("Gap trimming found no silences below the configured " + "threshold; nothing was trimmed. Check gap_threshold_db.") + trimmed_duration = duration - silence_removed clean_stats = measure(cleaned, progress, "Measuring cleaned voice") if clean_stats["sample_peak_dbfs"] < -100: raise ValueError("The audio is silent or too quiet to master safely; " @@ -127,8 +145,9 @@ def process(source: Path, destination: Path, settings: Settings, progress, warnings.append("Input level correction reached its configured gain limit.") chain = ([f"volume={gain}dB"] if gain else []) + filters(settings) ffmpeg_run(ffmpeg, ["-i", str(cleaned), "-af", ",".join(chain), - "-c:a", "pcm_f32le", "-rf64", "auto", str(shaped)], "EQ, dynamics and de-essing", duration, progress) - before = loudness(ffmpeg, shaped, settings, duration, progress, "Loudness analysis (pass 1)") + "-c:a", "pcm_f32le", "-rf64", "auto", str(shaped)], + "EQ, dynamics and de-essing", trimmed_duration, progress) + before = loudness(ffmpeg, shaped, settings, trimmed_duration, progress, "Loudness analysis (pass 1)") if settings.normalize and any(before.get(k) is None for k in ("input_i", "input_tp", "input_lra", "input_thresh", "target_offset")): raise ValueError("Cannot normalize: insufficient measurable audio after processing") final_filters = [] @@ -142,7 +161,7 @@ def process(source: Path, destination: Path, settings: Settings, progress, master = work / "master.wav" render_log = ffmpeg_run(ffmpeg, ["-i", str(shaped), "-af", ",".join(final_filters), "-ar", str(settings.sample_rate), "-c:a", "pcm_s32le", "-rf64", "auto", str(master)], - "Mastering (pass 2)", duration, progress) + "Mastering (pass 2)", trimmed_duration, progress) # Bounded disk use: drop consumed intermediates instead of waiting for # the temporary directory teardown at the end of the run. shaped.unlink(missing_ok=True) @@ -154,10 +173,10 @@ def process(source: Path, destination: Path, settings: Settings, progress, if start < 0 or end < start: raise RuntimeError("FFmpeg did not report the applied normalization mode") normalization_mode = json.loads(render_log[start:end + 1])["normalization_type"] - final = loudness(ffmpeg, master, settings, duration, progress, "Verifying exported master") + final = loudness(ffmpeg, master, settings, trimmed_duration, progress, "Verifying exported master") final_stats = measure(master, progress, "Checking exported samples") - if abs(final_stats["duration_seconds"] - duration) > 0.02: - raise RuntimeError("Output duration differs from input by more than 20 ms") + if abs(final_stats["duration_seconds"] - trimmed_duration) > 0.02: + raise RuntimeError("Output duration differs from the trimmed timeline by more than 20 ms") if (final_stats["channels"] != 1 or final_stats["sample_rate"] != settings.sample_rate or final_stats["subtype"] != "PCM_32"): raise RuntimeError("Exported master is not the expected mono PCM_32 WAV") @@ -171,13 +190,14 @@ def process(source: Path, destination: Path, settings: Settings, progress, if settings.mp3: mp3 = work / "delivery.mp3" ffmpeg_run(ffmpeg, ["-i", str(master), "-c:a", "libmp3lame", "-b:a", - f"{settings.mp3_bitrate}k", str(mp3)], "Encoding MP3", duration, progress) - mp3_stats = loudness(ffmpeg, mp3, settings, duration, progress, "Verifying decoded MP3") + f"{settings.mp3_bitrate}k", str(mp3)], "Encoding MP3", trimmed_duration, progress) + mp3_stats = loudness(ffmpeg, mp3, settings, trimmed_duration, progress, "Verifying decoded MP3") if mp3_stats["input_tp"] is not None and mp3_stats["input_tp"] > settings.true_peak_db + 0.1: warnings.append("MP3 encoding increased true peak beyond the configured ceiling; use more headroom.") report = {"producer_version": __version__, "source": str(source), "output": str(destination), "settings": asdict(settings), "input": original, "cleaned": clean_stats, "level_correction_db": gain, + "gaps_trimmed": gaps_trimmed, "silence_removed_seconds": round(silence_removed, 3), "pre_master_loudness": before, "master_loudness": final, "master": final_stats, "mp3_loudness": mp3_stats, "normalization_mode": normalization_mode, @@ -194,7 +214,7 @@ def process(source: Path, destination: Path, settings: Settings, progress, fcntl.flock(publication_fd, fcntl.LOCK_EX) if settings.mp3: publish(mp3, destination.with_suffix(".mp3"), overwrite) - publish(staged_report, report_path, overwrite) + publish(staged_report, report_path, True) publish(master, destination, overwrite) finally: os.close(publication_fd) diff --git a/lib/project/tests/test_config.py b/lib/project/tests/test_config.py index 02b9eba..8da7a7f 100644 --- a/lib/project/tests/test_config.py +++ b/lib/project/tests/test_config.py @@ -131,3 +131,17 @@ def test_shaping_filter_toggles(bypass, toggle, prefix, count): chain = filters(bypass) assert len(chain) == count assert all(item.startswith(prefix) for item in chain) + + +def test_clearvoice_and_trim_settings_parse_and_validate(): + settings = resolve(overrides={"denoiser": "clearvoice", "trim_silence": True, + "max_gap_s": 2.0, "gap_pad_s": 0.5, + "gap_threshold_db": -45.0}) + assert settings.denoiser == "clearvoice" and settings.trim_silence is True + assert settings_for(parser().parse_args(["config", "--denoiser", "clearvoice"])).denoiser == "clearvoice" + with pytest.raises(ValueError, match="gap_pad_s must be smaller than max_gap_s"): + resolve(overrides={"trim_silence": True, "max_gap_s": 1.0, "gap_pad_s": 1.0}) + with pytest.raises(ValueError, match="max_gap_s"): + resolve(overrides={"max_gap_s": 0.4}) + with pytest.raises(ValueError, match="gap_threshold_db"): + resolve(overrides={"gap_threshold_db": -19.0}) diff --git a/lib/project/tests/test_pipeline.py b/lib/project/tests/test_pipeline.py index 797bc12..fe04519 100644 --- a/lib/project/tests/test_pipeline.py +++ b/lib/project/tests/test_pipeline.py @@ -37,11 +37,11 @@ def test_original_is_never_overwritten(wav, tmp_path, bypass, progress, monkeypa assert source.read_bytes() == original -@pytest.mark.parametrize("suffix", [".wav", ".report.json", ".mp3"]) +@pytest.mark.parametrize("suffix", [".wav", ".mp3"]) def test_existing_output_or_sidecar_rejected_before_processing(wav, tmp_path, bypass, progress, monkeypatch, suffix): source = wav() target = tmp_path / "master.wav" - existing = pipeline.report_for(target) if suffix == ".report.json" else target.with_suffix(suffix) + existing = target.with_suffix(suffix) existing.parent.mkdir(parents=True, exist_ok=True) existing.write_bytes(b"existing output") bypass.mp3 = True @@ -51,6 +51,16 @@ def test_existing_output_or_sidecar_rejected_before_processing(wav, tmp_path, by assert existing.read_bytes() == b"existing output" +def test_existing_report_log_is_replaced_without_overwrite(wav, tmp_path, bypass, progress, ffmpeg): + source = wav() + target = tmp_path / "master.wav" + existing = pipeline.report_for(target) + existing.parent.mkdir(parents=True, exist_ok=True) + existing.write_bytes(b"stale report") + report = pipeline.process(source, target, bypass, progress) + assert json.loads(existing.read_text()) == report + + @pytest.mark.parametrize("kind,message", [ ("empty", "empty"), ("silence", "silent or too quiet"), ("quiet", "silent or too quiet"), ("surround", "Only mono or stereo"), @@ -226,3 +236,48 @@ def test_publish_new_output_moves_staged_file(tmp_path): pipeline.publish(staged, target, overwrite=False) assert target.read_bytes() == b"finished" assert not staged.exists() + + +def test_trim_gaps_shortens_silence_and_keeps_speech_exact(wav, tmp_path, progress): + from producer.audio import trim_gaps + sr = 48000 + speech_a = 0.2 * np.sin(2 * np.pi * 440 * np.arange(2 * sr) / sr) + room_tone = 0.001 * np.random.default_rng(7).normal(0, 1, int(3 * sr)) # -60 dBFS + speech_b = 0.2 * np.sin(2 * np.pi * 880 * np.arange(2 * sr) / sr) + source = tmp_path / "gaps.wav" + sf.write(source, np.concatenate([speech_a, room_tone, speech_b]), sr, format="WAV", subtype="FLOAT") + target = tmp_path / "trimmed.wav" + trimmed, removed = trim_gaps(source, target, threshold_db=-50, max_gap_s=1.5, + pad_s=0.6, progress=progress) + result, rate = sf.read(target, dtype="float32") + assert rate == 48000 and trimmed == 1 + assert removed == pytest.approx(3 - 0.6, abs=0.02) # 0.3 s pad kept at each edge + # Speech is copied through unattenuated and sample-exact. + np.testing.assert_allclose(result[:len(speech_a)], speech_a, atol=1e-7) + np.testing.assert_allclose(result[-len(speech_b):], speech_b, atol=1e-6) + kept_gap = result[len(speech_a):len(speech_a) + int(0.6 * sr)] + assert np.count_nonzero(kept_gap) == int(0.6 * sr) # room tone preserved, no gate + + +def test_trim_and_clearvoice_dispatch_flow(wav, tmp_path, bypass, progress, ffmpeg, monkeypatch): + def fake_denoise(source, target, device, strength, progress, backend): + assert backend == "clearvoice" + import shutil + shutil.copyfile(source, target) + + monkeypatch.setattr("producer.ai.denoise", fake_denoise) + bypass.denoiser = "clearvoice" + bypass.denoise_strength = 1.0 + bypass.trim_silence = True + bypass.max_gap_s = 1.0 + bypass.gap_pad_s = 0.5 + sr = 48000 + speech = 0.15 * np.sin(2 * np.pi * 440 * np.arange(sr) / sr) + room_tone = 0.001 * np.random.default_rng(7).normal(0, 1, int(2 * sr)) + source = wav(np.concatenate([speech, room_tone, speech])) + report = pipeline.process(source, tmp_path / "master.wav", bypass, progress) + assert report["gaps_trimmed"] == 1 + assert report["silence_removed_seconds"] > 0 + assert report["master"]["duration_seconds"] == pytest.approx( + report["input"]["duration_seconds"] - report["silence_removed_seconds"], abs=0.02) + assert report["warnings"] == [] diff --git a/producer.sh b/producer.sh index f2dc6be..d2a139f 100755 --- a/producer.sh +++ b/producer.sh @@ -7,9 +7,10 @@ # # Trust boundary for downloads: official PyPI (uv, application wheels) and # Astral's managed CPython distribution. AI setup additionally trusts -# download.pytorch.org and the pinned DeepFilterNet3 model host. Host -# package-manager settings (extra indexes, targets, constraints, config -# files) cannot add to or redirect these sources. +# download.pytorch.org, the pinned DeepFilterNet3 model host, and Hugging Face +# (ClearVoice MossFormer2_SE_48K weights). Host package-manager settings +# (extra indexes, targets, constraints, config files) cannot add to or +# redirect these sources. # # First bootstrap needs an existing `uv`, or a Python 3 with venv support. # Later runs, including moving or renaming this whole directory, are handled @@ -44,6 +45,7 @@ Usage: ./producer.sh COMMAND [options], or simply: config Print resolved settings (TOML, or JSON with --json) doctor Check FFmpeg, data directory and CUDA driver setup --device cpu|cuda Install and self-test the AI backend + (--denoiser clearvoice for the MossFormer2 backend) Outputs are written beside each input by default. Run './producer.sh COMMAND --help' for the options of a command. -- cgit v1.2.3