fix: silently failing whisper transcription

4 files changed, 239 insertions, 29 deletions

diff --git a/app/converter/clients/__init__.py b/app/converter/clients/__init__.py
index 1fcfd1a..d02fd9f 100644
--- a/app/converter/clients/__init__.py
+++ b/app/converter/clients/__init__.py
@@ -20,7 +20,9 @@ from .languages import LANGUAGE_CHOICES, LANGUAGE_ISO_CODES, TTS_LANGUAGES, \
     TTS_LANGUAGE_ALIASES, normalize_language
 from .speakers import QWEN3_TTS_SPEAKERS, SPEAKER_DISPLAY_NAMES, \
     is_builtin_speaker, speaker_display_name, speaker_display_name_for
-from .transcribe import transcribe_reference_audio, whisper_backend_available
+from .transcribe import (transcribe_reference_audio,
+                         transcribe_reference_audio_detailed,
+                         whisper_backend_available, whisper_backend_problem)
 from .qwen import CUSTOM_VOICE_MODEL_ID, MODEL_SIZE, QwenTTSClient
 from .faster import SAMPLE_RATE, FasterTTSClient
 from .audiocpp import (
@@ -57,7 +59,8 @@ __all__ = [
     "QWEN3_TTS_SPEAKERS", "SPEAKER_DISPLAY_NAMES",
     "speaker_display_name", "speaker_display_name_for", "is_builtin_speaker",
     # transcription
-    "transcribe_reference_audio", "whisper_backend_available",
+    "transcribe_reference_audio", "transcribe_reference_audio_detailed",
+    "whisper_backend_available", "whisper_backend_problem",
     # audio.cpp family profiles
     "AUDIOCPP_LANG_DISPLAY", "AUDIOCPP_LANG_ISO", "AUDIOCPP_LANG_OMIT",
     "AUDIOCPP_FAMILY_QWEN3_TTS", "AUDIOCPP_TASK_TTS", "AUDIOCPP_TASK_VDES",
diff --git a/app/converter/clients/audiocpp.py b/app/converter/clients/audiocpp.py
index 4a161cb..8c446d3 100644
--- a/app/converter/clients/audiocpp.py
+++ b/app/converter/clients/audiocpp.py
@@ -13,7 +13,8 @@ from typing import Any, Dict, List, Optional
 from .. import config
 from ..audio import concat_audio_files
 from ..chunking import split_into_chunks
-from .base import BaseTTSClient, ConversionCancelled, resolve_request_seed
+from .base import (BaseTTSClient, ConversionCancelled,
+                   NonRetryableTTSError, resolve_request_seed)
 from .languages import LANGUAGE_ISO_CODES, normalize_language
 from .speakers import (is_builtin_speaker, speaker_display_name,
                        speaker_display_name_for)
@@ -56,6 +57,92 @@ AUDIOCPP_VOICE_SPEAKER = "speaker"  # built-in speaker name (Qwen CustomVoice)
 AUDIOCPP_VOICE_CLONE = "clone"      # server-side preset / voice_dir (Base, others)
 AUDIOCPP_VOICE_DESIGN = "design"    # voice described by --instructions (vdes)
 
+# HTTP error body fragments identifying deterministic request-configuration
+# problems: the identical request will fail on every retry, so the chunk
+# loop must give up immediately instead of burning its attempt budget.
+# Matched case-insensitively against the server's error message; the
+# fragments come from audio.cpp itself, so they hold for every hosted
+# family (none are model-specific).
+AUDIOCPP_NON_RETRYABLE_ERRORS = (
+    # Cloning without the reference transcript (Qwen3-TTS Base ICL mode):
+    # the server-side voice has reference audio but no transcript for it.
+    "requires reference text",
+    # The server cannot resolve a model contract for the family (its own
+    # hint text about model_specs/--model-spec-override follows the fragment).
+    "model contract spec not found for family",
+    "does not embed an audio.cpp model spec",
+    "embeds a legacy model spec",
+    # The request named a model the server does not host.
+    "unknown model id",
+)
+_REFERENCE_TEXT_FRAGMENT = AUDIOCPP_NON_RETRYABLE_ERRORS[0]
+
+
+def _server_error_message(detail: str) -> str:
+    """The server's error message from an HTTP error body, else the body.
+
+    The speech endpoint wraps failures as {"error": {"message": ...}};
+    the inner message is what matches AUDIOCPP_NON_RETRYABLE_ERRORS and
+    what the user should see. Unparseable bodies are returned as-is.
+    """
+    try:
+        payload = json.loads(detail)
+    except ValueError:
+        return detail
+    if isinstance(payload, dict):
+        error = payload.get("error")
+        if isinstance(error, dict) and isinstance(error.get("message"), str):
+            return error["message"]
+        if isinstance(error, str):
+            return error
+    return detail
+
+
+def _reference_text_error(voice: Optional[str], server_message: str) -> str:
+    """Actionable message for the missing-reference-transcript failure.
+
+    The server resolved the requested voice to reference audio but has no
+    transcript for it, so its ICL voice-clone path rejects every request.
+    The fix is server-side data, not a client retry: prompt_text (or the
+    voice preset's reference_text) supplies it, read per request, so no
+    server restart is needed. x_vector_only_mode is the transcript-free
+    escape hatch, at the cost of speaker similarity.
+    """
+    name = f"'{voice}'" if voice else "the requested voice"
+    return (
+        f"The audio.cpp server cannot clone voice {name}: its reference "
+        "audio has no transcript, and this model family's voice cloning "
+        f"requires one ({server_message}). Add the transcript to the "
+        "prompt_text file in the server's voice directory (one "
+        "'<voice>|<transcript>' line per voice) or set reference_text on "
+        "the voice preset in server.json; the server reads it per request, "
+        "no restart needed. Re-running the audio.cpp setup re-transcribes "
+        "the reference wavs with whisper. Alternatively rerun with "
+        "--option x_vector_only_mode=true to clone from the speaker "
+        "embedding alone (no transcript needed; lower similarity)."
+    )
+
+
+def audiocpp_request_error(status: int, detail: str,
+                           voice: Optional[str] = None) -> Exception:
+    """The exception for a failed audio.cpp speech request.
+
+    Deterministic request-configuration errors (a fragment in
+    AUDIOCPP_NON_RETRYABLE_ERRORS) become NonRetryableTTSError so the
+    chunk retry loop skips attempts that cannot succeed; everything else
+    returns the plain RuntimeError the retry loop has always retried.
+    """
+    message = _server_error_message(detail)
+    lowered = message.lower()
+    if _REFERENCE_TEXT_FRAGMENT in lowered:
+        return NonRetryableTTSError(
+            _reference_text_error(voice, message))
+    if any(fragment in lowered for fragment in AUDIOCPP_NON_RETRYABLE_ERRORS):
+        return NonRetryableTTSError(
+            f"audio.cpp server returned HTTP {status} (not retryable): "
+            f"{message}")
+    return RuntimeError(f"audio.cpp server returned HTTP {status}: {detail}")
+
 
 class AudioCppFamilyProfile:
     """Request conventions of one audio.cpp model family.
@@ -665,7 +752,8 @@ class AudioCppTTSClient(BaseTTSClient):
                 detail = exc.read().decode("utf-8", errors="replace")[:200]
             except Exception:
                 pass
-            raise RuntimeError(f"audio.cpp server returned HTTP {exc.code}: {detail}") from exc
+            raise audiocpp_request_error(exc.code, detail,
+                                         voice=self.voice) from exc
         except urllib.error.URLError as exc:
             raise RuntimeError(f"audio.cpp request failed: {exc.reason}") from exc
         if len(wav) < 12 or wav[:4] != b"RIFF" or wav[8:12] != b"WAVE":
@@ -710,6 +798,10 @@ class AudioCppTTSClient(BaseTTSClient):
 
         except ConversionCancelled:
             raise
+        except NonRetryableTTSError:
+            # Propagate past the generic handler so the retry loop skips
+            # its remaining attempts for deterministic server errors.
+            raise
         except Exception as exc:
             logger.error("audio.cpp chunk processing failed for chunk %d: %s",
                          chunk_num, exc)
diff --git a/app/converter/clients/base.py b/app/converter/clients/base.py
index d0d57fb..8a0b4e4 100644
--- a/app/converter/clients/base.py
+++ b/app/converter/clients/base.py
@@ -17,6 +17,19 @@ class ConversionCancelled(Exception):
     """Raised when the run's cancel event is set (between requests)."""
 
 
+class NonRetryableTTSError(RuntimeError):
+    """A deterministic server-side request/config error; retrying cannot help.
+
+    Raised by clients for failures that will reproduce identically on every
+    attempt (missing server-side reference transcripts, unknown model ids,
+    missing model contracts, ...). process_chunk_with_retry skips its
+    remaining attempts and back-off sleeps for these and re-raises, so the
+    converter aborts with the actionable message instead of burning the
+    retry budget on a request that can never succeed. Subclasses RuntimeError
+    so except-handlers written for the plain HTTP-error case keep working.
+    """
+
+
 # How a run supplies its voice: a built-in CustomVoice speaker, by cloning
 # a reference audio clip (the faster and audiocpp backends always clone
 # server-side; only the Qwen client branches on this at request time), or
@@ -109,6 +122,8 @@ class BaseTTSClient:
 
         Returns the generated chunk file's path, or None when all attempts
         failed. Raises ConversionCancelled when the run was cancelled.
+        NonRetryableTTSError is logged once and re-raised without consuming
+        the remaining attempts (the identical request can never succeed).
         """
         for attempt in range(config.MAX_RETRIES):
             self._check_cancelled()
@@ -119,6 +134,9 @@ class BaseTTSClient:
                 logger.warning("Chunk %d attempt %d failed", chunk_num, attempt + 1)
             except ConversionCancelled:
                 raise
+            except NonRetryableTTSError as exc:
+                logger.error("Chunk %d error (not retried): %s", chunk_num, exc)
+                raise
             except Exception as exc:
                 logger.warning("Chunk %d attempt %d error: %s", chunk_num, attempt + 1, exc)
 
diff --git a/app/converter/clients/transcribe.py b/app/converter/clients/transcribe.py
index d2db9f1..28c4907 100644
--- a/app/converter/clients/transcribe.py
+++ b/app/converter/clients/transcribe.py
@@ -1,39 +1,115 @@
 """Optional local Whisper transcription of reference audio."""
 
 import logging
-from typing import Optional
+import threading
+from typing import Dict, Optional, Tuple
 
 logger = logging.getLogger(__name__)
 
+# One loaded Whisper model per (backend, model size), reused across calls.
+# The audio.cpp setup transcribes every reference wav in one run; loading
+# the model once instead of per file saves seconds per voice. The lock
+# keeps a TUI lane and a concurrent conversion from racing the load.
+_MODEL_LOCK = threading.Lock()
+_MODELS: Dict[Tuple[str, str], object] = {}
 
-def transcribe_reference_audio(audio_path: str, model_name: str = "base") -> Optional[str]:
-    """Transcribe reference audio locally using an optional Whisper backend.
 
-    The current qwen-tts demo does not expose a transcription endpoint, so
-    transcription is done client-side when a Whisper package is available.
-    Returns None if no backend is installed.
+def _cached_model(key: Tuple[str, str], loader):
+    """The model for KEY, loaded by LOADER() on first use and cached."""
+    with _MODEL_LOCK:
+        model = _MODELS.get(key)
+        if model is None:
+            model = loader()
+            _MODELS[key] = model
+        return model
+
+
+def _import_failure_reason(backend: str, exc: ImportError) -> str:
+    """Why importing BACKEND failed: missing package or broken install.
+
+    A backend whose own compiled dependency fails to load raises the same
+    ImportError as a missing package (e.g. faster-whisper whose av build
+    cannot load its shared libraries); telling the two apart turns 'was it
+    even installed?' into the actual fix. A ModuleNotFoundError naming the
+    backend itself is a plain missing package; anything else is checked
+    against the installed distributions.
     """
-    for backend in ("faster_whisper", "whisper"):
-        try:
-            if backend == "faster_whisper":
+    if isinstance(exc, ModuleNotFoundError) \
+            and getattr(exc, "name", "") == backend:
+        return f"{backend} is not installed"
+    try:
+        import importlib.util
+        present = importlib.util.find_spec(backend) is not None
+    except (ImportError, ValueError):
+        present = True
+    if not present:
+        return f"{backend} is not installed"
+    return f"{backend} is installed but failed to import: {exc}"
+
+
+def _transcribe_with(backend: str, audio_path: str,
+                     model_name: str) -> Tuple[Optional[str], str]:
+    """One transcription attempt with BACKEND; returns (text, reason).
+
+    TEXT is the transcript, or None on any failure; REASON then explains
+    why in one human-readable line (missing package, broken import, model
+    or transcribe error, or an empty result), so callers can surface the
+    cause instead of a bare 'no transcript'.
+    """
+    try:
+        if backend == "faster_whisper":
+            def load():
                 from faster_whisper import WhisperModel
-                model = WhisperModel(model_name, device="cpu", compute_type="int8")
-                segments, _ = model.transcribe(audio_path)
-                text = " ".join(seg.text.strip() for seg in segments).strip()
-            else:
+                return WhisperModel(model_name, device="cpu", compute_type="int8")
+            model = _cached_model(("faster_whisper", model_name), load)
+            segments, _ = model.transcribe(audio_path)
+            text = " ".join(seg.text.strip() for seg in segments).strip()
+        else:
+            def load():
                 import whisper
-                model = whisper.load_model(model_name)
-                result = model.transcribe(audio_path)
-                text = (result.get("text") or "").strip()
-            if text:
-                logger.info("Transcription complete via %s: %s", backend, text)
-                return text
-        except ImportError:
-            continue
-        except Exception as exc:
-            logger.warning("%s transcription failed: %s", backend, exc)
-    logger.warning("No Whisper backend available; transcription skipped.")
-    return None
+                return whisper.load_model(model_name)
+            model = _cached_model(("whisper", model_name), load)
+            result = model.transcribe(audio_path)
+            text = (result.get("text") or "").strip()
+    except ImportError as exc:
+        return None, _import_failure_reason(backend, exc)
+    except Exception as exc:
+        return None, f"{backend} transcription failed: {exc}"
+    if not text:
+        return None, f"{backend} heard no speech in this audio"
+    logger.info("Transcription complete via %s: %s", backend, text)
+    return text, "ok"
+
+
+def transcribe_reference_audio_detailed(
+        audio_path: str, model_name: str = "base") -> Tuple[Optional[str], str]:
+    """Transcribe one reference wav locally; returns (text, reason).
+
+    The current qwen-tts demo does not expose a transcription endpoint, so
+    transcription is done client-side when a Whisper backend is available.
+    TEXT is the transcript, or None when no backend produced one; REASON
+    is "ok" on success and otherwise explains the failure (tried backends
+    in order), e.g. "faster_whisper is installed but failed to import:
+    ...; whisper is not installed" — what the audio.cpp setup prints so a
+    voice that cannot be transcribed is never a silent blank.
+    """
+    reasons = []
+    for backend in ("faster_whisper", "whisper"):
+        text, reason = _transcribe_with(backend, audio_path, model_name)
+        if text:
+            return text, "ok"
+        reasons.append(reason)
+    return None, "; ".join(reasons)
+
+
+def transcribe_reference_audio(audio_path: str, model_name: str = "base") -> Optional[str]:
+    """Transcribe reference audio locally using an optional Whisper backend.
+
+    Returns None if no backend is installed or transcription failed;
+    transcribe_reference_audio_detailed also explains why.
+    """
+    text, _ = transcribe_reference_audio_detailed(audio_path, model_name)
+    return text
 
 
 def whisper_backend_available() -> Optional[str]:
@@ -52,3 +128,24 @@ def whisper_backend_available() -> Optional[str]:
             continue
         return backend
     return None
+
+
+def whisper_backend_problem() -> Optional[str]:
+    """None when a Whisper backend is importable, else why none is usable.
+
+    One line per backend in probe order, distinguishing 'not installed'
+    from 'installed but failed to import: <error>' — the setup prints this
+    before transcribing so a broken compiled dependency (which surfaces as
+    the same ImportError as a missing package) is visible as such.
+    """
+    problems = []
+    for backend in ("faster_whisper", "whisper"):
+        try:
+            __import__(backend)
+        except ImportError as exc:
+            problems.append(_import_failure_reason(backend, exc))
+        except Exception as exc:
+            problems.append(f"{backend} failed to import: {exc}")
+        else:
+            return None
+    return "; ".join(problems)