fix(converter): stream audio concat and harden error/encoding handling

7 files changed, 183 insertions, 102 deletions

diff --git a/README.md b/README.md
index ecf4d04..44422b9 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # Qwen3 Audiobook Converter
 
-Convert TXT, PDF, EPUB, DOCX, and DOC files into audiobooks using the Qwen3-TTS voice model.
+Convert TXT, PDF, and EPUB files into audiobooks using the Qwen3-TTS voice model.
 
 Original project: [https://github.com/WhiskeyCoder/Qwen3-Audiobook-Converter](https://github.com/WhiskeyCoder/Qwen3-Audiobook-Converter
 ). This repo just has minor fixes, flags, and documentation updates. It also splits the qwen3-tts server into two processes running models on different ports.
@@ -9,7 +9,7 @@ Original project: [https://github.com/WhiskeyCoder/Qwen3-Audiobook-Converter](ht
 
 The converter sends text extracted from your books to a locally running Qwen3-TTS server and assembles the returned audio into a single audiobook file.
 
-- Supported input: `.txt`, `.pdf`, `.epub`, `.docx`, `.doc`
+- Supported input: `.txt`, `.pdf`, `.epub`
 - Output: `.mp3`
 - Two voice modes:
   - Custom voice: pre-built speakers
@@ -75,7 +75,7 @@ Put your book files (epub, txt, etc.) in the `book_to_convert/` folder. Then run
 python audiobook_converter.py
 ```
 
-Edit the parameters at the top of `audiobook_converter.py` to change which built-in voice is used.
+Edit `converter/config.py` to change which built-in voice is used.
 
 ```
 CUSTOM_VOICE_SPEAKER = "Vivian"   # Serena, Vivian, Uncle_Fu, Aiden, Ono_Anna, Sohee, Eric, Dylan
@@ -105,6 +105,14 @@ Adjust the speed of the final audiobook without changing pitch (uses ffmpeg `ate
 python audiobook_converter.py --speed 0.9
 ```
 
+The `chunks/` folder is scratch space for the current book only — it is emptied before and after every conversion, so an interrupted run never affects the next one.
+
+## Running tests
+
+```bash
+python -m unittest discover -s tests -t .
+```
+
 ## FlashAttention (optional)
 
 The server tries to use FlashAttention 2 by default, but `--no-flash-attn` works without it. On supported GPUs FlashAttention can give a modest speedup.
diff --git a/audiobook_converter.py b/audiobook_converter.py
index bb21e64..d2e7f41 100644
--- a/audiobook_converter.py
+++ b/audiobook_converter.py
@@ -101,14 +101,17 @@ Examples:
             skip_transcription=args.no_transcription,
             speed=args.speed,
         )
-        converter.run()
+        ok = converter.run()
     except KeyboardInterrupt:
         print("\n[WARNING] Shutdown requested by user")
+        sys.exit(130)
     except Exception as exc:
         print(f"[FATAL] Fatal error: {exc}")
         traceback.print_exc()
         sys.exit(1)
 
+    sys.exit(0 if ok else 1)
+
 
 if __name__ == "__main__":
     main()
diff --git a/converter/audio.py b/converter/audio.py
index ab55e9b..e7fa0bb 100644
--- a/converter/audio.py
+++ b/converter/audio.py
@@ -1,6 +1,8 @@
 """Audio assembly: combining chunks, speed adjustment, cleanup."""
 
 import logging
+import shutil
+import subprocess
 import traceback
 from pathlib import Path
 from typing import Dict, List, Optional
@@ -10,16 +12,16 @@ from . import config
 logger = logging.getLogger(__name__)
 
 
-def speed_export_params(speed: float) -> List[str]:
-    """Return ffmpeg filter args for pitch-preserving speed adjustment.
+def atempo_filters(speed: float) -> str:
+    """Return a comma-joined ffmpeg ``atempo`` filter chain for ``speed``.
 
-    Uses ffmpeg's atempo filter, which accepts 0.5..2.0 per filter. Values
-    outside that range are handled by chaining multiple atempo filters.
+    ``atempo`` accepts 0.5..2.0 per filter; values outside that range are
+    handled by chaining multiple filters. Returns "" when ``speed`` is 1.0.
     """
     if speed <= 0:
         raise ValueError(f"Speed must be a positive number, got {speed}")
     if abs(speed - 1.0) < 1e-6:
-        return []
+        return ""
     remaining = float(speed)
     chain = []
     while remaining > 2.0:
@@ -29,84 +31,119 @@ def speed_export_params(speed: float) -> List[str]:
         chain.append("atempo=0.5")
         remaining /= 0.5
     chain.append(f"atempo={remaining:g}")
-    return ["-filter:a", ",".join(chain)]
+    return ",".join(chain)
+
+
+def speed_export_params(speed: float) -> List[str]:
+    """Return ffmpeg filter args for pitch-preserving speed adjustment."""
+    filters = atempo_filters(speed)
+    if not filters:
+        return []
+    return ["-filter:a", filters]
+
+
+def _concat_escape(path: str) -> str:
+    """Escape a path for use inside single quotes in an ffmpeg concat list."""
+    return path.replace("'", "'\\''")
 
 
 def combine_chunks(total_chunks: int, output_path: Path,
                    results: Optional[Dict[int, bool]] = None, speed: float = 1.0) -> bool:
-    """Combine audio chunks into the final audiobook.
+    """Combine audio chunks into the final audiobook using ffmpeg's concat demuxer.
 
     ``results`` maps chunk numbers to success flags; failed chunks are
     skipped. When ``speed`` differs from 1.0, an additional speed-adjusted
-    copy is written next to the normal-speed file.
+    copy is written next to the normal-speed file. Chunks are streamed by
+    ffmpeg, so the whole book is never held in memory.
     """
-    try:
-        from pydub import AudioSegment
-    except ImportError:
-        logger.error("pydub is required to combine audio chunks (pip install pydub)")
+    if shutil.which("ffmpeg") is None:
+        logger.error("ffmpeg is required to combine audio chunks (install ffmpeg)")
         return False
 
-    try:
-        combined = AudioSegment.empty()
-        successful = 0
-        missing_chunks = []
-
-        for i in range(1, total_chunks + 1):
-            # Skip chunks that failed if we have results tracking
-            if results is not None and not results.get(i, False):
-                missing_chunks.append(i)
-                continue
-
-            chunk_file = config.CHUNKS_FOLDER / f"chunk_{i:04d}.wav"
-            if chunk_file.exists():
-                try:
-                    combined += AudioSegment.from_wav(str(chunk_file))
-                    successful += 1
-                    if successful % 10 == 0:
-                        logger.info("Combined %d chunks", successful)
-                except Exception as exc:
-                    logger.warning("Failed to load chunk %d: %s", i, exc)
-                    missing_chunks.append(i)
-            else:
-                logger.warning("Chunk file not found: %s", chunk_file)
-                missing_chunks.append(i)
-
-        if successful == 0:
-            raise RuntimeError("No valid chunks found")
+    chunk_files = []
+    missing_chunks = []
+    for i in range(1, total_chunks + 1):
+        # Skip chunks that failed if we have results tracking
+        if results is not None and not results.get(i, False):
+            missing_chunks.append(i)
+            continue
+
+        matches = sorted(config.CHUNKS_FOLDER.glob(f"chunk_{i:04d}.*"))
+        if matches:
+            chunk_files.append(matches[0])
+        else:
+            missing_chunks.append(i)
+
+    if not chunk_files:
+        logger.error("No valid chunks found")
+        return False
 
-        if missing_chunks:
-            logger.warning("Missing chunks: %s", missing_chunks)
+    if missing_chunks:
+        logger.warning("Missing chunks: %s", missing_chunks)
 
-        combined.export(str(output_path), format=config.AUDIO_FORMAT, bitrate=config.AUDIO_BITRATE)
-        logger.info("Audiobook saved: %s (%d/%d chunks)", output_path, successful, total_chunks)
-        print(f"[INFO] Saved audiobook: {output_path.name} ({successful}/{total_chunks} chunks)")
+    concat_list = config.CHUNKS_FOLDER / "_concat_list.txt"
+    try:
+        with open(concat_list, "w", encoding="utf-8") as list_file:
+            for chunk_file in chunk_files:
+                list_file.write(f"file '{_concat_escape(str(chunk_file))}'\n")
 
-        export_params = speed_export_params(speed)
-        if export_params:
+        filters = atempo_filters(speed)
+        if filters:
             speed_path = output_path.with_name(f"{output_path.stem}_{speed:g}{output_path.suffix}")
-            combined.export(str(speed_path), format=config.AUDIO_FORMAT,
-                            bitrate=config.AUDIO_BITRATE, parameters=export_params)
+            cmd = [
+                "ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", str(concat_list),
+                "-filter_complex",
+                f"[0:a]split=2[base][spd];[spd]{filters}[spdout]",
+                "-map", "[base]", "-b:a", config.AUDIO_BITRATE, str(output_path),
+                "-map", "[spdout]", "-b:a", config.AUDIO_BITRATE, str(speed_path),
+            ]
+        else:
+            speed_path = None
+            cmd = [
+                "ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", str(concat_list),
+                "-b:a", config.AUDIO_BITRATE, str(output_path),
+            ]
+
+        proc = subprocess.run(cmd, capture_output=True, text=True)
+        if proc.returncode != 0:
+            logger.error("ffmpeg failed: %s", proc.stderr[-2000:])
+            return False
+
+        logger.info("Audiobook saved: %s (%d/%d chunks)", output_path, len(chunk_files), total_chunks)
+        print(f"[INFO] Saved audiobook: {output_path.name} ({len(chunk_files)}/{total_chunks} chunks)")
+
+        if filters:
             logger.info("Saved speed-adjusted audiobook (%gx): %s", speed, speed_path)
             print(f"[INFO] Saved speed-adjusted audiobook: {speed_path.name} ({speed:g}x)")
 
         if missing_chunks:
             print(f"[WARNING] Missing chunks: {missing_chunks}")
+
         return True
 
+    except FileNotFoundError:
+        logger.error("ffmpeg not found on PATH (install ffmpeg and try again)")
+        return False
     except Exception as exc:
         logger.error("Failed to combine chunks: %s", exc)
         logger.error(traceback.format_exc())
         return False
+    finally:
+        try:
+            concat_list.unlink(missing_ok=True)
+        except OSError:
+            pass
 
 
 def cleanup_chunks() -> None:
     """Remove temporary chunk files from the scratch folder."""
     try:
         chunk_count = 0
-        for chunk_file in config.CHUNKS_FOLDER.glob("chunk_*.wav"):
+        for chunk_file in config.CHUNKS_FOLDER.glob("chunk_*"):
             try:
-                chunk_file.unlink()
-                chunk_count += 1
+                if chunk_file.is_file():
+                    chunk_file.unlink()
+                    chunk_count += 1
             except Exception as exc:
                 logger.warning("Failed to delete %s: %s", chunk_file, exc)
 
diff --git a/converter/converter.py b/converter/converter.py
index eea84ed..65efb68 100644
--- a/converter/converter.py
+++ b/converter/converter.py
@@ -6,7 +6,7 @@ import time
 import traceback
 from datetime import datetime
 from pathlib import Path
-from typing import Optional
+from typing import Dict, Optional
 
 from . import audio, chunking, config, extractors
 from .tts import QwenTTSClient
@@ -60,17 +60,17 @@ class AudiobookConverter:
         """Validate configuration settings."""
         if self.voice_mode == "voice_clone":
             if not self.voice_clone_ref_audio:
-                print("[ERROR] Configuration Error!")
-                print("Voice Clone mode requires a reference audio file.")
-                print("Use --voice-sample <path> to specify the reference audio.")
-                sys.exit(1)
+                raise ValueError(
+                    "Voice Clone mode requires a reference audio file. "
+                    "Use --voice-sample <path> to specify it."
+                )
 
             if not Path(self.voice_clone_ref_audio).exists():
-                print("[ERROR] Configuration Error!")
-                print(f"Reference audio file not found: {self.voice_clone_ref_audio}")
-                sys.exit(1)
+                raise ValueError(
+                    f"Reference audio file not found: {self.voice_clone_ref_audio}"
+                )
 
-    def convert_book(self, file_path: Path) -> bool:
+    def convert_book(self, file_path: Path, output_name: Optional[str] = None) -> bool:
         """Convert a single book to an audiobook."""
         logger.info("Converting: %s", file_path.name)
         start_time = time.time()
@@ -136,7 +136,6 @@ class AudiobookConverter:
 
             if successful_chunks == 0:
                 logger.error("No chunks were successfully processed")
-                audio.cleanup_chunks()  # Cleanup even on failure
                 return False
 
             if successful_chunks < total_chunks:
@@ -144,7 +143,7 @@ class AudiobookConverter:
                                successful_chunks, total_chunks)
 
             # Combine chunks (only the successful ones)
-            output_path = config.AUDIOBOOKS_FOLDER / f"{file_path.stem}.{config.AUDIO_FORMAT}"
+            output_path = config.AUDIOBOOKS_FOLDER / f"{output_name or file_path.stem}.{config.AUDIO_FORMAT}"
             success = audio.combine_chunks(total_chunks, output_path, results, speed=self.speed)
 
             if success:
@@ -156,19 +155,18 @@ class AudiobookConverter:
             else:
                 logger.error("Failed to combine chunks into final audiobook")
 
-            # Always cleanup, even on failure
-            audio.cleanup_chunks()
             return success
 
         except Exception as exc:
             logger.error("Conversion failed: %s", exc)
             logger.error(traceback.format_exc())
-            # Cleanup on exception
-            audio.cleanup_chunks()
             return False
+        finally:
+            # Always cleanup, even on failure or interrupt
+            audio.cleanup_chunks()
 
-    def run(self) -> None:
-        """Main conversion process."""
+    def run(self) -> bool:
+        """Main conversion process. Returns True if all books converted."""
         api_url = config.VOICE_CLONE_API_URL if self.voice_mode == "voice_clone" else config.QWEN_API_URL
 
         print("=" * 70)
@@ -209,15 +207,23 @@ class AudiobookConverter:
                 encoding="utf-8",
             )
             print(f"[INFO] Created sample file: {sample_file}")
-            return
+            return True
 
         print(f"[INFO] Found {len(book_files)} books to convert")
 
+        # Avoid output collisions when two books share a stem (e.g. dune.txt + dune.epub).
+        stem_counts: Dict[str, int] = {}
+        for book_file in book_files:
+            stem_counts[book_file.stem] = stem_counts.get(book_file.stem, 0) + 1
+
         # Convert each book
         results = {}
         for book_file in book_files:
+            output_name = book_file.stem
+            if stem_counts[book_file.stem] > 1:
+                output_name = f"{book_file.stem}_{book_file.suffix.lstrip('.')}"
             try:
-                success = self.convert_book(book_file)
+                success = self.convert_book(book_file, output_name=output_name)
                 results[book_file.name] = success
             except KeyboardInterrupt:
                 print("\n[WARNING] Conversion interrupted by user")
@@ -242,3 +248,5 @@ class AudiobookConverter:
 
         if successful > 0:
             print(f"\n[INFO] Audiobooks saved to: {config.AUDIOBOOKS_FOLDER}/")
+
+        return total > 0 and successful == total
diff --git a/converter/extractors.py b/converter/extractors.py
index e49b48a..e6e27ad 100644
--- a/converter/extractors.py
+++ b/converter/extractors.py
@@ -1,5 +1,6 @@
 """Text extraction from book files (TXT, PDF, EPUB) and text/HTML cleaning."""
 
+import codecs
 import logging
 import re
 import zipfile
@@ -53,10 +54,8 @@ def clean_html(html_content: str) -> str:
             soup = BeautifulSoup(html_content, "html.parser")
             for tag in soup(["script", "style"]):
                 tag.decompose()
-            text = soup.get_text()
-            lines = (line.strip() for line in text.splitlines())
-            chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
-            return " ".join(chunk for chunk in chunks if chunk)
+            text = soup.get_text(separator=" ")
+            return re.sub(r"\s+", " ", text).strip()
         except Exception as exc:
             logger.debug("BeautifulSoup cleaning failed, falling back to regex: %s", exc)
 
@@ -115,11 +114,17 @@ def _extract_epub_ebooklib(file_path: Path) -> str:
     return "\n\n".join(text_parts)
 
 
+def _natural_key(name: str):
+    """Sort key that orders numeric runs numerically (chapter2 before chapter10)."""
+    return [int(part) if part.isdigit() else part.lower()
+            for part in re.split(r"(\d+)", name)]
+
+
 def _extract_epub_zipfile(file_path: Path) -> str:
     """Extract by parsing HTML members of the EPUB zip directly."""
     text_parts = []
     with zipfile.ZipFile(file_path, "r") as epub_zip:
-        for file_name in sorted(epub_zip.namelist()):
+        for file_name in sorted(epub_zip.namelist(), key=_natural_key):
             if file_name.lower().endswith((".html", ".xhtml", ".htm")):
                 try:
                     content = epub_zip.read(file_name).decode("utf-8", errors="ignore")
@@ -136,7 +141,7 @@ def _extract_epub_manual(file_path: Path) -> str:
     skipped_extensions = (".jpg", ".jpeg", ".png", ".gif", ".css", ".js")
     text_parts = []
     with zipfile.ZipFile(file_path, "r") as epub_zip:
-        for file_name in sorted(epub_zip.namelist()):
+        for file_name in sorted(epub_zip.namelist(), key=_natural_key):
             if file_name.lower().endswith(skipped_extensions):
                 continue
             try:
@@ -151,11 +156,31 @@ def _extract_epub_manual(file_path: Path) -> str:
 
 
 def _extract_txt(file_path: Path) -> str:
-    """Extract from TXT, trying common encodings (latin-1 is the catch-all)."""
-    for encoding in ("utf-8", "utf-16", "cp1252", "latin-1"):
+    """Extract from TXT, handling BOMs and common encodings (latin-1 is the catch-all).
+
+    UTF-16 files without a BOM are detected via NUL bytes; otherwise they would
+    silently decode as NUL-interleaved UTF-8 or cp1252/latin-1 garbage.
+    """
+    data = file_path.read_bytes()
+
+    if data.startswith((codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE)):
+        return clean_text(data.decode("utf-32"))
+    if data.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)):
+        return clean_text(data.decode("utf-16"))
+    if data.startswith(codecs.BOM_UTF8):
+        return clean_text(data.decode("utf-8-sig"))
+
+    # No BOM: UTF-16 without BOM is common on Windows; detect via NUL bytes.
+    sample = data[:4096]
+    even_nuls = sum(1 for i, b in enumerate(sample) if b == 0 and i % 2 == 0)
+    odd_nuls = sum(1 for i, b in enumerate(sample) if b == 0 and i % 2 == 1)
+    if even_nuls or odd_nuls:
+        encoding = "utf-16-be" if even_nuls > odd_nuls else "utf-16-le"
+        return clean_text(data.decode(encoding))
+
+    for encoding in ("utf-8", "cp1252", "latin-1"):
         try:
-            with open(file_path, "r", encoding=encoding) as f:
-                return clean_text(f.read())
+            return clean_text(data.decode(encoding))
         except UnicodeError:
             continue
     raise ValueError(f"Could not decode text file: {file_path}")
diff --git a/converter/tts.py b/converter/tts.py
index ac47ecb..c3a4b81 100644
--- a/converter/tts.py
+++ b/converter/tts.py
@@ -36,6 +36,7 @@ class QwenTTSClient:
     # ------------------------------------------------------------------
 
     def _connect(self) -> None:
+        api_url = config.VOICE_CLONE_API_URL if self.voice_mode == "voice_clone" else config.QWEN_API_URL
         try:
             if self.voice_mode == "voice_clone":
                 # Voice clone uses the Base-model demo, which is a separate server
@@ -47,17 +48,12 @@ class QwenTTSClient:
                 self._init_client(config.QWEN_API_URL, clone=False)
                 print("[OK] Connected to Qwen API")
         except Exception as exc:
-            api_url = config.VOICE_CLONE_API_URL if self.voice_mode == "voice_clone" else config.QWEN_API_URL
-            print("[ERROR] Qwen API initialization failed!")
-            print(f"API endpoint: {api_url}")
-            print("Make sure:")
-            print("1. Qwen Gradio server is running")
-            print("2. The server is accessible at the configured URL")
-            print("3. The endpoint URL is correct")
-            print("4. Your installed Qwen3-TTS version matches this converter's API expectations")
-            print("   (voice clone requires the Base-model demo: Qwen/Qwen3-TTS-12Hz-1.7B-Base)")
-            print(f"Error: {exc}")
-            sys.exit(1)
+            raise RuntimeError(
+                f"Qwen API initialization failed at {api_url}: {exc}. "
+                "Make sure the Qwen Gradio server is running and reachable, and that your "
+                "installed Qwen3-TTS version matches this converter's API expectations "
+                "(voice clone requires the Base-model demo: Qwen/Qwen3-TTS-12Hz-1.7B-Base)."
+            ) from exc
 
     def _resolve_reference_text(self) -> None:
         """Resolve the reference transcript: explicit text, then local
@@ -172,15 +168,20 @@ class QwenTTSClient:
             else:
                 raise ValueError(f"Unknown voice mode: {self.voice_mode}")
 
-            if not result or len(result) < 2:
-                raise RuntimeError("Qwen API returned invalid result")
+            if not isinstance(result, (tuple, list)) or not result:
+                raise RuntimeError("Qwen API returned an invalid result")
 
             audio_path = result[0]  # First element is the audio file path
-            if not audio_path or not Path(audio_path).exists():
+            if not isinstance(audio_path, (str, Path)) or not audio_path:
+                raise RuntimeError("Qwen API did not return an audio file path")
+
+            source = Path(audio_path)
+            if not source.exists():
                 raise RuntimeError(f"Generated audio file not found: {audio_path}")
 
-            output_path = config.CHUNKS_FOLDER / f"chunk_{chunk_num:04d}.wav"
-            shutil.copy2(audio_path, output_path)
+            suffix = source.suffix or ".wav"
+            output_path = config.CHUNKS_FOLDER / f"chunk_{chunk_num:04d}{suffix}"
+            shutil.copy2(source, output_path)
 
             logger.debug("Chunk %d generated successfully", chunk_num)
             return str(output_path)
diff --git a/requirements.txt b/requirements.txt
index 8559cb6..8a0ea5a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,12 +2,11 @@
 gradio_client>=0.7.0
 pypdf>=4.0.0
 ebooklib>=0.18
-pydub>=0.25.1
 
 # Optional dependencies
 beautifulsoup4>=4.11.0  # better HTML cleaning for EPUB
 faster-whisper>=1.0.0  # reference-audio transcription for voice cloning
 
 # Audio processing
-# Note: ffmpeg is required for pydub audio processing
+# Note: ffmpeg is required to concatenate and encode the final audiobook.
 # Install separately: https://ffmpeg.org/download.html