fix(converter): stream audio concat and harden error/encoding handling

1 files changed, 35 insertions, 10 deletions

diff --git a/converter/extractors.py b/converter/extractors.py
index e49b48a..e6e27ad 100644
--- a/converter/extractors.py
+++ b/converter/extractors.py
@@ -1,5 +1,6 @@
 """Text extraction from book files (TXT, PDF, EPUB) and text/HTML cleaning."""
 
+import codecs
 import logging
 import re
 import zipfile
@@ -53,10 +54,8 @@ def clean_html(html_content: str) -> str:
             soup = BeautifulSoup(html_content, "html.parser")
             for tag in soup(["script", "style"]):
                 tag.decompose()
-            text = soup.get_text()
-            lines = (line.strip() for line in text.splitlines())
-            chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
-            return " ".join(chunk for chunk in chunks if chunk)
+            text = soup.get_text(separator=" ")
+            return re.sub(r"\s+", " ", text).strip()
         except Exception as exc:
             logger.debug("BeautifulSoup cleaning failed, falling back to regex: %s", exc)
 
@@ -115,11 +114,17 @@ def _extract_epub_ebooklib(file_path: Path) -> str:
     return "\n\n".join(text_parts)
 
 
+def _natural_key(name: str):
+    """Sort key that orders numeric runs numerically (chapter2 before chapter10)."""
+    return [int(part) if part.isdigit() else part.lower()
+            for part in re.split(r"(\d+)", name)]
+
+
 def _extract_epub_zipfile(file_path: Path) -> str:
     """Extract by parsing HTML members of the EPUB zip directly."""
     text_parts = []
     with zipfile.ZipFile(file_path, "r") as epub_zip:
-        for file_name in sorted(epub_zip.namelist()):
+        for file_name in sorted(epub_zip.namelist(), key=_natural_key):
             if file_name.lower().endswith((".html", ".xhtml", ".htm")):
                 try:
                     content = epub_zip.read(file_name).decode("utf-8", errors="ignore")
@@ -136,7 +141,7 @@ def _extract_epub_manual(file_path: Path) -> str:
     skipped_extensions = (".jpg", ".jpeg", ".png", ".gif", ".css", ".js")
     text_parts = []
     with zipfile.ZipFile(file_path, "r") as epub_zip:
-        for file_name in sorted(epub_zip.namelist()):
+        for file_name in sorted(epub_zip.namelist(), key=_natural_key):
             if file_name.lower().endswith(skipped_extensions):
                 continue
             try:
@@ -151,11 +156,31 @@ def _extract_epub_manual(file_path: Path) -> str:
 
 
 def _extract_txt(file_path: Path) -> str:
-    """Extract from TXT, trying common encodings (latin-1 is the catch-all)."""
-    for encoding in ("utf-8", "utf-16", "cp1252", "latin-1"):
+    """Extract from TXT, handling BOMs and common encodings (latin-1 is the catch-all).
+
+    UTF-16 files without a BOM are detected via NUL bytes; otherwise they would
+    silently decode as NUL-interleaved UTF-8 or cp1252/latin-1 garbage.
+    """
+    data = file_path.read_bytes()
+
+    if data.startswith((codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE)):
+        return clean_text(data.decode("utf-32"))
+    if data.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)):
+        return clean_text(data.decode("utf-16"))
+    if data.startswith(codecs.BOM_UTF8):
+        return clean_text(data.decode("utf-8-sig"))
+
+    # No BOM: UTF-16 without BOM is common on Windows; detect via NUL bytes.
+    sample = data[:4096]
+    even_nuls = sum(1 for i, b in enumerate(sample) if b == 0 and i % 2 == 0)
+    odd_nuls = sum(1 for i, b in enumerate(sample) if b == 0 and i % 2 == 1)
+    if even_nuls or odd_nuls:
+        encoding = "utf-16-be" if even_nuls > odd_nuls else "utf-16-le"
+        return clean_text(data.decode(encoding))
+
+    for encoding in ("utf-8", "cp1252", "latin-1"):
         try:
-            with open(file_path, "r", encoding=encoding) as f:
-                return clean_text(f.read())
+            return clean_text(data.decode(encoding))
         except UnicodeError:
             continue
     raise ValueError(f"Could not decode text file: {file_path}")