slop refactor/dedup

1 files changed, 93 insertions, 24 deletions

diff --git a/app/converter/extractors.py b/app/converter/extractors.py
index f05d451..8b01372 100644
--- a/app/converter/extractors.py
+++ b/app/converter/extractors.py
@@ -6,7 +6,7 @@ import re
 import zipfile
 from html import unescape
 from pathlib import Path
-from typing import List, NamedTuple
+from typing import List, NamedTuple, Optional
 
 try:
     from bs4 import BeautifulSoup
@@ -39,8 +39,6 @@ def extract_text(file_path: Path) -> str:
         return _extract_txt(file_path)
     if extension == ".pdf":
         return _extract_pdf(file_path)
-    if extension == ".epub":
-        return extract_epub(file_path)
     raise ValueError(f"Unsupported file format: {extension}")
 
 
@@ -186,19 +184,91 @@ def _read_epub_ebooklib(file_path: Path):
 
 
 def _read_epub_zipfile(file_path: Path):
-    """Read EPUB HTML members as (title, html) pairs, ordered by filename."""
+    """Read EPUB HTML members as (title, html) pairs, in spine order.
+
+    Fallback for EPUBs ebooklib cannot read. The package's OPF describes
+    the reading order (its ``<spine>`` itemrefs reference manifest items
+    by id), so documents are emitted in that order; the manifest's
+    ``properties="nav"`` item (the table of contents) and any document
+    outside the spine are skipped so the TOC is never narrated as a
+    chapter. EPUBs without a parsable OPF fall back to natural filename
+    order over every HTML member.
+    """
     items = []
     with zipfile.ZipFile(file_path, "r") as epub_zip:
-        for file_name in sorted(epub_zip.namelist(), key=_natural_key):
-            if file_name.lower().endswith((".html", ".xhtml", ".htm")):
-                try:
-                    content = epub_zip.read(file_name).decode("utf-8", errors="ignore")
-                    items.append((Path(file_name).stem, content))
-                except Exception as exc:
-                    logger.debug("Skipping EPUB member %r: %s", file_name, exc)
+        names = epub_zip.namelist()
+        html_names = [name for name in names
+                      if name.lower().endswith((".html", ".xhtml", ".htm"))]
+        order = _epub_spine_order(epub_zip, html_names)
+        if order is None:
+            order = sorted(html_names, key=_natural_key)
+        for file_name in order:
+            try:
+                content = epub_zip.read(file_name).decode("utf-8", errors="ignore")
+                items.append((Path(file_name).stem, content))
+            except Exception as exc:
+                logger.debug("Skipping EPUB member %r: %s", file_name, exc)
     return items
 
 
+def _epub_spine_order(epub_zip: zipfile.ZipFile, html_names: List[str]):
+    """The EPUB's HTML members in spine order, or None when unparsable.
+
+    Parses the package OPF (located via META-INF/container.xml, else the
+    only *.opf member): manifest item id -> href, then the spine's
+    idrefs. Returns member paths limited to HTML_NAMES; nav documents
+    (``properties`` containing "nav") and non-HTML items are excluded.
+    """
+    container = "META-INF/container.xml"
+    opf_name = None
+    try:
+        rootfile = epub_zip.read(container).decode("utf-8", errors="ignore")
+        match = re.search(r"full-path\s*=\s*[\"']([^\"']+)[\"']", rootfile)
+        if match and match.group(1) in epub_zip.namelist():
+            opf_name = match.group(1)
+    except (KeyError, OSError):
+        pass
+    if opf_name is None:
+        opf_candidates = [name for name in epub_zip.namelist()
+                          if name.lower().endswith(".opf")]
+        if len(opf_candidates) != 1:
+            return None
+        opf_name = opf_candidates[0]
+    try:
+        opf = epub_zip.read(opf_name).decode("utf-8", errors="ignore")
+    except (KeyError, OSError):
+        return None
+
+    def attr(tag: str, name: str) -> Optional[str]:
+        match = re.search(rf"\b{name}\s*=\s*[\"']([^\"']*)[\"']", tag)
+        return match.group(1) if match else None
+
+    base = "/".join(opf_name.split("/")[:-1])
+    item_tags = re.findall(r"<item\b[^>]*>", opf)
+
+    def is_nav(item_tag: str) -> bool:
+        properties = attr(item_tag, "properties") or ""
+        return "nav" in properties.split()
+
+    order: List[str] = []
+    for ref_tag in re.findall(r"<itemref\b[^>]*>", opf):
+        idref = attr(ref_tag, "idref")
+        if not idref:
+            continue
+        match = next((item_tag for item_tag in item_tags
+                      if attr(item_tag, "id") == idref), None)
+        if match is None or is_nav(match):
+            continue
+        href = attr(match, "href")
+        if not href:
+            continue
+        path = (f"{base}/{href}" if base else href)
+        path = re.sub(r"#.*$", "", path)
+        if path in html_names and path not in order:
+            order.append(path)
+    return order or None
+
+
 def _read_epub_manual(file_path: Path):
     """Last-resort read of any markup-looking EPUB member."""
     skipped_extensions = (".jpg", ".jpeg", ".png", ".gif", ".css", ".js")
@@ -219,15 +289,17 @@ def _read_epub_manual(file_path: Path):
 def clean_text(text: str) -> str:
     """Normalize whitespace and strip standalone page numbers.
 
-    Page numbers are removed only when they appear as a short number alone on
-    its own line (before whitespace collapsing), so inline numbers like
-    "42 years", "1,000" or "3.5" are preserved.
+    Page numbers are removed only when a short number (up to three digits)
+    appears alone on its own line (before whitespace collapsing), so inline
+    numbers like "42 years", "1,000" or "3.5" are preserved, as are
+    four-digit standalone lines, which are usually years ("1984") or
+    chapter numbers rather than page numbers.
     """
     if not text:
         return ""
     # Standalone page numbers (digits alone on a line) must go BEFORE the
     # newline-collapsing step below.
-    text = re.sub(r"(?m)^\s*\d{1,4}\s*$", " ", text)
+    text = re.sub(r"(?m)^\s*\d{1,3}\s*$", " ", text)
     text = re.sub(r"\s+", " ", text)
     return text.strip()
 
@@ -256,14 +328,6 @@ def clean_html(html_content: str) -> str:
     return html_content.strip()
 
 
-def extract_epub(file_path: Path) -> str:
-    """Extract the book's text from EPUB, trying several methods in order."""
-    chapters = _extract_epub_chapters(file_path)
-    if not chapters:
-        raise RuntimeError("All EPUB extraction methods failed")
-    return "\n\n".join(section.text for section in chapters)
-
-
 def _natural_key(name: str):
     """Sort key that orders numeric runs numerically (chapter2 before chapter10)."""
     return [int(part) if part.isdigit() else part.lower()
@@ -286,10 +350,15 @@ def _extract_txt(file_path: Path) -> str:
         return clean_text(data.decode("utf-8-sig"))
 
     # No BOM: UTF-16 without BOM is common on Windows; detect via NUL bytes.
+    # A real UTF-16 file of ASCII-range text has a NUL at every other byte
+    # position, so require a substantial NUL share before committing to
+    # UTF-16: a lone stray NUL in a UTF-8/cp1252 file must not flip the
+    # whole book into mojibake (the decode ladder below handles that).
     sample = data[:4096]
     even_nuls = sum(1 for i, b in enumerate(sample) if b == 0 and i % 2 == 0)
     odd_nuls = sum(1 for i, b in enumerate(sample) if b == 0 and i % 2 == 1)
-    if even_nuls or odd_nuls:
+    threshold = max(len(sample) // 4, 1)
+    if even_nuls >= threshold or odd_nuls >= threshold:
         encoding = "utf-16-be" if even_nuls > odd_nuls else "utf-16-le"
         return clean_text(data.decode(encoding))