1 files changed, 79 insertions, 17 deletions

diff --git a/app/converter/extractors.py b/app/converter/extractors.py
index 8b01372..0e30ac1 100644
--- a/app/converter/extractors.py
+++ b/app/converter/extractors.py
@@ -2,11 +2,13 @@
 
 import codecs
 import logging
+import posixpath
 import re
 import zipfile
 from html import unescape
 from pathlib import Path
-from typing import List, NamedTuple, Optional
+from typing import Dict, List, NamedTuple, Optional
+from urllib.parse import unquote
 
 try:
     from bs4 import BeautifulSoup
@@ -216,8 +218,13 @@ def _epub_spine_order(epub_zip: zipfile.ZipFile, html_names: List[str]):
 
     Parses the package OPF (located via META-INF/container.xml, else the
     only *.opf member): manifest item id -> href, then the spine's
-    idrefs. Returns member paths limited to HTML_NAMES; nav documents
-    (``properties`` containing "nav") and non-HTML items are excluded.
+    idrefs. Hrefs are URL-encoded, may carry "./" or "../" segments and
+    XML entities, and are relative to the OPF's folder — they are
+    normalized before matching against the zip's member names, so valid
+    chapters are not silently dropped when their written form differs
+    from the archived path. Returns member paths limited to HTML_NAMES;
+    nav documents (``properties`` containing "nav") and non-HTML items
+    are excluded.
     """
     container = "META-INF/container.xml"
     opf_name = None
@@ -246,6 +253,25 @@ def _epub_spine_order(epub_zip: zipfile.ZipFile, html_names: List[str]):
     base = "/".join(opf_name.split("/")[:-1])
     item_tags = re.findall(r"<item\b[^>]*>", opf)
 
+    # Zip member names indexed under their normalized spellings, so an
+    # exact archive path can be found from any equivalent href form.
+    # (Percent-encoded and plain spellings both map to the same member.)
+    members: Dict[str, str] = {}
+    for name in html_names:
+        members.setdefault(posixpath.normpath(name), name)
+        members.setdefault(posixpath.normpath(unquote(name)), name)
+
+    def member_for_href(href: str) -> Optional[str]:
+        """The zip member an OPF manifest HREF points at (best effort)."""
+        path = unescape(href).strip()
+        path = path.split("#", 1)[0]
+        path = unquote(path).strip()
+        if not path:
+            return None
+        if base:
+            path = f"{base}/{path}"
+        return members.get(posixpath.normpath(path).lstrip("/"))
+
     def is_nav(item_tag: str) -> bool:
         properties = attr(item_tag, "properties") or ""
         return "nav" in properties.split()
@@ -262,10 +288,9 @@ def _epub_spine_order(epub_zip: zipfile.ZipFile, html_names: List[str]):
         href = attr(match, "href")
         if not href:
             continue
-        path = (f"{base}/{href}" if base else href)
-        path = re.sub(r"#.*$", "", path)
-        if path in html_names and path not in order:
-            order.append(path)
+        member = member_for_href(href)
+        if member and member not in order:
+            order.append(member)
     return order or None
 
 
@@ -287,45 +312,82 @@ def _read_epub_manual(file_path: Path):
 
 
 def clean_text(text: str) -> str:
-    """Normalize whitespace and strip standalone page numbers.
+    """Normalize whitespace, strip standalone page numbers, and keep
+    paragraph breaks.
 
     Page numbers are removed only when a short number (up to three digits)
     appears alone on its own line (before whitespace collapsing), so inline
     numbers like "42 years", "1,000" or "3.5" are preserved, as are
     four-digit standalone lines, which are usually years ("1984") or
     chapter numbers rather than page numbers.
+
+    Runs of blank lines survive as a single paragraph break (``\\n\\n``) so
+    the smart chunker can still end requests at paragraph boundaries and
+    reset its quotation state there; single line breaks (soft wraps) and
+    all other whitespace collapse to spaces.
     """
     if not text:
         return ""
     # Standalone page numbers (digits alone on a line) must go BEFORE the
-    # newline-collapsing step below.
+    # whitespace-collapsing steps below.
     text = re.sub(r"(?m)^\s*\d{1,3}\s*$", " ", text)
-    text = re.sub(r"\s+", " ", text)
+    # Normalize line breaks so the steps below see plain "\n".
+    text = re.sub(r"\r\n?", "\n", text)
+    # Any run of blank lines becomes exactly one paragraph break.
+    text = re.sub(r"\n[^\S\n]*\n(?:[^\S\n]*\n)*", "\n\n", text)
+    # Remaining single line breaks (soft wraps) become spaces.
+    text = re.sub(r"(?<!\n)\n(?!\n)", " ", text)
+    # Collapse every other whitespace run (including nbsp) to one space.
+    text = re.sub(r"[^\S\n]+", " ", text)
+    # Trim whatever sits against a paragraph break.
+    text = re.sub(r"[^\S\n]*\n\n[^\S\n]*", "\n\n", text)
     return text.strip()
 
 
+# Block-level elements structure the document into paragraphs and lines:
+# replacing them with a paragraph break keeps the chunker's paragraph
+# boundaries (and headings separated from prose). Inline elements must NOT
+# break words, so they are left for text extraction to unwrap without
+# spaces ("un<em>believ</em>able" stays one word). The lookahead keeps
+# lookalike tags (<parameter>, <preheat>) out of the match.
+_BLOCK_BREAK_RE = re.compile(
+    r"</?(?:address|article|aside|blockquote|body|br|caption|center|col"
+    r"|colgroup|dd|div|dl|dt|fieldset|figcaption|figure|footer|form"
+    r"|h[1-6]|head|header|hr|html|legend|li|main|nav|ol|option|p|pre"
+    r"|section|summary|table|tbody|td|tfoot|th|thead|tr|ul)(?=[\s/>])"
+    r"[^>]*>",
+    re.IGNORECASE)
+
+
 def clean_html(html_content: str) -> str:
-    """Strip markup, scripts and styles from HTML content."""
+    """Strip markup, scripts and styles from HTML content.
+
+    Block-level tags become paragraph breaks (so the chunker sees the
+    document's paragraphs); inline tags vanish without adding spaces, so
+    no artificial word boundaries are injected.
+    """
     if not html_content:
         return ""
 
+    html_content = _BLOCK_BREAK_RE.sub("\n\n", html_content)
+
     if BS4_AVAILABLE:
         try:
             soup = BeautifulSoup(html_content, "html.parser")
             for tag in soup(["script", "style"]):
                 tag.decompose()
-            text = soup.get_text(separator=" ")
-            return re.sub(r"\s+", " ", text).strip()
+            return clean_text(soup.get_text(separator=""))
         except Exception as exc:
             logger.debug("BeautifulSoup cleaning failed, falling back to regex: %s", exc)
 
-    # Fallback regex cleaning
+    # Fallback regex cleaning. Block tags are already paragraph breaks;
+    # every other tag (inline) disappears without adding a space, so no
+    # artificial word boundary is injected.
     html_content = re.sub(r"<style[^>]*>.*?</style>", "", html_content, flags=re.DOTALL | re.IGNORECASE)
     html_content = re.sub(r"<script[^>]*>.*?</script>", "", html_content, flags=re.DOTALL | re.IGNORECASE)
-    html_content = re.sub(r"<[^>]+>", " ", html_content)
+    html_content = re.sub(r"<[^>]+>", "", html_content)
     html_content = unescape(html_content)
-    html_content = re.sub(r"\s+", " ", html_content)
-    return html_content.strip()
+    return clean_text(html_content)
 
 
 def _natural_key(name: str):