"""Tests for file text extraction."""
import tempfile
import unittest
from pathlib import Path
from converter.extractors import extract_text
def _ebooklib_usable() -> bool:
"""True when ebooklib's EPUB reader imports (it needs a working lxml)."""
try:
from ebooklib import epub # noqa: F401
except Exception:
return False
return True
# The managed env can end up with compiled wheels that cannot load on this
# platform (e.g. glibc lxml under a musl interpreter) — the tool repairs or
# degrades at runtime, and these tests must degrade with it instead of
# failing. Relaunch audiobook.py once (or delete app/envs/tts) to rebuild.
requires_epub = unittest.skipUnless(
_ebooklib_usable(),
"ebooklib is unusable in this environment "
"(its compiled dependency failed to import)")
class TxtExtractionTests(unittest.TestCase):
def _extract(self, data: bytes) -> str:
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "book.txt"
path.write_bytes(data)
return extract_text(path)
def test_utf8(self):
self.assertEqual(self._extract("héllo wörld".encode("utf-8")), "héllo wörld")
def test_utf16_with_bom(self):
self.assertEqual(self._extract("héllo".encode("utf-16")), "héllo")
def test_cp1252(self):
self.assertEqual(self._extract("“quotes”".encode("cp1252")), "“quotes”")
def test_latin1_fallback(self):
# 0x81 is undefined in cp1252, forcing the latin-1 catch-all
self.assertEqual(self._extract(b"caf\x81"), "caf\x81")
def test_unsupported_format(self):
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "book.xyz"
path.write_bytes(b"data")
with self.assertRaises(ValueError):
extract_text(path)
def test_utf16_without_bom_detected(self):
self.assertEqual(self._extract("chapter one".encode("utf-16-le")),
"chapter one")
def test_lone_nul_does_not_flip_to_utf16(self):
# A single stray NUL byte in an otherwise-ASCII UTF-8 file must not
# switch the whole book to a UTF-16 decode (mojibake): the text
# comes back readable instead.
self.assertEqual(self._extract(b"hello world\x00rest"),
"hello world\x00rest")
def test_standalone_page_numbers_removed_but_years_kept(self):
from converter.extractors import clean_text
cleaned = clean_text("Chapter 1\n\n42\n\nIt was 1984.")
self.assertNotIn("42", cleaned)
self.assertIn("1984", cleaned)
kept = clean_text("It was the year\n\n1984\n\nwhen it began.")
self.assertIn("1984", kept)
class EpubZipfileFallbackTests(unittest.TestCase):
"""The no-ebooklib EPUB fallback: spine order, no TOC narration."""
@staticmethod
def _write_epub(path: Path):
import zipfile
container = (""
"
Contents
") zf.writestr("OEBPS/text/chapterA.xhtml", "Alpha text.
") zf.writestr("OEBPS/text/chapterB.xhtml", "Beta text.
") def test_spine_order_and_no_nav(self): from converter.extractors import _read_epub_zipfile with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "book.epub" self._write_epub(path) items = _read_epub_zipfile(path) titles = [title for title, _ in items] self.assertNotIn("nav", titles) # Spine order (B before A) beats filename sort (A before B). self.assertEqual(titles, ["chapterB", "chapterA"]) def test_unparsable_opf_falls_back_to_filename_order(self): import zipfile from converter.extractors import _read_epub_zipfile with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "book.epub" with zipfile.ZipFile(path, "w") as zf: zf.writestr("a.xhtml", "A
") zf.writestr("b.xhtml", "B
") items = _read_epub_zipfile(path) self.assertEqual([title for title, _ in items], ["a", "b"]) def _build_test_epub(path: Path, chapters=(("One", "First chapter text."), ("Two", "Second chapter text."))) -> None: from ebooklib import epub book = epub.EpubBook() book.set_identifier("test-id") book.set_title("Test Book") book.set_language("en") book.add_author("Test Author") items = [] for index, (title, text) in enumerate(chapters, 1): chapter = epub.EpubHtml(title=title, file_name=f"chap{index}.xhtml", lang="en") chapter.content = f"{text}
" book.add_item(chapter) items.append(chapter) book.toc = tuple(items) book.spine = ["nav", *items] book.add_item(epub.EpubNcx()) book.add_item(epub.EpubNav()) epub.write_epub(str(path), book) class EpubExtractionTests(unittest.TestCase): def setUp(self): try: import ebooklib # noqa: F401 except ImportError: self.skipTest("ebooklib not installed") @requires_epub def test_ebooklib_extraction(self): # Regression test: the ebooklib path used to silently return "" due to # isinstance(item, ebooklib.ITEM_DOCUMENT) (an int, not a class). from converter.extractors import _read_epub_ebooklib with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "book.epub" _build_test_epub(path) items = _read_epub_ebooklib(path) html = "\n".join(content for _, content in items) self.assertIn("First chapter text.", html) self.assertIn("Second chapter text.", html) @requires_epub def test_epub_extraction_follows_spine_order(self): with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "book.epub" _build_test_epub(path) text = extract_text(path) self.assertIn("First chapter text.", text) self.assertIn("Second chapter text.", text) self.assertLess(text.index("First chapter text."), text.index("Second chapter text.")) class ExtractSectionsTests(unittest.TestCase): def setUp(self): try: import ebooklib # noqa: F401 except ImportError: self.skipTest("ebooklib not installed") @requires_epub def test_epub_sections_split_on_chapters(self): from converter.extractors import extract_sections with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "book.epub" _build_test_epub(path) sections = extract_sections(path) self.assertEqual(len(sections), 2) self.assertEqual(sections[0].title, "One") self.assertEqual(sections[1].title, "Two") self.assertIn("First chapter text.", sections[0].text) self.assertIn("Second chapter text.", sections[1].text) def test_txt_is_single_section(self): from converter.extractors import extract_sections with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "book.txt" path.write_text("Hello world.", encoding="utf-8") sections = extract_sections(path) self.assertEqual(len(sections), 1) self.assertEqual(sections[0].title, "book") self.assertEqual(sections[0].text, "Hello world.") @requires_epub def test_single_chapter_epub_keeps_chapter_title(self): from converter.extractors import extract_sections with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "book.epub" _build_test_epub(path, chapters=(("Only", "Just one chapter."),)) sections = extract_sections(path) self.assertEqual(len(sections), 1) self.assertEqual(sections[0].title, "Only") self.assertIn("Just one chapter.", sections[0].text) class ExtractBookTests(unittest.TestCase): def test_txt_falls_back_to_stem_and_blank_author(self): from converter.extractors import extract_book with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "mybook.txt" path.write_text("Hello world.", encoding="utf-8") book = extract_book(path) self.assertEqual(book.title, "mybook") self.assertEqual(book.author, "") self.assertEqual(len(book.sections), 1) @requires_epub def test_epub_metadata_harvested(self): try: import ebooklib # noqa: F401 except ImportError: self.skipTest("ebooklib not installed") from converter.extractors import extract_book with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "book.epub" _build_test_epub(path) book = extract_book(path) self.assertEqual(book.title, "Test Book") self.assertEqual(book.author, "Test Author") self.assertEqual([s.title for s in book.sections], ["One", "Two"]) def test_pdf_metadata_harvested(self): from converter.extractors import extract_book try: from pypdf import PdfWriter except ImportError: self.skipTest("pypdf not installed") with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "book.pdf" writer = PdfWriter() writer.add_metadata({"/Title": "PDF Title", "/Author": "PDF Author"}) writer.add_blank_page(width=612, height=792) with open(path, "wb") as handle: writer.write(handle) book = extract_book(path) self.assertEqual(book.title, "PDF Title") self.assertEqual(book.author, "PDF Author") self.assertEqual(len(book.sections), 1) def test_pdf_without_metadata_falls_back(self): from converter.extractors import extract_book try: from pypdf import PdfWriter except ImportError: self.skipTest("pypdf not installed") with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "plain.pdf" writer = PdfWriter() writer.add_blank_page(width=612, height=792) with open(path, "wb") as handle: writer.write(handle) book = extract_book(path) self.assertEqual(book.title, "plain") self.assertEqual(book.author, "") if __name__ == "__main__": unittest.main()