diff options
Diffstat (limited to 'epubgen.py')
| -rwxr-xr-x | epubgen.py | 492 |
1 files changed, 492 insertions, 0 deletions
diff --git a/epubgen.py b/epubgen.py new file mode 100755 index 0000000..8599b35 --- /dev/null +++ b/epubgen.py @@ -0,0 +1,492 @@ +#!/usr/bin/env python3 +"""epubgen - generate spec-perfect EPUB 3.3 files for testing readers and validators. + +Two modes: + normal - a clean, readable EPUB that uses the full feature set + (cover, title page, TOC, landmarks, chapters, index, colophon, + EPUB2 NCX fallback, EPUB3 nav, CSS, metadata refinements). + torture - a technically valid EPUB with deliberately hostile syntax + (bidi overrides, combining chars, numeric char refs, deep nested + markup, unusual-but-valid xml:lang, weird filenames, ...). + +Every XML file is re-parsed with xml.etree before zipping, so the output is +guaranteed well-formed. Zero dependencies beyond the Python standard library. + +Usage: + python3 epubgen.py [options] +""" +import argparse +import html +import os +import random +import re +import struct +import sys +import uuid +import xml.etree.ElementTree as ET +import zipfile +import zlib +from datetime import datetime, timezone +from urllib.parse import quote + +VERSION = "1.0" +HERE = os.path.dirname(os.path.abspath(__file__)) +DEFAULT_FILE = os.path.join(HERE, "paragraphs.txt") + +EMBEDDED = [ + "The wind carried the smell of rain across the valley, and the last light of the afternoon lingered on the hills.", + "She opened the ledger and began to read, though the words had been written in another hand long ago.", + "It was not the first time the old clock had stopped, but it was the first time anyone had noticed.", +] + +NS = 'xmlns="http://www.w3.org/1999/xhtml" xmlns:epub="http://www.idpf.org/2007/ops"' + +# Hostile-but-valid Unicode (all fine in XML 1.0 / UTF-8) +COMBINING = "\u0300\u0301\u0308\u030a\u20dd\u20e3" +ZW = "\u200b\u200c\u200d\u200e\u200f" +SPACES = "\u00a0\u2009\u200a\u202f\u3000" +BIDI = "\u202a\u202b\u202d\u202e" +PRIV = "\ue000\ue100\uf8ff" +CJK = "\u6c49\u5b57\u6d4b\u8bd5\u30c6\u30b9\u30c8" +EMOJI = "\U0001f600\U0001f389\U0001f680\U0001f480\U0001f518\U0001f30d" + + +def esc(s): + """XML-escape text (& < > and quotes; escaping '>' makes ']]>' impossible).""" + return html.escape(s, quote=True) + + +def xhtml(title, body, body_type=None, link_css=True): + """Wrap body content in a spec-compliant XHTML5 document.""" + head = f'<head><meta charset="utf-8"/><title>{esc(title)}</title>' + if link_css: + head += '<link rel="stylesheet" type="text/css" href="style.css"/>' + head += '</head>' + btype = f' epub:type="{body_type}"' if body_type else '' + return ('<?xml version="1.0" encoding="UTF-8"?>\n' + f'<html {NS} lang="en" xml:lang="en">{head}<body{btype}>{body}</body></html>') + + +def first_word(texts, rng): + t = rng.choice(texts) + m = re.search(r"[A-Za-z0-9]+", t) + return m.group(0) if m else "word" + + +# --------------------------------------------------------------------------- +# torture helpers +# --------------------------------------------------------------------------- + +def torture_text(s, rng): + """Inject combining marks, zero-width/bidi chars, exotic spaces, CJK/emoji/private-use.""" + out = [] + for ch in s: + out.append(ch) + if ch.isalnum() and rng.random() < 0.15: + out.append(rng.choice(COMBINING)) + r = rng.random() + if r < 0.04: + out.append(rng.choice(ZW)) + elif r < 0.05: + out.append(rng.choice(BIDI)) + elif r < 0.055: + out.append(rng.choice(PRIV)) + elif r < 0.06: + out.append(rng.choice(CJK + EMOJI)) + s = "".join(out) + s = "".join(rng.choice(SPACES) if c == " " and rng.random() < 0.4 else c for c in s) + if len(s) > 40 and rng.random() < 0.3: + i = rng.randrange(0, len(s) - 20) + j = rng.randrange(i + 10, min(len(s), i + 60)) + s = s[:i] + "\u202e" + s[i:j] + "\u202c" + s[j:] + return s + + +def torture_entities(s, rng): + """Replace some chars with numeric character references (post-escaping, skips existing entities).""" + out, i, n = [], 0, len(s) + while i < n: + if s[i] == "&": + j = s.find(";", i) + if j != -1: + out.append(s[i:j + 1]) + i = j + 1 + continue + ch = s[i] + if (ch.isalnum() or ch == " ") and rng.random() < 0.12: + out.append(f"&#x{ord(ch):04x};") + else: + out.append(ch) + i += 1 + return "".join(out) + + +def torture_para(rng, p, kid): + kind = rng.random() + if kind < 0.18: + body = '<pre xml:space="preserve">\n\t' + p + '\n</pre>' + elif kind < 0.30: + depth = rng.randint(3, 9) + inner = p + for d in range(depth): + inner = f'<span class="t{d}" data-x="{d}">{inner}</span>' + body = f'<p id="p{kid}" dir="auto" xml:lang="{rng.choice(["en-US", "de-AT-1901", "fr", "en"])}">{inner}</p>' + elif kind < 0.36: + body = f'<blockquote><p id="p{kid}">{p}</p></blockquote>' + else: + body = f'<p id="p{kid}">{p}</p>' + return body + + +def torture_extra(rng): + if rng.random() < 0.5: + cells = "".join(f"<tr><th>k{k}</th><td>{esc(str(rng.random()))}</td></tr>" for k in range(3)) + return f"<table><caption>{esc('data')}</caption>{cells}</table>" + safe = "AZaz019()[]{}_" + return "<!--" + "".join(rng.choice(safe) for _ in range(rng.randint(10, 30))) + "-->" + + +# --------------------------------------------------------------------------- +# document builders +# --------------------------------------------------------------------------- + +def container_xml(): + return ('<?xml version="1.0" encoding="UTF-8"?>\n' + '<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">' + '<rootfiles><rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/>' + '</rootfiles></container>') + + +def cover_xhtml(): + body = ('<section epub:type="cover">' + '<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" ' + 'version="1.1" width="100%" height="100%" viewBox="0 0 600 800">' + '<image width="600" height="800" xlink:href="cover.png"/></svg>' + '</section>') + return xhtml("Cover", body, link_css=False) + + +def title_xhtml(title, author): + body = ('<section epub:type="titlepage">' + f'<h1 class="book-title">{esc(title)}</h1>' + f'<p class="byline">{esc(author)}</p>' + '<p class="imprint">Test Press</p>' + '</section>') + return xhtml(title, body) + + +def chapter_xhtml(mode, rng, idx, total, npar, ctitle, texts): + ps = [] + for k in range(1, npar + 1): + raw = rng.choice(texts) + if mode == "torture": + raw = torture_text(raw, rng) + p = esc(raw) + if mode == "torture": + p = torture_entities(p, rng) + ps.append(torture_para(rng, p, k)) + else: + ps.append(f'<p id="p{k}">{p}</p>') + body = "\n".join(ps) + if mode == "torture": + body += "\n" + torture_extra(rng) + section = (f'<section epub:type="chapter"><h1>{esc(ctitle)}</h1>{body}' + f'<footer>{esc("Chapter %d of %d" % (idx, total))}</footer></section>') + return xhtml(ctitle, section, body_type="bodymatter chapter") + + +def nav_xhtml(first_href, chapters, back_hrefs): + def li(href, label): + return f'<li><a href="{href}">{esc(label)}</a></li>' + + fl = "\n".join(li(h, l) for h, l in + [("cover.xhtml", "Cover"), ("title.xhtml", "Title page"), ("nav.xhtml", "Contents")]) + cl = "\n".join(f'<li><a href="{h}">{esc(t)}</a></li>' for _, h, t in chapters) + bl = "\n".join(li(h, l) for h, l in back_hrefs) + toc = ('<nav epub:type="toc" id="toc">' + f'<h1>{esc("Table of Contents")}</h1><ol>' + f'<li><span>Front matter</span><ol>{fl}</ol></li>' + f'<li><span>Chapters</span><ol>{cl}</ol></li>' + f'<li><span>Back matter</span><ol>{bl}</ol></li>' + '</ol></nav>') + lms = "".join(f'<li><a href="{h}" epub:type="{t}">{esc(l)}</a></li>' + for h, t, l in [("cover.xhtml", "cover", "Cover"), + ("title.xhtml", "titlepage", "Title page"), + ("nav.xhtml", "toc", "Table of contents"), + (first_href, "bodymatter", "Start"), + ("index.xhtml", "index", "Index")]) + landmarks = f'<nav epub:type="landmarks" id="landmarks"><h2>{esc("Guide")}</h2><ol>{lms}</ol></nav>' + return xhtml("Contents", toc + landmarks, body_type="frontmatter") + + +def ncx_xml(mode, rng, title, chapters, uid): + po = [0] + + def point(pid, href, label, children=""): + po[0] += 1 + return (f'<navPoint id="{pid}" playOrder="{po[0]}"><navLabel><text>{esc(label)}</text></navLabel>' + f'<content src="{href}"/>{children}</navPoint>') + + cover = point("npx", "cover.xhtml", "Cover") + cps = "\n".join(point(f"np{i}", h, t) for i, (_, h, t) in enumerate(chapters)) + if mode == "torture": + cps = point("npWrap", chapters[0][1], rng.choice(["Chapters", "Contents", "Edição"]), cps) + navmap = f"<navMap>{cover}{cps}</navMap>" + head = (f'<head><meta name="dtb:uid" content="{uid}"/>' + '<meta name="dtb:depth" content="1"/>' + '<meta name="dtb:totalPageCount" content="0"/>' + '<meta name="dtb:maxPageNumber" content="0"/></head>') + doctype = '' + if mode == "normal": + doctype = '<!DOCTYPE ncx PUBLIC "-//NISO//DTD ncx 2005-1//EN" "http://www.dtd-helper.com/ncx/ncx-2005-1.dtd">\n' + return ('<?xml version="1.0" encoding="UTF-8"?>\n' + doctype + + '<ncx xmlns="http://www.daisy.org/z3986/2005/ncx/" version="2005-1" xml:lang="en">' + f'{head}<docTitle><text>{esc(title)}</text></docTitle>{navmap}</ncx>') + + +def opf_xml(mode, rng, title, author, uid, now, chapters, ncx): + meta = [ + f'<dc:identifier id="pub-id">{uid}</dc:identifier>', + f'<dc:title id="t1">{esc(title)}</dc:title>', + '<meta refines="#t1" property="title-type">main</meta>', + f'<dc:creator id="c1">{esc(author)}</dc:creator>', + '<meta refines="#c1" property="role" scheme="marc:relators">aut</meta>', + '<dc:language>en</dc:language>', + '<meta property="dcterms:conformsTo">EPUB 3.3</meta>', + '<dc:publisher>Test Press</dc:publisher>', + f'<dc:date>{now}</dc:date>', + f'<meta property="dcterms:modified">{now}</meta>', + '<dc:rights>Public domain test text.</dc:rights>', + ] + if mode == "torture": + meta += [ + f'<dc:creator id="c2">{esc(rng.choice(["A. Nonymous", "Écrivain", "テスト作者"]))}</dc:creator>', + '<meta refines="#c2" property="role" scheme="marc:relators">edt</meta>', + '<meta property="schema:accessMode">textual</meta>', + '<meta property="schema:accessibilityFeature">readingOrder</meta>', + '<dc:subject>testing, books, epub, torture</dc:subject>', + f'<dc:description>{esc(torture_text("A book for testing parsers.", rng))}</dc:description>', + ] + metadata = f'<metadata xmlns:dc="http://purl.org/dc/elements/1.1/">{"".join(meta)}</metadata>' + + manifest = [ + '<item id="cover" href="cover.png" media-type="image/png" properties="cover-image"/>', + '<item id="cover-page" href="cover.xhtml" media-type="application/xhtml+xml" properties="svg"/>', + '<item id="title-page" href="title.xhtml" media-type="application/xhtml+xml"/>', + '<item id="nav" href="nav.xhtml" media-type="application/xhtml+xml" properties="nav"/>', + '<item id="css" href="style.css" media-type="text/css"/>', + '<item id="index" href="index.xhtml" media-type="application/xhtml+xml"/>', + '<item id="colophon" href="colophon.xhtml" media-type="application/xhtml+xml"/>', + ] + if ncx: + manifest.append('<item id="ncx" href="toc.ncx" media-type="application/x-dtbncx+xml"/>') + for i, (_, h, _) in enumerate(chapters): + manifest.append(f'<item id="chap{i}" href="{h}" media-type="application/xhtml+xml"/>') + manifest_xml = f'<manifest>{"".join(manifest)}</manifest>' + + spine = ['<itemref idref="cover-page"/>', '<itemref idref="title-page"/>', '<itemref idref="nav"/>'] + for i in range(len(chapters)): + spine.append(f'<itemref idref="chap{i}" linear="yes"/>') + spine.append('<itemref idref="index" linear="yes"/>') + spine.append('<itemref idref="colophon" linear="no"/>') + spine_xml = f'<spine>{"".join(spine)}</spine>' + + pkg_lang = "zh-Hant-TW-u-ca-gregory" if mode == "torture" else "en" + pkg_dir = "rtl" if mode == "torture" else "ltr" + return ('<?xml version="1.0" encoding="UTF-8"?>\n' + '<package xmlns="http://www.idpf.org/2007/opf" version="3.0" unique-identifier="pub-id" ' + f'xml:lang="{pkg_lang}" dir="{pkg_dir}">{metadata}{manifest_xml}{spine_xml}</package>') + + +def index_xhtml(mode, rng, chapters, words): + entries = [] + for i, (_, h, _) in enumerate(chapters): + entries.append(f'<li><a href="{h}#p1">{esc(words[i])}</a><span>{i + 1}</span></li>') + if mode == "torture": + entries.append(f'<li>{esc("index of indexes")} - see <a href="{chapters[0][1]}#p1">{esc(words[0])}</a></li>') + body = f'<section epub:type="index"><h1>{esc("Index")}</h1><ul class="index">{"\n".join(entries)}</ul></section>' + return xhtml("Index", body, body_type="backmatter") + + +def colophon_xhtml(mode, uid, now): + text = ("This ebook was generated by epubgen for testing EPUB readers and validators. " + f"Identifier: {uid}. Generated: {now}. Mode: {mode}.") + body = f'<section epub:type="colophon"><h1>{esc("Colophon")}</h1><p>{esc(text)}</p></section>' + return xhtml("Colophon", body, body_type="backmatter") + + +def css(mode): + base = ( + 'body{margin:5% auto;max-width:34em;font-family:Georgia,"Times New Roman",serif;line-height:1.5}\n' + 'h1{font-size:1.8em;line-height:1.2}\n' + 'h2{font-size:1.3em}\n' + 'p{text-indent:1.2em;margin:0.4em 0}\n' + 'p:first-of-type{text-indent:0}\n' + 'blockquote{font-style:italic;border-left:3px solid #999;margin:1em 2em;padding-left:1em}\n' + 'footer{font-size:0.85em;color:#555;text-align:center;margin-top:2em}\n' + '.index li{list-style:none;margin:0.2em 0}\n' + '.index li span{color:#999;margin-left:0.5em}\n' + 'pre{white-space:pre-wrap;font-family:monospace}\n' + ) + if mode == "torture": + base = ('@import url("http://example.invalid/legacy.css");\n' + base + + 'p:first-of-type{text-indent:0!important}\n' + 'pre[lang]{white-space:pre!important}\n' + '@media screen and (min-width:1px){h1{-webkit-hyphens:auto;-ms-hyphens:auto;hyphens:auto}}\n' + 'h1::before{content:"\\00a7\\2002"}\n' + 'table{border-collapse:collapse}\n' + 'td,th{border:1px solid #ccc;padding:0.2em 0.5em}\n' + '@page{margin:5%}\n') + return base + + +def make_cover_png(w=600, h=800): + rows = [] + for y in range(h): + row = bytearray(b"\x00") + for x in range(w): + row += bytes((min(255, int(90 + 165 * x / w)), + min(255, int(60 + 120 * y / h)), + min(255, int(150 + 60 * (x + y) / (w + h))))) + rows.append(bytes(row)) + raw = b"".join(rows) + + def chunk(tag, data): + return (struct.pack(">I", len(data)) + tag + data + + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF)) + + ihdr = struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + + chunk(b"IDAT", zlib.compress(raw, 9)) + chunk(b"IEND", b"")) + + +# --------------------------------------------------------------------------- +# orchestration +# --------------------------------------------------------------------------- + +def check_xml(label, data): + try: + ET.fromstring(data) + except Exception as ex: + print("ERROR: %s is not well-formed XML: %s" % (label, ex), file=sys.stderr) + sys.exit(1) + + +def build_zip(out, files): + os.makedirs(os.path.dirname(os.path.abspath(out)) or ".", exist_ok=True) + with zipfile.ZipFile(out, "w") as zf: + for name, data in files: + zi = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + zi.compress_type = zipfile.ZIP_STORED if name == "mimetype" else zipfile.ZIP_DEFLATED + zi.external_attr = 0o644 << 16 + zf.writestr(zi, data) + with zipfile.ZipFile(out) as zf: + names = zf.namelist() + if (not names or names[0] != "mimetype" or + zf.getinfo("mimetype").compress_type != zipfile.ZIP_STORED or + zf.read("mimetype") != b"application/epub+zip"): + print("ERROR: mimetype entry violates the OCF spec", file=sys.stderr) + sys.exit(1) + for i in zf.infolist(): + print(" %-42s %8d bytes %s" % ( + i.filename, i.file_size, + "STORED" if i.compress_type == zipfile.ZIP_STORED else "DEFLATED")) + + +def load_texts(path): + for p in (path, DEFAULT_FILE): + if p and os.path.exists(p): + with open(p, encoding="utf-8") as f: + lines = [ln.strip() for ln in f if ln.strip()] + if lines: + return lines + return EMBEDDED + + +def generate(out, rng, texts, title, author, mode, nchap, npar, ncx, seed): + now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + uid = "urn:uuid:%s" % uuid.uuid4() + if mode == "torture": + title = torture_text(title, rng) + + chapters, words = [], [] + for i in range(1, nchap + 1): + fname = "chap %02d & tale.xhtml" % i if mode == "torture" else "chap%02d.xhtml" % i + href = quote(fname) + ctitle = "Chapter %d" % i if mode == "normal" else torture_text("Chapter %d" % i, rng) + chapters.append((fname, href, ctitle)) + words.append(first_word(texts, rng)) + + first_href = chapters[0][1] + back_hrefs = [("index.xhtml", "Index"), ("colophon.xhtml", "Colophon")] + + docs = { + "META-INF/container.xml": container_xml(), + "OEBPS/content.opf": opf_xml(mode, rng, title, author, uid, now, chapters, ncx), + "OEBPS/nav.xhtml": nav_xhtml(first_href, chapters, back_hrefs), + "OEBPS/style.css": css(mode), + "OEBPS/cover.xhtml": cover_xhtml(), + "OEBPS/title.xhtml": title_xhtml(title, author), + "OEBPS/index.xhtml": index_xhtml(mode, rng, chapters, words), + "OEBPS/colophon.xhtml": colophon_xhtml(mode, uid, now), + } + if ncx: + docs["OEBPS/toc.ncx"] = ncx_xml(mode, rng, title, chapters, uid) + for i, (fname, href, ctitle) in enumerate(chapters, 1): + docs["OEBPS/" + fname] = chapter_xhtml(mode, rng, i, nchap, npar, ctitle, texts) + + for label, data in docs.items(): + if label.endswith(".xml") or label.endswith((".opf", ".ncx", ".xhtml")): + check_xml(label, data) + + files = [("mimetype", b"application/epub+zip")] + files += [("META-INF/container.xml", docs["META-INF/container.xml"])] + files += [(name, data) for name, data in docs.items() if name.startswith("OEBPS/")] + files += [("OEBPS/cover.png", make_cover_png())] + + build_zip(out, files) + print("OK: %s (%d chapters x %d paragraphs, mode=%s, seed=%s)" % + (out, nchap, npar, mode, seed if seed is not None else "random")) + + +def resolve_out_paths(out, mode): + if mode != "both": + return [out or f"{mode}.epub"] + stem = out[:-5] if out and out.lower().endswith(".epub") else (out or "") + base = f"{stem}." if stem else "" + return [f"{base}normal.epub", f"{base}torture.epub"] + + +def main(): + ap = argparse.ArgumentParser( + description="Generate spec-perfect EPUB 3.3 files for testing readers and validators.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + ap.add_argument("-o", "--out", default=None, + help="output .epub path (single mode) or name prefix (both mode)") + ap.add_argument("-m", "--mode", choices=["normal", "torture", "both"], default="both", + help="normal = clean; torture = hostile-but-valid; both = one of each (default)") + ap.add_argument("-c", "--chapters", type=int, default=5, help="number of chapters") + ap.add_argument("-p", "--paragraphs", type=int, default=10, help="paragraphs per chapter") + ap.add_argument("-t", "--title", default="The Testing EPUB") + ap.add_argument("-a", "--author", default="Test Author") + ap.add_argument("--seed", type=int, default=None, help="RNG seed for reproducibility") + ap.add_argument("--texts", default=None, help="file with one paragraph per line") + ap.add_argument("--no-ncx", action="store_true", help="omit the EPUB2 NCX fallback") + ap.add_argument("-v", "--version", action="version", version="epubgen %s" % VERSION) + args = ap.parse_args() + + if args.chapters < 1 or args.paragraphs < 1: + print("ERROR: chapters and paragraphs must be >= 1", file=sys.stderr) + sys.exit(1) + + rng = random.Random(args.seed) + texts = load_texts(args.texts) + ncx = not args.no_ncx + modes = ["normal", "torture"] if args.mode == "both" else [args.mode] + for mode, path in zip(modes, resolve_out_paths(args.out, args.mode)): + generate(path, rng, texts, args.title, args.author, + mode, args.chapters, args.paragraphs, ncx, args.seed) + + +if __name__ == "__main__": + main() |
