1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
|
"""Tests for text and HTML cleaning."""
import unittest
from converter.extractors import clean_html, clean_text
class CleanTextTests(unittest.TestCase):
def test_empty_input(self):
self.assertEqual(clean_text(""), "")
self.assertEqual(clean_text(None), "")
def test_collapses_whitespace_inside_paragraphs(self):
self.assertEqual(clean_text("a b \t c"), "a b c")
def test_paragraph_breaks_survive(self):
# Blank lines are the smart chunker's paragraph boundaries (and
# reset its quotation state), so they survive cleaning while a
# run of blank lines collapses to a single break.
self.assertEqual(clean_text("a\n\n b \t c"), "a\n\nb c")
self.assertEqual(clean_text("a\n\n\n\nb"), "a\n\nb")
self.assertEqual(clean_text("a\n \nb"), "a\n\nb")
self.assertEqual(clean_text("line one\nline two"), "line one line two")
self.assertEqual(clean_text("a \n\n b"), "a\n\nb")
def test_paragraph_breaks_not_glued_to_text(self):
self.assertEqual(
clean_text("End of chapter.\n\n\n New chapter. \n\nStarts here."),
"End of chapter.\n\nNew chapter.\n\nStarts here.",
)
def test_preserves_inline_numbers(self):
self.assertEqual(clean_text("He was 42 years old."), "He was 42 years old.")
def test_preserves_grouped_and_decimal_numbers(self):
self.assertEqual(
clean_text("Over 1,000 pages and 3.5 stars."),
"Over 1,000 pages and 3.5 stars.",
)
def test_removes_standalone_page_numbers(self):
# The page number's own line becomes a paragraph break (a safe
# chunk boundary), not a glued sentence.
self.assertEqual(
clean_text("End of page.\n7\nNext page text."),
"End of page.\n\nNext page text.",
)
def test_page_number_removal_leaves_paragraph_break(self):
result = clean_text("Chapter one\n\n12\n\nChapter two")
self.assertEqual(result, "Chapter one\n\nChapter two")
self.assertNotIn(" ", result)
class CleanHtmlTests(unittest.TestCase):
def test_strips_tags(self):
self.assertEqual(clean_html("<p>Hello <b>world</b></p>"), "Hello world")
def test_removes_script_and_style(self):
html = "<style>.x{color:red}</style><p>Text</p><script>var a=1;</script>"
self.assertEqual(clean_html(html), "Text")
def test_unescapes_entities(self):
self.assertEqual(clean_html("Tom & Jerry"), "Tom & Jerry")
def test_empty(self):
self.assertEqual(clean_html(""), "")
self.assertEqual(clean_html(None), "")
def test_inline_markup_never_splits_words(self):
# Inline tags must not inject spaces mid-word (they become chunk
# boundaries and corrupt pronunciation).
self.assertEqual(
clean_html("<p>He was un<em>believ</em>able and didn<i>'</i>t stop.</p>"),
"He was unbelievable and didn't stop.",
)
def test_block_tags_become_paragraph_breaks(self):
self.assertEqual(
clean_html("<p>First para.</p><p>Second para.</p><h2>Head</h2>"),
"First para.\n\nSecond para.\n\nHead",
)
def test_div_sections_keep_paragraph_boundaries(self):
html = ('<div>“An unfinished quotation.</div>'
'<div>A new paragraph outside the quotation.</div>')
self.assertEqual(
clean_html(html),
"“An unfinished quotation.\n\nA new paragraph outside the quotation.",
)
def test_table_cells_do_not_glue(self):
self.assertEqual(
clean_html("<table><tr><td>A</td><td>B</td></tr>"
"<tr><td>C</td><td>D</td></tr></table>"),
"A\n\nB\n\nC\n\nD",
)
def test_regex_fallback_matches_bs4_behavior(self):
html = "<p>un<em>believ</em>able</p><div>After a block.</div>"
self.assertEqual(clean_html(html), "unbelievable\n\nAfter a block.")
if __name__ == "__main__":
unittest.main()
|
