fix: smart chunking, extraction, and process-safety issuesHEADmain

1 files changed, 56 insertions, 5 deletions

diff --git a/app/tests/test_cleaning.py b/app/tests/test_cleaning.py
index 41f4ed7..40f2d53 100644
--- a/app/tests/test_cleaning.py
+++ b/app/tests/test_cleaning.py
@@ -10,8 +10,24 @@ class CleanTextTests(unittest.TestCase):
         self.assertEqual(clean_text(""), "")
         self.assertEqual(clean_text(None), "")
 
-    def test_collapses_whitespace(self):
-        self.assertEqual(clean_text("a\n\n   b \t c"), "a b c")
+    def test_collapses_whitespace_inside_paragraphs(self):
+        self.assertEqual(clean_text("a   b \t c"), "a b c")
+
+    def test_paragraph_breaks_survive(self):
+        # Blank lines are the smart chunker's paragraph boundaries (and
+        # reset its quotation state), so they survive cleaning while a
+        # run of blank lines collapses to a single break.
+        self.assertEqual(clean_text("a\n\n   b \t c"), "a\n\nb c")
+        self.assertEqual(clean_text("a\n\n\n\nb"), "a\n\nb")
+        self.assertEqual(clean_text("a\n  \nb"), "a\n\nb")
+        self.assertEqual(clean_text("line one\nline two"), "line one line two")
+        self.assertEqual(clean_text("a \n\n b"), "a\n\nb")
+
+    def test_paragraph_breaks_not_glued_to_text(self):
+        self.assertEqual(
+            clean_text("End of chapter.\n\n\n  New chapter.  \n\nStarts here."),
+            "End of chapter.\n\nNew chapter.\n\nStarts here.",
+        )
 
     def test_preserves_inline_numbers(self):
         self.assertEqual(clean_text("He was 42 years old."), "He was 42 years old.")
@@ -23,14 +39,16 @@ class CleanTextTests(unittest.TestCase):
         )
 
     def test_removes_standalone_page_numbers(self):
+        # The page number's own line becomes a paragraph break (a safe
+        # chunk boundary), not a glued sentence.
         self.assertEqual(
             clean_text("End of page.\n7\nNext page text."),
-            "End of page. Next page text.",
+            "End of page.\n\nNext page text.",
         )
 
-    def test_page_number_removal_leaves_single_spacing(self):
+    def test_page_number_removal_leaves_paragraph_break(self):
         result = clean_text("Chapter one\n\n12\n\nChapter two")
-        self.assertEqual(result, "Chapter one Chapter two")
+        self.assertEqual(result, "Chapter one\n\nChapter two")
         self.assertNotIn("  ", result)
 
 
@@ -49,6 +67,39 @@ class CleanHtmlTests(unittest.TestCase):
         self.assertEqual(clean_html(""), "")
         self.assertEqual(clean_html(None), "")
 
+    def test_inline_markup_never_splits_words(self):
+        # Inline tags must not inject spaces mid-word (they become chunk
+        # boundaries and corrupt pronunciation).
+        self.assertEqual(
+            clean_html("<p>He was un<em>believ</em>able and didn<i>'</i>t stop.</p>"),
+            "He was unbelievable and didn't stop.",
+        )
+
+    def test_block_tags_become_paragraph_breaks(self):
+        self.assertEqual(
+            clean_html("<p>First para.</p><p>Second para.</p><h2>Head</h2>"),
+            "First para.\n\nSecond para.\n\nHead",
+        )
+
+    def test_div_sections_keep_paragraph_boundaries(self):
+        html = ('<div>“An unfinished quotation.</div>'
+                '<div>A new paragraph outside the quotation.</div>')
+        self.assertEqual(
+            clean_html(html),
+            "“An unfinished quotation.\n\nA new paragraph outside the quotation.",
+        )
+
+    def test_table_cells_do_not_glue(self):
+        self.assertEqual(
+            clean_html("<table><tr><td>A</td><td>B</td></tr>"
+                       "<tr><td>C</td><td>D</td></tr></table>"),
+            "A\n\nB\n\nC\n\nD",
+        )
+
+    def test_regex_fallback_matches_bs4_behavior(self):
+        html = "<p>un<em>believ</em>able</p><div>After a block.</div>"
+        self.assertEqual(clean_html(html), "unbelievable\n\nAfter a block.")
+
 
 if __name__ == "__main__":
     unittest.main()