aboutsummaryrefslogtreecommitdiff
path: root/app/tests
diff options
context:
space:
mode:
Diffstat (limited to 'app/tests')
-rw-r--r--app/tests/test_chunking.py129
-rw-r--r--app/tests/test_converter.py18
-rw-r--r--app/tests/test_hub.py53
3 files changed, 197 insertions, 3 deletions
diff --git a/app/tests/test_chunking.py b/app/tests/test_chunking.py
index 2904e40..bcae6b2 100644
--- a/app/tests/test_chunking.py
+++ b/app/tests/test_chunking.py
@@ -113,6 +113,135 @@ class SplitIntoChunksTests(unittest.TestCase):
self.assertNotIn("1, 000", joined)
self.assertNotIn("12: 30", joined)
+ def test_smart_strategy_distributes_sentences(self):
+ # The smart strategy leaves whole sentences intact: a chunk
+ # crossing the target (85% of the limit) ends at the next clean
+ # boundary early; legacy keeps packing until the limit.
+ sentences = " ".join(
+ " ".join(f"Sw{j}" for j in range(n)) + "."
+ for n in (9, 8, 3, 9, 8, 3))
+ smart = split_into_chunks(sentences, max_words=20, smart=True)
+ legacy = split_into_chunks(sentences, max_words=20, smart=False)
+ self.assertEqual([len(c.split()) for c in smart], [17, 20, 3])
+ self.assertEqual([len(c.split()) for c in legacy], [20, 20])
+ self.assertEqual(self._content(smart),
+ self._content(sentences.split(" ")))
+
+ def test_dialogue_tag_stays_with_its_quote(self):
+ # A closing quote followed by a lowercase attribution ("she
+ # said.") is one sentence unit: the chunk never separates them.
+ text = "“Come here!” she said. “Are you coming?” He did not answer."
+ chunks = split_into_chunks(text, max_words=10, smart=True)
+ self.assertIn("“Come here!” she said.", chunks[0])
+ self._assert_bounded(chunks, 10)
+ self._assert_content(text, chunks)
+
+ def test_abbreviations_and_initials_do_not_split(self):
+ text = ("Dr. Smith met J. K. R. at the U.S. border with No. 3 "
+ "shortly. He continued onward.")
+ chunks = split_into_chunks(text, max_words=50, smart=True)
+ self.assertEqual(len(chunks), 1)
+ self.assertIn("Dr. Smith", chunks[0])
+ self.assertIn("J. K. R.", chunks[0])
+ self.assertIn("U.S.", chunks[0])
+ self._assert_content(text, chunks)
+
+ def test_decimals_are_never_boundaries(self):
+ text = "Pi is 3.14159 and the toll was 1,000,000 miles. Next one."
+ chunks = split_into_chunks(text, max_words=12, smart=True)
+ joined = " ".join(chunks)
+ self.assertIn("3.14159", joined)
+ self.assertIn("1,000,000", joined)
+ self._assert_bounded(chunks, 12)
+ self._assert_content(text, chunks)
+
+ def test_unclosed_quote_still_respects_the_limit(self):
+ # A quote with no closing mark must not grow chunks or silence
+ # later boundaries; units carry the open state and stay bounded.
+ text = "“No closing quote follows. " + "filler words here. " * 20
+ chunks = split_into_chunks(text, max_words=10, smart=True)
+ self._assert_bounded(chunks, 10)
+ self._assert_content(text, chunks)
+
+ def test_paragraph_boundary_resets_an_open_quote(self):
+ # Blank lines reset the quotation state: an unclosed quote in
+ # one paragraph cannot trap later paragraphs inside the quote.
+ text = ("“Still open. filler words inside the quote here.\n\n"
+ "New paragraph. Words outside any quote now, plenty.")
+ chunks = split_into_chunks(text, max_words=10, smart=True)
+ self._assert_bounded(chunks, 10)
+ self._assert_content(text, chunks)
+ second = [c for c in chunks if "New paragraph." in c]
+ self.assertTrue(second)
+ # The paragraph's text starts a fresh chunk: the open-quote
+ # prefix did not swallow it.
+ self.assertTrue(
+ any(chunk.startswith("New paragraph.") or
+ "“Still open." not in chunk
+ for chunk in chunks))
+
+ def test_quote_retreat_keeps_short_quotes_whole(self):
+ # When the next sentence would overflow, the break retreats to
+ # the last boundary that ended outside a quotation — before the
+ # quote — so a short quotation lands in one request.
+ text = ("Plain opening words here to fill. "
+ "“A short quote. With two sentences. Inside.” "
+ "More trailing prose follows this quote. And more.")
+ chunks = split_into_chunks(text, max_words=14, smart=True)
+ self._assert_bounded(chunks, 14)
+ self._assert_content(text, chunks)
+ quote_chunks = [c for c in chunks if "short quote" in c]
+ self.assertTrue(quote_chunks)
+ self.assertIn("Inside.", quote_chunks[0])
+
+ def test_oversized_single_tokens_respect_char_cap(self):
+ # Degenerate text (punctuation-free run): bounded by the hard
+ # character cap, with every character preserved.
+ text = "a" * 5000
+ chunks = split_into_chunks(text, max_words=250, smart=True)
+ self.assertGreater(len(chunks), 1)
+ self.assertTrue(all(len(c) <= 250 * 8 for c in chunks))
+ self.assertEqual("".join(chunks), text)
+
+ def test_non_spaced_script_stays_bounded(self):
+ # CJK text: sentence terminators break units without
+ # whitespace, and each ideograph counts as a word.
+ text = "。".join(["字" * 15 for _ in range(30)]) + "。"
+ chunks = split_into_chunks(text, max_words=40, smart=True)
+ self._assert_bounded(chunks, 40)
+ self._assert_content(text, chunks)
+ joined = " ".join(chunks).split()
+ self.assertTrue(all(len(j) <= 33 for j in joined))
+
+ def test_legacy_and_smart_matching_smart_negation(self):
+ # smart=False preserves the legacy splitting exactly.
+ sentences = " ".join(
+ " ".join(f"Sw{j}" for j in range(9)) + "." for _ in range(10))
+ with patch.object(config, "CHUNK_SIZE", 25):
+ legacy_default = split_into_chunks(sentences, smart=False)
+ self.assertEqual([len(c.split()) for c in legacy_default],
+ [18] * 5)
+ self.assertEqual(self._content(legacy_default),
+ self._content(sentences.split()))
+
+ # -- helpers ---------------------------------------------------------
+
+ @staticmethod
+ def _content(chunks_or_words):
+ """Non-whitespace content of the pieces (or of a token list)."""
+ pieces = (chunks_or_words if isinstance(chunks_or_words, str)
+ else " ".join(chunks_or_words))
+ return "".join(pieces.split())
+
+ def _assert_content(self, text, chunks):
+ self.assertEqual(self._content(chunks), self._content(text))
+
+ def _assert_bounded(self, chunks, max_words):
+ from converter.chunking import _count_words
+ for chunk in chunks:
+ self.assertLessEqual(_count_words(chunk), max_words,
+ chunk[:40])
+
if __name__ == "__main__":
unittest.main()
diff --git a/app/tests/test_converter.py b/app/tests/test_converter.py
index 85ae6ca..9084898 100644
--- a/app/tests/test_converter.py
+++ b/app/tests/test_converter.py
@@ -585,6 +585,24 @@ class ChunkClampPromptTests(unittest.TestCase):
with patch.object(config, "CHUNK_SIZE", 80):
self.assertFalse(chunk_clamp_needed(self._entry()))
+ def test_per_run_clamp_caps_the_chapter_chunks(self):
+ # The popup's clamp caps the chapter chunks themselves, so each
+ # progress unit and debug dump spans one generation request.
+ converter = AudiobookConverter.__new__(AudiobookConverter)
+ converter.chunk_size = 4
+ with patch.object(config, "CHUNK_SIZE", 250):
+ chunks = converter._chapter_chunks(" ".join(f"Sw{i} ." for i in range(12)))
+ self.assertGreater(len(chunks), 1)
+ self.assertTrue(all(len(c.split()) <= 4 for c in chunks))
+
+ def test_chapter_chunks_follow_chunk_size_without_a_clamp(self):
+ converter = AudiobookConverter.__new__(AudiobookConverter)
+ self.assertIsNone(converter.chunk_size)
+ with patch.object(config, "CHUNK_SIZE", 8):
+ chunks = converter._chapter_chunks(" ".join(f"Sw{i} ." for i in range(20)))
+ self.assertGreater(len(chunks), 1)
+ self.assertTrue(all(len(c.split()) <= 8 for c in chunks))
+
def test_prompt_answers(self):
entry = self._entry()
with patch("builtins.input", return_value=""):
diff --git a/app/tests/test_hub.py b/app/tests/test_hub.py
index 90ed2d4..cbcdc8a 100644
--- a/app/tests/test_hub.py
+++ b/app/tests/test_hub.py
@@ -3529,7 +3529,8 @@ class SettingsTests(unittest.TestCase):
# Keys _apply_settings persists; every test that triggers a real or
# fake config write restores these afterwards.
_SETTING_KEYS = ("AUDIO_FORMAT", "AUDIO_BITRATE", "LANGUAGE",
- "CHUNK_SIZE", "INPUT_DIR", "OUTPUT_DIR",
+ "CHUNK_SIZE", "SMART_CHUNKING", "INPUT_DIR",
+ "OUTPUT_DIR",
"CLONE_WAV_DIR",
"SPEED", "DEBUG", "STOP_SERVER_AND_EXIT",
"AUDIOCPP_UNLOAD_MODELS",
@@ -3604,6 +3605,7 @@ class SettingsTests(unittest.TestCase):
original_folders[1])
values = {"audio_format": "ogg", "audio_bitrate": " 192k ",
"language": "en", "chunk_size": "300",
+ "smart_chunking": True,
"input_dir": " /books ", "output_dir": "/audiobooks",
"clone_wav_dir": " /refs/wavs ",
"speed": "1.5", "debug": True,
@@ -3626,6 +3628,7 @@ class SettingsTests(unittest.TestCase):
"AUDIO_BITRATE": "192k",
"LANGUAGE": "English",
"CHUNK_SIZE": 300,
+ "SMART_CHUNKING": True,
"INPUT_DIR": "/books",
"OUTPUT_DIR": "/audiobooks",
"CLONE_WAV_DIR": "/refs/wavs",
@@ -3652,6 +3655,7 @@ class SettingsTests(unittest.TestCase):
self.assertEqual(hub.config.AUDIO_BITRATE, "192k")
self.assertEqual(hub.config.LANGUAGE, "English")
self.assertEqual(hub.config.CHUNK_SIZE, 300)
+ self.assertEqual(hub.config.SMART_CHUNKING, True)
self.assertEqual(hub.config.INPUT_DIR, "/books")
self.assertEqual(hub.config.OUTPUT_DIR, "/audiobooks")
self.assertEqual(hub.config.CLONE_WAV_DIR, "/refs/wavs")
@@ -3671,6 +3675,7 @@ class SettingsTests(unittest.TestCase):
self._snapshot_settings()
base = {"audio_format": "m4b", "audio_bitrate": "128k",
"language": "English", "chunk_size": "250",
+ "smart_chunking": True,
"input_dir": "./input", "output_dir": "./output",
"clone_wav_dir": "./voices",
"speed": "1.0", "debug": False,
@@ -3701,9 +3706,46 @@ class SettingsTests(unittest.TestCase):
"audiocpp_remote_url": "not a url"})
mk_update.assert_not_called()
+ def test_smart_chunking_toggle_persists(self):
+ self._snapshot_settings()
+
+ def fake_update(key, value, config_path=None):
+ setattr(hub.config, key, value)
+ return True
+
+ values = {"audio_format": "m4b", "audio_bitrate": "128k",
+ "language": "English", "chunk_size": "250",
+ "smart_chunking": False,
+ "input_dir": "./input", "output_dir": "./output",
+ "clone_wav_dir": "./voices",
+ "speed": "1.0", "debug": False,
+ "stop_and_exit": True, "unload_models": True,
+ "qwen_port": "7860",
+ "faster_port": "8000", "audiocpp_port": "8080",
+ "sglomni_port": "8100"}
+ with patch.object(hub.common, "update_config_value", fake_update), \
+ patch.object(hub, "_sync_audiocpp_server_port"):
+ hub._apply_settings(values)
+ self.assertEqual(hub.config.SMART_CHUNKING, False)
+
+ values["smart_chunking"] = True
+ with patch.object(hub.common, "update_config_value", fake_update), \
+ patch.object(hub, "_sync_audiocpp_server_port"):
+ hub._apply_settings(values)
+ self.assertEqual(hub.config.SMART_CHUNKING, True)
+
+ def test_smart_chunking_field_defaults_on_with_help(self):
+ fields = hub._settings_fields()
+ field = next(f for f in fields if f["key"] == "smart_chunking")
+ self.assertEqual(field["kind"], "bool")
+ self.assertEqual(field["value"], hub.config.SMART_CHUNKING)
+ self.assertTrue(field["help"])
+ self.assertTrue(field["label"], "Smart chunking")
+
def test_field_validators(self):
self.assertIsNone(hub._validate_bitrate("128k"))
self.assertIsNotNone(hub._validate_bitrate(" "))
+ self.assertIsNotNone(hub._validate_bitrate(" "))
self.assertIsNone(hub._validate_language("English"))
self.assertIsNone(hub._validate_language("en"))
self.assertIsNotNone(hub._validate_language("Klingon"))
@@ -3794,6 +3836,7 @@ class SettingsTests(unittest.TestCase):
captured["fields"] = fields
return {"audio_format": "ogg", "audio_bitrate": "192k",
"language": "English", "chunk_size": "300",
+ "smart_chunking": True,
"input_dir": "/books", "output_dir": "/audiobooks",
"clone_wav_dir": "/refs/wavs",
"speed": "1.0", "debug": False,
@@ -3817,7 +3860,8 @@ class SettingsTests(unittest.TestCase):
hub._Hub(None).screen_settings()
self.assertEqual([f["key"] for f in captured["fields"]],
["audio_format", "audio_bitrate", "language",
- "chunk_size", "input_dir", "output_dir",
+ "chunk_size", "smart_chunking", "input_dir",
+ "output_dir",
"clone_wav_dir",
"speed", "debug", "stop_and_exit",
"unload_models",
@@ -3867,6 +3911,7 @@ class SettingsTests(unittest.TestCase):
"audio_bitrate": "192k",
"language": "English",
"chunk_size": "300",
+ "smart_chunking": True,
"input_dir": "/books",
"output_dir": "/audiobooks",
"clone_wav_dir": "/refs/wavs",
@@ -4020,7 +4065,8 @@ class SettingsTests(unittest.TestCase):
original = {name: getattr(hub.config, name) for name in
("AUDIO_FORMAT", "AUDIO_BITRATE", "LANGUAGE",
- "CHUNK_SIZE", "INPUT_DIR", "OUTPUT_DIR",
+ "CHUNK_SIZE", "SMART_CHUNKING", "INPUT_DIR",
+ "OUTPUT_DIR",
"CLONE_WAV_DIR",
"SPEED", "DEBUG", "STOP_SERVER_AND_EXIT",
"AUDIOCPP_UNLOAD_MODELS",
@@ -4042,6 +4088,7 @@ class SettingsTests(unittest.TestCase):
'LANGUAGE = "English"\n'
"\n"
"CHUNK_SIZE = 250\n"
+ "SMART_CHUNKING = True\n"
'INPUT_DIR = "./input"\n'
'OUTPUT_DIR = "./output"\n'
'CLONE_WAV_DIR = "./voices"\n'
generated by cgit v1.2.3 (git 2.25.1) at 2026-09-10 11:33:56 +0000