1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
|
"""Tests for text chunking."""
import unittest
from unittest.mock import patch
from converter import config
from converter.chunking import MAX_REQUEST_WORDS, split_into_chunks
class ChunkSizeDefaultTests(unittest.TestCase):
"""Guard the request-size settings: each API call is one model
generation, and the servers silently truncate audio past their caps
(~2.5 min faster backend, ~11 min Gradio demo), so both the default
chunk size and the hard ceiling must stay well inside that budget."""
def test_default_chunk_size_within_request_ceiling(self):
self.assertLessEqual(config.CHUNK_SIZE, MAX_REQUEST_WORDS)
def test_request_ceiling_within_single_generation_budget(self):
self.assertLessEqual(MAX_REQUEST_WORDS, 300)
def test_sizes_are_positive(self):
self.assertGreaterEqual(config.CHUNK_SIZE, 1)
self.assertGreaterEqual(MAX_REQUEST_WORDS, 1)
class RequestCeilingClampTests(unittest.TestCase):
def test_oversized_chunk_size_is_clamped_with_warning(self):
text = " ".join(f"word{i}" for i in range(30)) + "."
with patch("converter.chunking.MAX_REQUEST_WORDS", 10), \
self.assertLogs("converter.chunking", level="WARNING") as logs:
chunks = split_into_chunks(text, max_words=5000)
self.assertTrue(all(len(chunk.split()) <= 10 for chunk in chunks))
self.assertIn("clamped", " ".join(logs.output))
def test_default_ceiling_clamps_realistic_configuration(self):
sentences = " ".join(
f"S{i} " + " ".join(["word"] * 8) + "." for i in range(60))
chunks = split_into_chunks(sentences, max_words=5000)
self.assertGreater(len(chunks), 1)
self.assertTrue(all(len(chunk.split()) <= MAX_REQUEST_WORDS
for chunk in chunks))
class SplitIntoChunksTests(unittest.TestCase):
def test_empty_input(self):
self.assertEqual(split_into_chunks(""), [])
self.assertEqual(split_into_chunks(" \n "), [])
def test_short_text_single_chunk(self):
self.assertEqual(split_into_chunks("One short sentence."), ["One short sentence."])
def test_respects_word_limit_across_sentences(self):
# 10 sentences of 9 words each = 90 words total
sentences = [f"S{i} " + " ".join(["word"] * 8) + "." for i in range(10)]
chunks = split_into_chunks(" ".join(sentences), max_words=25)
self.assertGreater(len(chunks), 1)
self.assertTrue(all(len(c.split()) <= 25 for c in chunks))
self.assertEqual(sum(len(c.split()) for c in chunks), 90)
def test_long_sentence_split_keeps_punctuation(self):
# 10 clauses of 5 words each, joined by comma+space
sentence = ", ".join([" ".join(["w"] * 5) for _ in range(10)]) + "."
chunks = split_into_chunks(sentence, max_words=12)
self.assertGreater(len(chunks), 1)
self.assertTrue(all(len(c.split()) <= 12 for c in chunks))
self.assertIn(",", chunks[0]) # commas retained for TTS prosody
def test_clause_split_never_breaks_numbers(self):
# Regression: the clause split used to fire at every comma even
# without whitespace, mutating "1,000,000" into "1, 000, 000".
sentence = ("There were exactly 1,000,000 soldiers marching at 12:30, "
+ "and they kept marching onward " * 30) + "endlessly."
chunks = split_into_chunks(sentence, max_words=25)
self.assertGreater(len(chunks), 1)
joined = " ".join(chunks)
self.assertIn("1,000,000", joined)
self.assertIn("12:30", joined)
self.assertNotIn("1, 000", joined)
self.assertNotIn("000, 000", joined)
self.assertNotIn("12: 30", joined)
def test_clause_split_requires_whitespace_after_punctuation(self):
# Run-on clauses without spaces after commas have no clause split
# point, so the last-resort word-boundary split fires instead.
# Tokens themselves (and numbers like "1,000,000") stay intact.
sentence = ",".join([" ".join(["w"] * 5) for _ in range(10)]) + "."
chunks = split_into_chunks(sentence, max_words=12)
self.assertGreater(len(chunks), 1)
self.assertTrue(all(len(chunk.split()) <= 12 for chunk in chunks))
tokens = sentence.replace(",", " , ").split()
rejoined = " ".join(chunks).replace(",", " , ").split()
self.assertEqual(rejoined, tokens)
def test_single_oversized_sentence_is_word_split(self):
# A punctuation-free sentence longer than the limit is split at word
# boundaries: the request-size ceiling is a hard limit because the
# TTS servers silently truncate oversized generations.
sentence = " ".join(["word"] * 30) + "."
chunks = split_into_chunks(sentence, max_words=10)
self.assertGreater(len(chunks), 1)
self.assertTrue(all(len(chunk.split()) <= 10 for chunk in chunks))
self.assertEqual(sum(len(chunk.split()) for chunk in chunks), 30)
def test_word_split_never_breaks_number_tokens(self):
# Numbers and other punctuation-bearing tokens are single words and
# must never be broken apart by the last-resort word split.
sentence = ("There were exactly 1,000,000 soldiers marching at 12:30 "
"and " + "they kept marching onward " * 20) + "endlessly."
chunks = split_into_chunks(sentence, max_words=10)
self.assertGreater(len(chunks), 1)
joined = " ".join(chunks)
self.assertIn("1,000,000", joined)
self.assertIn("12:30", joined)
self.assertNotIn("1, 000", joined)
self.assertNotIn("12: 30", joined)
if __name__ == "__main__":
unittest.main()
|
