blob: f64931046d8e1946f1caaff1af5edbaa9b96af1a (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
|
"""Split extracted book text into TTS-sized chunks."""
import re
from typing import List
from . import config
def split_into_chunks(text: str, max_words: int = config.CHUNK_SIZE_WORDS) -> List[str]:
"""Split text into chunks of at most ``max_words`` words.
Splits on sentence boundaries. Sentences longer than the limit are split
further at clause punctuation (which is kept attached for TTS prosody).
A single sentence with no clause punctuation longer than the limit is
kept intact as one oversized chunk.
"""
if not text.strip():
return []
sentences = re.split(r"(?<=[.!?])\s+", text)
chunks = []
current_chunk = ""
current_words = 0
for sentence in sentences:
sentence_words = len(sentence.split())
if sentence_words > max_words:
if current_chunk:
chunks.append(current_chunk.strip())
current_chunk = ""
current_words = 0
# Split long sentences at clause boundaries, keeping punctuation.
parts = re.split(r"(?<=[,;:])\s*", sentence)
for part in parts:
part_words = len(part.split())
if current_words + part_words <= max_words:
current_chunk += part + " "
current_words += part_words
else:
if current_chunk:
chunks.append(current_chunk.strip())
current_chunk = part + " "
current_words = part_words
else:
if current_words + sentence_words <= max_words:
current_chunk += sentence + " "
current_words += sentence_words
else:
if current_chunk:
chunks.append(current_chunk.strip())
current_chunk = sentence + " "
current_words = sentence_words
if current_chunk.strip():
chunks.append(current_chunk.strip())
return [chunk for chunk in chunks if chunk.strip()]
|
