aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorhistoria <historiavg@proton.me>2026-08-18 01:56:25 -0400
committerhistoria <historiavg@proton.me>2026-08-18 01:56:25 -0400
commitd72d274bd9895bdf97d31d56690b7df07fa74ad4 (patch)
tree85b001881055fa1f7087de84a162a8a5a880484a
parent68ee76514a98169a5e7a075648b70ab40417fbdf (diff)
downloadtts-audiobook-generator-d72d274bd9895bdf97d31d56690b7df07fa74ad4.tar.gz
fix: timing/punctuation edge cases, process chunks as uncompressed wavs
-rw-r--r--README.md27
-rwxr-xr-x[-rw-r--r--]audiobook_converter.py0
-rw-r--r--converter/audio.py188
-rw-r--r--converter/chunking.py11
-rw-r--r--converter/converter.py18
-rw-r--r--input/test.txt1
-rw-r--r--tests/test_audio.py132
-rw-r--r--tests/test_chunking.py25
8 files changed, 336 insertions, 66 deletions
diff --git a/README.md b/README.md
index 0f0d090..9c52213 100644
--- a/README.md
+++ b/README.md
@@ -24,12 +24,11 @@ The converter sends text extracted from your books to a locally running Qwen3-TT
## Installation
-```bash
-# Arch Linux
-sudo pacman -S conda ffmpeg
+Install ffmpeg and conda, e.g.
-# Debian, conda must be installed separately
-sudo apt-get install ffmpeg
+```bash
+sudo pacman -S conda ffmpeg #Arch Linux
+sudo apt-get install ffmpeg #Debian, conda must be installed separately
```
### Install Qwen3-TTS (Server)
@@ -96,17 +95,13 @@ Whisper (`faster_whisper` or `whisper`) is used automatically to transcribe the
### Options
-| Flag | Description |
-| --------------------------- | ----------------------------------------------------------------------------------------- |
-| `--speed <n>` | Playback speed, pitch-preserving (`1.0` = normal). A normal-speed copy is also output. |
-| `--format {mp3,m4b}` | Output format (default `mp3`). `m4b` uses AAC audio and is always a single file. |
-| `--single-file` | Merge all chapters into a single mp3 (default: one mp3 per chapter). |
-| `--voice-sample-text "..."` | Transcript of the reference audio (voice clone only). |
-| `--no-transcription` | Skip auto-transcription of the reference audio (voice clone only). |
-
-Books with chapters (e.g. EPUB) are converted to **one mp3 per chapter** by default, named `output/<Book>_01_<Chapter>.mp3`, `output/<Book>_02_<Chapter>.mp3`, etc.; use `--single-file` to merge them. `m4b` output is always a single file with chapter markers embedded so listeners can skip between chapters. TXT and PDF files have no chapter structure and always produce a single file.
-
-The `chunks/` folder is scratch space for the current book only — it is emptied before and after every conversion, so an interrupted run never affects the next one.
+| Flag | Description |
+| --------------------------- | ----------------------------------------------------------------------------------------------------- |
+| `--speed <n>` | Playback speed, pitch-preserving (`1.0` = normal). A normal-speed copy is also output. |
+| `--format {mp3,m4b}` | Output format (default `mp3`). `m4b` uses AAC audio and has built-in chapters. |
+| `--single-file` | mp3 only: Merge all chapters into a single mp3 (default: one mp3 per chapter). |
+| `--voice-sample-text "..."` | Override whisper auto-transcription with your own manual reference audio transcript. Not required. |
+| `--no-transcription` | Skip auto-transcription of the reference audio. Usually worse, but can give a different voice affect. |
## Running tests
diff --git a/audiobook_converter.py b/audiobook_converter.py
index 36bd5c2..36bd5c2 100644..100755
--- a/audiobook_converter.py
+++ b/audiobook_converter.py
diff --git a/converter/audio.py b/converter/audio.py
index 2f40777..9912004 100644
--- a/converter/audio.py
+++ b/converter/audio.py
@@ -51,9 +51,132 @@ def _encode_args(output_format: str) -> List[str]:
"""Return ffmpeg output codec/bitrate args for the requested container."""
if output_format == "m4b":
return ["-c:a", "aac", "-b:a", config.AUDIO_BITRATE]
+ if output_format == "wav":
+ # Lossless intermediate for per-chapter scratch audio; avoids
+ # generational loss when the final m4b re-encodes to AAC.
+ return ["-c:a", "pcm_s16le"]
return ["-b:a", config.AUDIO_BITRATE]
+_brand_supported: Optional[bool] = None
+
+
+def _detect_brand_support() -> bool:
+ """Check whether the local ffmpeg muxer accepts the ``-brand`` option."""
+ if shutil.which("ffmpeg") is None:
+ return False
+ try:
+ proc = subprocess.run(
+ ["ffmpeg", "-hide_banner", "-h", "muxer=ipod"],
+ capture_output=True, text=True, timeout=15,
+ )
+ except (OSError, subprocess.TimeoutExpired):
+ return False
+ return "-brand" in proc.stdout
+
+
+def _m4b_container_args() -> List[str]:
+ """Return per-output container flags for m4b files.
+
+ ``+faststart`` moves the ``moov`` index to the front of the file so
+ streaming players (and naive linear readers like web players) can index
+ it; without it they may misreport the duration or refuse the file.
+ The ``M4B `` major brand identifies the file as an audiobook to
+ players that sniff brands instead of trusting the extension (the ffmpeg
+ default brand for .m4b is ``M4A ``).
+ """
+ global _brand_supported
+ if _brand_supported is None:
+ _brand_supported = _detect_brand_support()
+ args = ["-movflags", "+faststart"]
+ if _brand_supported:
+ args += ["-brand", "M4B "]
+ return args
+
+
+def build_concat_command(concat_list: Path, output_path: Path, output_format: str,
+ speed: float = 1.0, speed_path: Optional[Path] = None) -> List[str]:
+ """Build the ffmpeg command that concatenates chunk audio into a book file."""
+ encode = _encode_args(output_format)
+ container = _m4b_container_args() if output_format == "m4b" else []
+ filters = atempo_filters(speed)
+ if filters:
+ return [
+ "ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", str(concat_list),
+ "-filter_complex",
+ f"[0:a]split=2[base][spd];[spd]{filters}[spdout]",
+ "-map", "[base]", *encode, *container, str(output_path),
+ "-map", "[spdout]", *encode, *container, str(speed_path),
+ ]
+ return [
+ "ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", str(concat_list),
+ *encode, *container, str(output_path),
+ ]
+
+
+def build_m4b_chapters_command(concat_list: Path, metadata_file: Path, output_path: Path,
+ speed: float = 1.0, speed_path: Optional[Path] = None,
+ speed_metadata_file: Optional[Path] = None) -> List[str]:
+ """Build the ffmpeg command that assembles chapter audio into one m4b.
+
+ Chapter metadata inputs are bound to their outputs with explicit
+ ``-map_chapters`` so the normal-speed and speed-adjusted copies each get
+ their own (correctly scaled) chapter markers.
+ """
+ encode = _encode_args("m4b")
+ container = _m4b_container_args()
+ if speed_path is not None:
+ return [
+ "ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", str(concat_list),
+ "-i", str(metadata_file), "-i", str(speed_metadata_file),
+ "-filter_complex",
+ f"[0:a]split=2[base][spd];[spd]{atempo_filters(speed)}[spdout]",
+ "-map", "[base]", "-map_metadata", "1", "-map_chapters", "1",
+ *encode, *container, str(output_path),
+ "-map", "[spdout]", "-map_metadata", "2", "-map_chapters", "2",
+ *encode, *container, str(speed_path),
+ ]
+ return [
+ "ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", str(concat_list),
+ "-i", str(metadata_file),
+ "-map", "0:a", "-map_metadata", "1", "-map_chapters", "1",
+ *encode, *container, str(output_path),
+ ]
+
+
+_DURATION_WARN_TOLERANCE = 0.05 # warn when output duration drifts >5%
+_DURATION_FAIL_TOLERANCE = 0.25 # fail when output duration drifts >25%
+
+
+def verify_output_duration(path: Path, expected_ms: int) -> bool:
+ """Sanity-check an assembled file's duration against the expected total.
+
+ Catches corrupt assembly (truncated concat, bogus container metadata)
+ before the file reaches audiobook players. Duration drift beyond the
+ warn tolerance is logged; drift beyond the fail tolerance is an error
+ and the output is treated as broken. Returns True when unverifiable.
+ """
+ if expected_ms <= 0:
+ return True
+ actual_ms = probe_duration_ms(path)
+ if actual_ms <= 0:
+ logger.warning("Could not verify duration of %s (ffprobe failed)", path)
+ return True
+ drift = abs(actual_ms - expected_ms) / expected_ms
+ if drift > _DURATION_FAIL_TOLERANCE:
+ logger.error(
+ "Duration mismatch for %s: expected ~%.1fs, got %.1fs (%.0f%% off); output is likely corrupt",
+ path.name, expected_ms / 1000.0, actual_ms / 1000.0, drift * 100.0,
+ )
+ return False
+ if drift > _DURATION_WARN_TOLERANCE:
+ logger.warning(
+ "Duration drift for %s: expected ~%.1fs, got %.1fs (%.0f%% off)",
+ path.name, expected_ms / 1000.0, actual_ms / 1000.0, drift * 100.0,
+ )
+ return True
+
+
def combine_chunks(total_chunks: int, output_path: Path,
results: Optional[Dict[int, bool]] = None, speed: float = 1.0,
output_format: str = "mp3") -> bool:
@@ -95,33 +218,30 @@ def combine_chunks(total_chunks: int, output_path: Path,
for chunk_file in chunk_files:
list_file.write(f"file '{_concat_escape(str(chunk_file))}'\n")
- filters = atempo_filters(speed)
- encode = _encode_args(output_format)
- if filters:
+ speed_path = None
+ if atempo_filters(speed):
speed_path = output_path.with_name(f"{output_path.stem}_{speed:g}{output_path.suffix}")
- cmd = [
- "ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", str(concat_list),
- "-filter_complex",
- f"[0:a]split=2[base][spd];[spd]{filters}[spdout]",
- "-map", "[base]", *encode, str(output_path),
- "-map", "[spdout]", *encode, str(speed_path),
- ]
- else:
- speed_path = None
- cmd = [
- "ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", str(concat_list),
- *encode, str(output_path),
- ]
+ cmd = build_concat_command(concat_list, output_path, output_format,
+ speed=speed, speed_path=speed_path)
proc = subprocess.run(cmd, capture_output=True, text=True)
if proc.returncode != 0:
logger.error("ffmpeg failed: %s", proc.stderr[-2000:])
return False
+ # Verify the assembled duration against the sum of chunk durations
+ # so corrupt output is caught before it reaches audiobook players.
+ expected_ms = sum(probe_duration_ms(chunk_file) for chunk_file in chunk_files)
+ duration_ok = verify_output_duration(output_path, expected_ms)
+ if speed_path is not None:
+ duration_ok = verify_output_duration(speed_path, int(expected_ms / speed)) and duration_ok
+ if not duration_ok:
+ return False
+
logger.info("Audiobook saved: %s (%d/%d chunks)", output_path, len(chunk_files), total_chunks)
print(f"[INFO] Saved audiobook: {output_path.name} ({len(chunk_files)}/{total_chunks} chunks)")
- if filters:
+ if speed_path is not None:
logger.info("Saved speed-adjusted audiobook (%gx): %s", speed, speed_path)
print(f"[INFO] Saved speed-adjusted audiobook: {speed_path.name} ({speed:g}x)")
@@ -229,35 +349,29 @@ def combine_chapters_to_m4b(chapter_files: List[Path], titles: List[str],
build_ffmetadata(chapters, metadata_file)
- filters = atempo_filters(speed)
- encode = _encode_args("m4b")
- if filters:
+ speed_path = None
+ if atempo_filters(speed):
speed_path = output_path.with_name(f"{output_path.stem}_{speed:g}{output_path.suffix}")
scaled = [(int(s / speed), int(e / speed), t) for s, e, t in chapters]
build_ffmetadata(scaled, speed_metadata_file)
- # The speed-adjusted stream needs rescaled chapter markers, so the
- # rescaled metadata is passed as a third input.
- cmd = [
- "ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", str(concat_list),
- "-i", str(metadata_file), "-i", str(speed_metadata_file),
- "-filter_complex",
- f"[0:a]split=2[base][spd];[spd]{filters}[spdout]",
- "-map", "[base]", "-map_metadata", "1", *encode, str(output_path),
- "-map", "[spdout]", "-map_metadata", "2", *encode, str(speed_path),
- ]
- else:
- speed_path = None
- cmd = [
- "ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", str(concat_list),
- "-i", str(metadata_file),
- "-map", "0:a", "-map_metadata", "1", *encode, str(output_path),
- ]
+ cmd = build_m4b_chapters_command(concat_list, metadata_file, output_path,
+ speed=speed, speed_path=speed_path,
+ speed_metadata_file=speed_metadata_file
+ if speed_path is not None else None)
proc = subprocess.run(cmd, capture_output=True, text=True)
if proc.returncode != 0:
logger.error("ffmpeg failed: %s", proc.stderr[-2000:])
return False
+ # The last chapter's end time is the expected total duration.
+ expected_ms = chapters[-1][1]
+ duration_ok = verify_output_duration(output_path, expected_ms)
+ if speed_path is not None:
+ duration_ok = verify_output_duration(speed_path, int(expected_ms / speed)) and duration_ok
+ if not duration_ok:
+ return False
+
logger.info("Audiobook saved: %s (%d chapters)", output_path, len(chapter_files))
print(f"[INFO] Saved audiobook: {output_path.name} ({len(chapter_files)} chapters)")
diff --git a/converter/chunking.py b/converter/chunking.py
index f649310..0c85adf 100644
--- a/converter/chunking.py
+++ b/converter/chunking.py
@@ -11,8 +11,10 @@ def split_into_chunks(text: str, max_words: int = config.CHUNK_SIZE_WORDS) -> Li
Splits on sentence boundaries. Sentences longer than the limit are split
further at clause punctuation (which is kept attached for TTS prosody).
- A single sentence with no clause punctuation longer than the limit is
- kept intact as one oversized chunk.
+ Clause splits only happen at whitespace after punctuation, so tokens like
+ "1,000,000" or "12:30" are never broken apart or re-joined with added
+ spaces. A single sentence with no usable split point longer than the
+ limit is kept intact as one oversized chunk.
"""
if not text.strip():
return []
@@ -32,7 +34,10 @@ def split_into_chunks(text: str, max_words: int = config.CHUNK_SIZE_WORDS) -> Li
current_words = 0
# Split long sentences at clause boundaries, keeping punctuation.
- parts = re.split(r"(?<=[,;:])\s*", sentence)
+ # Only split where whitespace already follows the punctuation so
+ # the reassembled text is byte-identical to the input (no spaces
+ # injected into "1,000,000" or "12:30").
+ parts = re.split(r"(?<=[,;:])\s+", sentence)
for part in parts:
part_words = len(part.split())
if current_words + part_words <= max_words:
diff --git a/converter/converter.py b/converter/converter.py
index f63a412..0454892 100644
--- a/converter/converter.py
+++ b/converter/converter.py
@@ -130,12 +130,17 @@ class AudiobookConverter:
def _convert_m4b_with_chapters(self, sections, stem: str, start_time: float) -> bool:
"""Convert each chapter to audio, then assemble a single m4b with
- embedded chapter markers."""
+ embedded chapter markers.
+
+ Chapters are synthesized to lossless WAV scratch files (~170 MB per
+ hour of audio) so the final AAC pass is the only lossy encode.
+ """
chapter_files = []
titles = []
for index, section in enumerate(sections, 1):
- chapter_path = config.CHUNKS_FOLDER / f"chapter_{index:04d}.{self.output_format}"
- if not self._convert_text(section.text, chapter_path, start_time, speed=1.0):
+ chapter_path = config.CHUNKS_FOLDER / f"chapter_{index:04d}.wav"
+ if not self._convert_text(section.text, chapter_path, start_time,
+ speed=1.0, output_format="wav"):
logger.warning("Skipping chapter %d (%s) due to conversion failure",
index, section.title)
continue
@@ -150,10 +155,13 @@ class AudiobookConverter:
return audio.combine_chapters_to_m4b(chapter_files, titles, output_path, speed=self.speed)
def _convert_text(self, text: str, output_path: Path, start_time: float,
- speed: Optional[float] = None) -> bool:
+ speed: Optional[float] = None,
+ output_format: Optional[str] = None) -> bool:
"""Chunk, synthesize, and assemble ``text`` into ``output_path``."""
if speed is None:
speed = self.speed
+ if output_format is None:
+ output_format = self.output_format
try:
if not text.strip():
@@ -217,7 +225,7 @@ class AudiobookConverter:
# Combine chunks (only the successful ones)
success = audio.combine_chunks(total_chunks, output_path, results,
- speed=speed, output_format=self.output_format)
+ speed=speed, output_format=output_format)
if success:
duration = time.time() - start_time
diff --git a/input/test.txt b/input/test.txt
deleted file mode 100644
index a316be8..0000000
--- a/input/test.txt
+++ /dev/null
@@ -1 +0,0 @@
-The truth about the world, he said, is that anything is possible. Had you not seen it all from birth and thereby bled it of its strangeness it would appear to you for what it is, a hat trick in a medicine show, a fevered dream, a trance bepopulate with chimeras having neither analogue nor precedent, an itinerant carnival, a migratory tentshow whose ultimate destination after many a pitch in a many a mudded field is unspeakable and calamitous beyond reckoning.
diff --git a/tests/test_audio.py b/tests/test_audio.py
index 29bf4ea..f36ed54 100644
--- a/tests/test_audio.py
+++ b/tests/test_audio.py
@@ -1,11 +1,21 @@
-"""Tests for audio helpers: speed parameters, chunk cleanup, and encoding."""
+"""Tests for audio helpers: speed parameters, chunk cleanup, encoding,
+command construction, and duration verification."""
import tempfile
import unittest
from pathlib import Path
+from converter import audio
from converter import config
-from converter.audio import _encode_args, build_ffmetadata, cleanup_chunks, speed_export_params
+from converter.audio import (
+ _encode_args,
+ build_concat_command,
+ build_ffmetadata,
+ build_m4b_chapters_command,
+ cleanup_chunks,
+ speed_export_params,
+ verify_output_duration,
+)
class SpeedExportParamsTests(unittest.TestCase):
@@ -76,6 +86,124 @@ class EncodeArgsTests(unittest.TestCase):
def test_m4b_uses_aac(self):
self.assertEqual(_encode_args("m4b"), ["-c:a", "aac", "-b:a", config.AUDIO_BITRATE])
+ def test_wav_is_lossless_pcm(self):
+ self.assertEqual(_encode_args("wav"), ["-c:a", "pcm_s16le"])
+
+
+class M4bContainerArgsTests(unittest.TestCase):
+ def setUp(self):
+ self._original = audio._brand_supported
+ audio._brand_supported = True
+
+ def tearDown(self):
+ audio._brand_supported = self._original
+
+ def test_includes_faststart_and_brand(self):
+ args = audio._m4b_container_args()
+ self.assertIn("+faststart", args)
+ self.assertIn("M4B ", args)
+
+ def test_brand_omitted_when_unsupported(self):
+ audio._brand_supported = False
+ self.assertEqual(audio._m4b_container_args(), ["-movflags", "+faststart"])
+
+
+class BuildConcatCommandTests(unittest.TestCase):
+ def setUp(self):
+ self._original = audio._brand_supported
+ audio._brand_supported = True
+
+ def tearDown(self):
+ audio._brand_supported = self._original
+
+ def test_mp3_has_no_container_flags(self):
+ cmd = build_concat_command(Path("list.txt"), Path("out.mp3"), "mp3")
+ self.assertEqual(cmd[:6], ["ffmpeg", "-y", "-f", "concat", "-safe", "0"])
+ self.assertNotIn("-movflags", cmd)
+ self.assertEqual(cmd[-1], "out.mp3")
+
+ def test_m4b_gets_faststart_and_brand(self):
+ cmd = build_concat_command(Path("list.txt"), Path("out.m4b"), "m4b")
+ self.assertIn("+faststart", cmd)
+ self.assertIn("M4B ", cmd)
+ self.assertEqual(cmd[-1], "out.m4b")
+
+ def test_speed_copy_writes_two_outputs(self):
+ cmd = build_concat_command(Path("list.txt"), Path("out.m4b"), "m4b",
+ speed=1.5, speed_path=Path("out_1.5.m4b"))
+ self.assertIn("out.m4b", cmd)
+ self.assertIn("out_1.5.m4b", cmd)
+ # faststart must apply to both outputs
+ self.assertEqual(cmd.count("+faststart"), 2)
+ self.assertTrue(any("atempo=1.5" in arg for arg in cmd))
+
+ def test_wav_intermediate(self):
+ cmd = build_concat_command(Path("list.txt"), Path("chapter.wav"), "wav")
+ self.assertIn("pcm_s16le", cmd)
+ self.assertNotIn("-movflags", cmd)
+
+
+class BuildM4bChaptersCommandTests(unittest.TestCase):
+ def setUp(self):
+ self._original = audio._brand_supported
+ audio._brand_supported = True
+
+ def tearDown(self):
+ audio._brand_supported = self._original
+
+ def test_base_output_maps_metadata_and_chapters(self):
+ cmd = build_m4b_chapters_command(Path("list.txt"), Path("meta.txt"), Path("out.m4b"))
+ self.assertIn("-map_metadata", cmd)
+ self.assertIn("-map_chapters", cmd)
+ self.assertIn("out.m4b", cmd)
+ self.assertIn("+faststart", cmd)
+ self.assertNotIn("filter_complex", cmd)
+
+ def test_speed_outputs_get_their_own_chapter_metadata(self):
+ cmd = build_m4b_chapters_command(
+ Path("list.txt"), Path("meta.txt"), Path("out.m4b"),
+ speed=2.0, speed_path=Path("out_2.m4b"), speed_metadata_file=Path("meta2.txt"),
+ )
+ self.assertEqual(cmd.count("+faststart"), 2)
+ self.assertEqual(cmd.count("-map_chapters"), 2)
+ self.assertTrue(any("atempo=2" in arg for arg in cmd))
+ # base output chapters come from metadata input 1, speed copy from 2
+ chapter_flags = [i for i, v in enumerate(cmd) if v == "-map_chapters"]
+ self.assertEqual(cmd[chapter_flags[0] + 1], "1")
+ self.assertEqual(cmd[chapter_flags[1] + 1], "2")
+ base_idx, speed_idx = cmd.index("out.m4b"), cmd.index("out_2.m4b")
+ self.assertLess(chapter_flags[0], base_idx)
+ self.assertGreater(chapter_flags[1], base_idx)
+ self.assertLess(chapter_flags[1], speed_idx)
+
+
+class VerifyOutputDurationTests(unittest.TestCase):
+ def _patch_probe(self, ms):
+ audio.probe_duration_ms = lambda path: ms
+
+ def setUp(self):
+ self._original_probe = audio.probe_duration_ms
+
+ def tearDown(self):
+ audio.probe_duration_ms = self._original_probe
+
+ def test_close_duration_passes(self):
+ self._patch_probe(100_000)
+ self.assertTrue(verify_output_duration(Path("x.m4b"), 101_000))
+
+ def test_unverifiable_duration_passes(self):
+ self._patch_probe(0)
+ self.assertTrue(verify_output_duration(Path("x.m4b"), 100_000))
+
+ def test_zero_expected_passes(self):
+ self._patch_probe(50_000)
+ self.assertTrue(verify_output_duration(Path("x.m4b"), 0))
+
+ def test_large_drift_fails(self):
+ self._patch_probe(3_600_000) # bogus "1 hour" for a 1 minute book
+ with self.assertLogs(level="ERROR"):
+ self.assertFalse(verify_output_duration(Path("x.m4b"), 60_000))
+
class BuildFFMetadataTests(unittest.TestCase):
def test_writes_chapters(self):
diff --git a/tests/test_chunking.py b/tests/test_chunking.py
index da45f65..659a771 100644
--- a/tests/test_chunking.py
+++ b/tests/test_chunking.py
@@ -22,13 +22,34 @@ class SplitIntoChunksTests(unittest.TestCase):
self.assertEqual(sum(len(c.split()) for c in chunks), 90)
def test_long_sentence_split_keeps_punctuation(self):
- # 10 clauses of 5 words each, joined by commas
- sentence = ",".join([" ".join(["w"] * 5) for _ in range(10)]) + "."
+ # 10 clauses of 5 words each, joined by comma+space
+ sentence = ", ".join([" ".join(["w"] * 5) for _ in range(10)]) + "."
chunks = split_into_chunks(sentence, max_words=12)
self.assertGreater(len(chunks), 1)
self.assertTrue(all(len(c.split()) <= 12 for c in chunks))
self.assertIn(",", chunks[0]) # commas retained for TTS prosody
+ def test_clause_split_never_breaks_numbers(self):
+ # Regression: the clause split used to fire at every comma even
+ # without whitespace, mutating "1,000,000" into "1, 000, 000".
+ sentence = ("There were exactly 1,000,000 soldiers marching at 12:30, "
+ + "and they kept marching onward " * 30) + "endlessly."
+ chunks = split_into_chunks(sentence, max_words=25)
+ self.assertGreater(len(chunks), 1)
+ joined = " ".join(chunks)
+ self.assertIn("1,000,000", joined)
+ self.assertIn("12:30", joined)
+ self.assertNotIn("1, 000", joined)
+ self.assertNotIn("000, 000", joined)
+ self.assertNotIn("12: 30", joined)
+
+ def test_clause_split_requires_whitespace_after_punctuation(self):
+ # Run-on clauses without spaces after commas have no split point and
+ # must stay byte-identical rather than being re-joined with spaces.
+ sentence = ",".join([" ".join(["w"] * 5) for _ in range(10)]) + "."
+ chunks = split_into_chunks(sentence, max_words=12)
+ self.assertEqual(chunks, [sentence])
+
def test_single_oversized_sentence_stays_intact(self):
sentence = " ".join(["word"] * 30) + "."
chunks = split_into_chunks(sentence, max_words=10)