fix: smart chunking, extraction, and process-safety issuesHEADmain

1 files changed, 85 insertions, 12 deletions

diff --git a/app/converter/chunking.py b/app/converter/chunking.py
index 0ffaa21..bf104bc 100644
--- a/app/converter/chunking.py
+++ b/app/converter/chunking.py
@@ -21,7 +21,8 @@ _FLOOR_RATIO = 0.7
 # boundary; if such a token runs longer than this many characters per
 # allowed word it is sliced at the hard character cap (a last-resort
 # bound for punctuation-free input such as some CJK text or damage from
-# text extraction).
+# text extraction). The slice also honors the word limit itself, since
+# every non-spaced-script character counts as one word.
 _MAX_CHARS_PER_WORD = 8
 
 _ABBREVIATIONS = frozenset(
@@ -203,8 +204,8 @@ def _split_oversized(text: str, max_words: int) -> List[str]:
     First at clause punctuation (kept attached for TTS prosody, and
     never at commas without whitespace so tokens like "1,000,000" and
     "12:30" survive), then at word boundaries. A token with no
-    whitespace at all is sliced at the hard character cap so a
-    punctuation-free run cannot exceed the bound.
+    whitespace at all is sliced at the hard character cap and the word
+    limit so a punctuation-free run cannot exceed either bound.
     """
     parts = re.split(r"(?:(?<=[,;:])\s+|(?<=[，、；：]))", text)
     pieces: List[str] = []
@@ -220,21 +221,50 @@ def _split_oversized(text: str, max_words: int) -> List[str]:
     return pieces
 
 
+def _slice_token(word: str, max_words: int, cap: int) -> List[str]:
+    """Slice a single whitespace-free token into packable pieces.
+
+    Cuts before the character cap is exceeded and before the word count
+    is exceeded: each non-spaced-script character counts as one word,
+    so a punctuation-free CJK token is bounded by the word limit too,
+    not only by the character cap.
+    """
+    slices: List[str] = []
+    cur: List[str] = []
+    cur_chars = 0
+    # Any slice is itself one whitespace-delimited word before its
+    # non-spaced-script characters are counted on top.
+    cur_words = 1
+    for ch in word:
+        ch_words = 1 if _NON_SPACED_RUN.match(ch) else 0
+        if cur and (cur_chars + 1 > cap or cur_words + ch_words > max_words):
+            slices.append("".join(cur))
+            cur, cur_chars, cur_words = [], 0, 1
+        cur.append(ch)
+        cur_chars += 1
+        cur_words += ch_words
+    if cur:
+        slices.append("".join(cur))
+    return slices
+
+
 def _hard_slices(text: str, max_words: int, cap: int) -> List[str]:
-    """Slice TEXT at word boundaries, then at CAP characters when a
-    single token has no whitespace to break at. Buffers never exceed
-    the word limit or the character cap, so every piece is packable."""
+    """Slice TEXT at word boundaries, then within tokens when a single
+    token (no whitespace to break at) exceeds the character cap or the
+    word limit on its own. Buffers never exceed the word limit or the
+    character cap, so every piece is packable."""
     pieces: List[str] = []
     buf: List[str] = []
     buf_words = 0
     buf_chars = 0
     for word in text.split():
-        if len(word) > cap:
+        if len(word) > cap or _count_words(word) > max_words:
+            # A single token too big to buffer: slice it at the hard
+            # character cap and the word limit together.
             if buf:
                 pieces.append(" ".join(buf))
                 buf, buf_words, buf_chars = [], 0, 0
-            for start in range(0, len(word), cap):
-                pieces.append(word[start:start + cap])
+            pieces.extend(_slice_token(word, max_words, cap))
             continue
         words = _count_words(word)
         if buf and buf_words + words <= max_words \
@@ -266,6 +296,41 @@ def _joined_chars(units: List[_Unit]) -> int:
     return total
 
 
+def _quote_states(pieces: List[str], in_quote_end: bool) -> List[bool]:
+    """The quote state after each piece of an oversized sentence.
+
+    Pieces of one sentence can sit inside a quotation the sentence does
+    not close, so ending a chunk at such a piece would sever the quote
+    (and marking them clean lets the target flush cut mid-quote). The
+    scanner's quote tracking is replayed over the pieces from the state
+    the sentence started in; that start state is unknown, so both
+    possibilities are replayed and the one whose end state matches the
+    unit's known end state wins (a unit rarely starts mid-quote). When
+    neither matches, every piece takes the unit's conservative end
+    state.
+    """
+    def simulate(in_quote: bool, depth: int) -> List[bool]:
+        states: List[bool] = []
+        for piece in pieces:
+            for ch in piece:
+                if ch == '"':
+                    in_quote = not in_quote
+                elif ch in "“«":
+                    depth += 1
+                elif ch in "”»":
+                    depth = max(0, depth - 1)
+            states.append(in_quote or depth > 0)
+        return states
+
+    states = simulate(False, 0)
+    if states and states[-1] == in_quote_end:
+        return states
+    states = simulate(True, 0)
+    if states and states[-1] == in_quote_end:
+        return states
+    return [in_quote_end] * len(pieces)
+
+
 def _split_smart(text: str, max_words: int) -> List[str]:
     """Smart strategy: fill chunks up to MAX_WORDS at sentence
     boundaries, end them at the target once a quotation has closed or
@@ -318,9 +383,14 @@ def _split_smart(text: str, max_words: int) -> List[str]:
             cur_chars = _joined_chars(cur)
             continue
         # One piece exceeds the limit itself: send its pieces out
-        # packed as tightly as the pieces allow.
+        # packed as tightly as the pieces allow, carrying the
+        # sentence's quotation state through the fallback so a chunk
+        # can end at a clean in-sentence boundary — and a piece still
+        # inside a quote is never mistaken for one.
         flush()
-        for piece_text in _split_oversized(unit.text, max_words):
+        piece_texts = _split_oversized(unit.text, max_words)
+        quote_states = _quote_states(piece_texts, unit.in_quote_end)
+        for index, piece_text in enumerate(piece_texts):
             if _count_words(piece_text) > max_words \
                     or len(piece_text) > char_cap:
                 # A character-sliced piece of degenerate input: its own
@@ -328,7 +398,10 @@ def _split_smart(text: str, max_words: int) -> List[str]:
                 flush()
                 chunks.append(piece_text)
                 continue
-            piece = _Unit(piece_text, _count_words(piece_text), False, False)
+            piece = _Unit(
+                piece_text, _count_words(piece_text),
+                unit.para_after and index == len(piece_texts) - 1,
+                quote_states[index])
             if cur and cur_words + piece.words <= max_words \
                     and cur_chars + 1 + len(piece_text) <= char_cap:
                 cur.append(piece)