From 5ed31309b6d0db94ccd566914653d367f4577c64 Mon Sep 17 00:00:00 2001 From: historia Date: Fri, 28 Aug 2026 04:45:29 -0400 Subject: fix: japanese characters not truncated correctly in tui terminal output --- app/tests/test_runview.py | 11 +++++++++++ app/tests/test_taskview.py | 19 +++++++++++++++++++ app/tests/test_tui.py | 25 +++++++++++++++++++++++++ app/ui/tui.py | 27 +++++++++++++++++++++++---- app/ui/viewkit.py | 8 ++------ 5 files changed, 80 insertions(+), 10 deletions(-) diff --git a/app/tests/test_runview.py b/app/tests/test_runview.py index 8feeee1..3f77ed0 100644 --- a/app/tests/test_runview.py +++ b/app/tests/test_runview.py @@ -62,10 +62,21 @@ class FormatTests(_FakeTui, unittest.TestCase): self.assertEqual(runview._fit("hello", 3), "he~") self.assertEqual(runview._fit("hi", 10), "hi") + def test_fit_truncates_wide_characters_by_display_columns(self): + # Japanese characters fill two terminal cells each, so fitting + # by character count would let lines overflow their pane. + self.assertEqual(runview._fit("你好", 4), "你好") + self.assertEqual(runview._fit("你好你好", 5), "你好~") + self.assertEqual(runview._fit("こんにちは", 3), "こ~") + def test_wrap_wraps_on_word_boundaries(self): self.assertEqual(runview._wrap("aaaa bbbb cccc dddd", 12), ["aaaa bbbb", "cccc dddd"]) + def test_wrap_counts_wide_characters_as_two_columns(self): + self.assertEqual(runview._wrap("日本語 テスト", 10), + ["日本語", "テスト"]) + class StateTransitionTests(_FakeTui, unittest.TestCase): def test_boot_flow_starting_to_ready(self): diff --git a/app/tests/test_taskview.py b/app/tests/test_taskview.py index 1db708d..78cc2f1 100644 --- a/app/tests/test_taskview.py +++ b/app/tests/test_taskview.py @@ -725,6 +725,25 @@ class LanesViewTests(_FakeTui, unittest.TestCase): pane_w = (screen.width - 3) // 2 self.assertLessEqual(x + len(label) - 1, 1 + pane_w - 2) + def test_split_render_truncates_wide_log_lines_to_their_pane(self): + # Regression: Japanese characters fill two terminal cells each, + # so fitting by character count let transcription results spill + # out of the right pane onto the left one. + view, screen = self.make_view(self._two_lanes(), width=80) + view._ingest_lane_line(view._lanes[1], + "[OK] 一号: " + "語" * 40) + view.render() + found = [(x, t) for _, x, t, _ in screen.strings + if t.startswith("[OK] ")] + self.assertEqual(len(found), 1) + x, line = found[0] + self.assertTrue(line.endswith("~")) + pane_w = (screen.width - 3) // 2 + px, pw = 1 + pane_w + 1, (screen.width - 3) - pane_w + self.assertGreaterEqual(x, px + 1) + self.assertLessEqual(x + taskview.tui._disp_width(line) - 1, + px + pw - 2) + class SilenceCueTests(_FakeTui, unittest.TestCase): """The "(no output 6m)" cue for a running step that stopped emitting.""" diff --git a/app/tests/test_tui.py b/app/tests/test_tui.py index 6fc3fd5..4c8b431 100644 --- a/app/tests/test_tui.py +++ b/app/tests/test_tui.py @@ -1501,6 +1501,31 @@ class TextViewerTests(TuiTestCase): self.assertTrue(drawn[0].endswith("~")) +class FitTests(TuiTestCase): + """Column-aware fitting: East Asian Wide characters occupy 2 cells.""" + + def test_disp_width_counts_wide_characters_twice(self): + self.assertEqual(tui._disp_width(""), 0) + self.assertEqual(tui._disp_width("hello"), 5) + self.assertEqual(tui._disp_width("一号"), 4) + self.assertEqual(tui._disp_width("mix語end"), 8) + + def test_fit_truncates_by_display_columns(self): + self.assertEqual(tui._fit("hello", 3), "he~") + self.assertEqual(tui._fit("hi", 10), "hi") + # Exact fit keeps the wide pair; one column more would overflow. + self.assertEqual(tui._fit("你好", 4), "你好") + self.assertEqual(tui._fit("你好你好", 5), "你好~") + self.assertEqual(tui._fit("こんにちは", 3), "こ~") + + def test_wrap_segments_counts_wide_characters_twice(self): + # "語 語 語" is 8 cells, so at width 5 only two words fit per + # line even though 5 characters would fit by count. + lines = tui._wrap_segments([("語 語 語", None)], 5) + self.assertEqual(["".join(t for t, _ in line) for line in lines], + ["語 語", "語"]) + + class WizardTests(unittest.TestCase): """The tui.Wizard screen-stack driver: Esc steps back one screen.""" diff --git a/app/ui/tui.py b/app/ui/tui.py index b22a186..60e2dda 100644 --- a/app/ui/tui.py +++ b/app/ui/tui.py @@ -37,6 +37,7 @@ import contextlib import os import re import textwrap +import unicodedata from pathlib import Path from typing import Callable, List, Optional, Sequence, Tuple, Union @@ -255,13 +256,31 @@ def _hline(scr, y: int, x: int, n: int, attr: int = 0) -> None: pass +def _char_width(ch: str) -> int: + """Terminal cells CH occupies (East Asian Wide/Fullwidth count 2).""" + return 2 if unicodedata.east_asian_width(ch) in ("W", "F") else 1 + + +def _disp_width(text: str) -> int: + """Terminal cells TEXT occupies (East Asian Wide/Fullwidth count 2).""" + return sum(_char_width(ch) for ch in text) + + def _fit(text: str, width: int) -> str: """Truncate TEXT to WIDTH columns, appending '~' when cut.""" if width < 1: return "" - if len(text) <= width: + if _disp_width(text) <= width: return text - return text[: max(0, width - 1)] + "~" + out: List[str] = [] + used = 0 + for ch in text: + w = _char_width(ch) + if used + w > width - 1: + break + out.append(ch) + used += w + return "".join(out) + "~" def _wrap_segments(segments: Sequence[Tuple[str, int]], width: int @@ -283,13 +302,13 @@ def _wrap_segments(segments: Sequence[Tuple[str, int]], width: int current: List[Tuple[str, int]] = [] used = 0 for text, attr in tokens: - if current and used + len(text) > width: + if current and used + _disp_width(text) > width: lines.append(current) current, used = [], 0 if text.isspace(): continue # the break eats the whitespace it happens at current.append((text, attr)) - used += len(text) + used += _disp_width(text) if current: lines.append(current) return lines or [[]] diff --git a/app/ui/viewkit.py b/app/ui/viewkit.py index 301a0af..e54aada 100644 --- a/app/ui/viewkit.py +++ b/app/ui/viewkit.py @@ -217,11 +217,7 @@ def _sep(scr, curses, theme, y, width) -> None: def _fit(text: str, width: int) -> str: """Truncate TEXT to WIDTH columns, appending '~' when cut.""" - if width < 1: - return "" - if len(text) <= width: - return text - return text[: max(0, width - 1)] + "~" + return tui._fit(text, width) def _wrap(text: str, width: int) -> List[str]: @@ -230,7 +226,7 @@ def _wrap(text: str, width: int) -> List[str]: current = "" for word in text.split(): candidate = f"{current} {word}".strip() - if len(candidate) <= max(10, width): + if tui._disp_width(candidate) <= max(10, width): current = candidate else: if current: -- cgit v1.2.3