Skip to content

An always-on second brain you talk to. Voice notes in Telegram → typed, linked knowledge in your Obsidian vault. Runs 24/7 on the Claude subscription you already have.

fix(formatters): split_text dropped characters at chunk boundaries

smixsaf742904252c
diff --git a/src/d_brain/bot/formatters.py b/src/d_brain/bot/formatters.py
index cf741e4..1e4002d 100644
--- a/src/d_brain/bot/formatters.py
+++ b/src/d_brain/bot/formatters.py
@@ -107,8 +107,17 @@ def truncate_html(text: str, max_length: int = 4096) -> str:
Returns:
Truncated text with balanced tags
"""
+ return _truncate_html_consumed(text, max_length)[0]
+
+
+def _truncate_html_consumed(text: str, max_length: int) -> tuple[str, int]:
+ """Truncate like truncate_html, also returning how many characters
+ of the ORIGINAL text the chunk covers — the decorated chunk gains
+ an ellipsis and closing tags, so its own length is useless for
+ advancing a split cursor.
+ """
if len(text) <= max_length:
- return text
+ return text, len(text)
# Find a safe cut point
cut_point = max_length - 50 # Leave room for closing tags and ellipsis
@@ -142,7 +151,7 @@ def truncate_html(text: str, max_length: int = 4096) -> str:
# Add closing tags in reverse order
closing_tags = "".join(f"</{tag}>" for tag in reversed(open_tags))
- return truncated + "..." + closing_tags
+ return truncated + "..." + closing_tags, cut_point
def format_process_report(report: dict[str, Any]) -> str:
@@ -236,14 +245,12 @@ def split_text(text: str, max_len: int) -> list[str]:
if len(remaining) <= max_len:
chunks.append(remaining)
break
- chunk = truncate_html(remaining, max_len)
- chunks.append(chunk)
- # Move past what we consumed
- consumed = len(chunk.rstrip(".").rstrip())
- if consumed == 0:
+ chunk, consumed = _truncate_html_consumed(remaining, max_len)
+ if consumed <= 0:
# Safety: force split to avoid infinite loop
- chunks[-1] = remaining[:max_len]
+ chunks.append(remaining[:max_len])
remaining = remaining[max_len:]
else:
- remaining = remaining[consumed:].lstrip()
+ chunks.append(chunk)
+ remaining = remaining[consumed:]
return chunks
diff --git a/tests/test_formatters.py b/tests/test_formatters.py
new file mode 100644
index 0000000..bc7a3ee
--- /dev/null
+++ b/tests/test_formatters.py
@@ -0,0 +1,55 @@
+"""Tests for Telegram delivery formatters: splitting and truncation."""
+
+from d_brain.bot.formatters import (
+ MAX_RESPONSE_LENGTH,
+ split_text,
+ truncate_html,
+)
+
+
+class TestSplitText:
+ def test_short_text_passes_through(self):
+ assert split_text("hello", MAX_RESPONSE_LENGTH) == ["hello"]
+
+ def test_chunks_respect_limit(self):
+ text = "слово " * 2000 # ~12000 chars
+ chunks = split_text(text, MAX_RESPONSE_LENGTH)
+ assert len(chunks) > 1
+ assert all(len(c) <= MAX_RESPONSE_LENGTH for c in chunks)
+
+ def test_no_content_lost_when_tag_spans_boundary(self):
+ # An open <b> at the cut point makes truncate_html append
+ # "...</b>" — the split must still advance by the ORIGINAL
+ # characters consumed, not by the decorated chunk length.
+ payload = "x" * 9000
+ text = f"<b>{payload}</b>"
+ chunks = split_text(text, MAX_RESPONSE_LENGTH)
+ joined = "".join(chunks)
+ assert joined.count("x") == 9000
+
+ def test_no_content_lost_or_duplicated_in_plain_text(self):
+ # The ellipsis decoration is dots-only, so letter counts must
+ # survive the split exactly (no loss, no duplication).
+ text = "abc. " * 2000 # 10000 chars, sentences end with dots
+ chunks = split_text(text, MAX_RESPONSE_LENGTH)
+ joined = "".join(chunks)
+ for letter in "abc":
+ assert joined.count(letter) == 2000
+
+ def test_progress_on_pathological_input(self):
+ # Must terminate even when the cut point math degenerates.
+ text = "." * 5000
+ chunks = split_text(text, MAX_RESPONSE_LENGTH)
+ assert all(len(c) <= MAX_RESPONSE_LENGTH for c in chunks)
+ assert "".join(chunks).count(".") >= 5000
+
+
+class TestTruncateHtml:
+ def test_short_text_untouched(self):
+ assert truncate_html("<b>hi</b>", 4096) == "<b>hi</b>"
+
+ def test_closes_open_tags(self):
+ text = "<b>" + "x" * 5000
+ out = truncate_html(text, 4096)
+ assert len(out) <= 4096
+ assert out.endswith("</b>")