Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(dedup): bound line-number digit run and never fold into a longer pointer (#4488)

HetCreep21e8867504df
diff --git a/headroom/transforms/cross_turn_dedup.py b/headroom/transforms/cross_turn_dedup.py
index af6cd6697..108c49108 100644
--- a/headroom/transforms/cross_turn_dedup.py
+++ b/headroom/transforms/cross_turn_dedup.py
@@ -39,9 +39,9 @@ from ..log_safety import describe_exception, safe_id
logger = logging.getLogger(__name__)
# A run must be at least this many lines AND this many chars to be worth a
-# pointer. Small dups are left alone (fragmenting context is not worth it) —
-# and a larger floor keeps the pointer comfortably shorter than the span it
-# replaces, so a fold is always a net byte win.
+# pointer. Small dups are left alone (fragmenting context is not worth it).
+# The floor alone does not make a fold a net byte win (the pointer can still be
+# as long as a short span); dedup_blocks checks the UTF-8 sizes before folding.
DEFAULT_MIN_LINES = 3
DEFAULT_MIN_CHARS = 40
# Cap anchor candidates examined per line so a hot line (e.g. `` return``)
@@ -54,7 +54,7 @@ MAX_ANCHOR_CANDIDATES = 16
# line must start with the digit, so ``number + delta`` re-numbers byte-exactly
# without touching alignment padding, keeping renumbered folds strictly lossless.
#
-# ``[1-9]\d*`` (not ``\d+``) is load-bearing: a LEADING-ZERO run (``08:00:01`` in
+# The leading ``[1-9]`` (not ``\d``) is load-bearing: a LEADING-ZERO run (``08:00:01`` in
# a timestamped log, ``007:...``) is not an unpadded grep/sed line number, and
# recovery is ``str(int(number) + delta)``, which drops the zero pad — ``int("08")
# + 1`` renders ``"9"``, not ``"09"``. Matching those would fold them under a
@@ -63,7 +63,11 @@ MAX_ANCHOR_CANDIDATES = 16
# fold only on an EXACT match (delta 0) — the "prefer false negatives" posture.
# Real grep -n / sed -n / rg -n numbers never carry a leading zero, so the
# intended renumber-fold feature is unaffected.
-_LINENO_RE = re.compile(r"^([1-9]\d*)(:|\t)(.*)$", re.DOTALL)
+#
+# ``\d{0,17}`` bounds the run at 18 digits: Python refuses ``int()`` on a run of
+# 4,301+ digits (ValueError), which would abort dedup for the whole request. A
+# longer run is not a real line number, so the line stays non-numbered.
+_LINENO_RE = re.compile(r"^([1-9]\d{0,17})(:|\t)(.*)$", re.DOTALL)
def _num_and_key(line: str) -> tuple[int | None, str, str]:
@@ -264,9 +268,12 @@ def dedup_blocks(
if m is not None and m[0] >= min_lines:
span = lines[i : i + m[0]]
span_text = "\n".join(span)
- if len(span_text) >= min_chars:
- ref_turn = blocks[m[1]].turn
- ptr = _pointer(span, ref_turn, m[3])
+ ptr = _pointer(span, blocks[m[1]].turn, m[3])
+ # Never fold when the pointer is not shorter than the span, in
+ # UTF-8 bytes ("↑" is 3 bytes, so character counts mislead).
+ if len(span_text) >= min_chars and len(ptr.encode("utf-8")) < len(
+ span_text.encode("utf-8")
+ ):
out.append(ptr)
# Folded span is NOT verbatim in this block's output:
# mark None so it can't seed a later contiguous match,
diff --git a/tests/test_cross_turn_dedup.py b/tests/test_cross_turn_dedup.py
index 391f87eb6..3d81fc9fa 100644
--- a/tests/test_cross_turn_dedup.py
+++ b/tests/test_cross_turn_dedup.py
@@ -472,3 +472,42 @@ def test_apply_recoverable_default_and_true_still_fold():
true_out = _apply_with_recoverable(msgs, recoverable=True)
assert "[↑" in true_out[2]["content"] # explicit recoverable -> folds
+
+
+def _five_numbered(digits):
+ return "\n".join("1" * digits + f":alpha line {i} of the file contents" for i in range(5))
+
+
+def test_long_digit_run_does_not_disable_dedup():
+ # int() raises ValueError on 4,301+ digits; that must not skip dedup.
+ for digits in (10, 4300, 4301):
+ t = _five_numbered(digits)
+ out, stats = dedup_blocks([_blk(t, 1), _blk(t, 2)])
+ assert stats["spans_folded"] == 1, digits
+ assert out[1].text != t
+
+
+def test_fold_never_longer_than_span():
+ # A 40-char 3-line span would fold into a 43-char pointer: leave it alone.
+ t = "a" * 21 + "\n" + "b" * 9 + "\n" + "c" * 8
+ assert len(t) == 40
+ out, stats = dedup_blocks([_blk(t, 1), _blk(t, 2)])
+ assert stats["spans_folded"] == 0
+ assert out[1].text == t
+ # Byte boundary: the pointer's "↑" is 3 UTF-8 bytes. A 44-char span has a
+ # 43-char pointer (shorter in characters) that is 45 bytes, so folding
+ # would grow the output; it must stay. A 46-byte span folds.
+ t44 = "a" * 25 + "\n" + "b" * 9 + "\n" + "c" * 8
+ assert len(t44) == 44
+ out, stats = dedup_blocks([_blk(t44, 1), _blk(t44, 2)])
+ assert stats["spans_folded"] == 0
+ assert out[1].text == t44
+ t46 = "a" * 27 + "\n" + "b" * 9 + "\n" + "c" * 8
+ out, stats = dedup_blocks([_blk(t46, 1), _blk(t46, 2)])
+ assert stats["spans_folded"] == 1
+ assert len(out[1].text.encode("utf-8")) < len(t46.encode("utf-8"))
+ # A long span still folds.
+ long_t = _code("", 5)
+ out, stats = dedup_blocks([_blk(long_t, 1), _blk(long_t, 2)])
+ assert stats["spans_folded"] == 1
+ assert len(out[1].text) < len(long_t)