Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(dedup): bound line-number digit run and never fold into a longer pointer (#4488)
HetCreep21e8867504df
diff --git a/headroom/transforms/cross_turn_dedup.py b/headroom/transforms/cross_turn_dedup.pyindex af6cd6697..108c49108 100644--- a/headroom/transforms/cross_turn_dedup.py+++ b/headroom/transforms/cross_turn_dedup.py@@ -39,9 +39,9 @@ from ..log_safety import describe_exception, safe_idlogger = logging.getLogger(__name__)# A run must be at least this many lines AND this many chars to be worth a-# pointer. Small dups are left alone (fragmenting context is not worth it) —-# and a larger floor keeps the pointer comfortably shorter than the span it-# replaces, so a fold is always a net byte win.+# pointer. Small dups are left alone (fragmenting context is not worth it).+# The floor alone does not make a fold a net byte win (the pointer can still be+# as long as a short span); dedup_blocks checks the UTF-8 sizes before folding.DEFAULT_MIN_LINES = 3DEFAULT_MIN_CHARS = 40# Cap anchor candidates examined per line so a hot line (e.g. `` return``)@@ -54,7 +54,7 @@ MAX_ANCHOR_CANDIDATES = 16# line must start with the digit, so ``number + delta`` re-numbers byte-exactly# without touching alignment padding, keeping renumbered folds strictly lossless.#-# ``[1-9]\d*`` (not ``\d+``) is load-bearing: a LEADING-ZERO run (``08:00:01`` in+# The leading ``[1-9]`` (not ``\d``) is load-bearing: a LEADING-ZERO run (``08:00:01`` in# a timestamped log, ``007:...``) is not an unpadded grep/sed line number, and# recovery is ``str(int(number) + delta)``, which drops the zero pad — ``int("08")# + 1`` renders ``"9"``, not ``"09"``. Matching those would fold them under a@@ -63,7 +63,11 @@ MAX_ANCHOR_CANDIDATES = 16# fold only on an EXACT match (delta 0) — the "prefer false negatives" posture.# Real grep -n / sed -n / rg -n numbers never carry a leading zero, so the# intended renumber-fold feature is unaffected.-_LINENO_RE = re.compile(r"^([1-9]\d*)(:|\t)(.*)$", re.DOTALL)+#+# ``\d{0,17}`` bounds the run at 18 digits: Python refuses ``int()`` on a run of+# 4,301+ digits (ValueError), which would abort dedup for the whole request. A+# longer run is not a real line number, so the line stays non-numbered.+_LINENO_RE = re.compile(r"^([1-9]\d{0,17})(:|\t)(.*)$", re.DOTALL)def _num_and_key(line: str) -> tuple[int | None, str, str]:@@ -264,9 +268,12 @@ def dedup_blocks(if m is not None and m[0] >= min_lines:span = lines[i : i + m[0]]span_text = "\n".join(span)- if len(span_text) >= min_chars:- ref_turn = blocks[m[1]].turn- ptr = _pointer(span, ref_turn, m[3])+ ptr = _pointer(span, blocks[m[1]].turn, m[3])+ # Never fold when the pointer is not shorter than the span, in+ # UTF-8 bytes ("↑" is 3 bytes, so character counts mislead).+ if len(span_text) >= min_chars and len(ptr.encode("utf-8")) < len(+ span_text.encode("utf-8")+ ):out.append(ptr)# Folded span is NOT verbatim in this block's output:# mark None so it can't seed a later contiguous match,diff --git a/tests/test_cross_turn_dedup.py b/tests/test_cross_turn_dedup.pyindex 391f87eb6..3d81fc9fa 100644--- a/tests/test_cross_turn_dedup.py+++ b/tests/test_cross_turn_dedup.py@@ -472,3 +472,42 @@ def test_apply_recoverable_default_and_true_still_fold():true_out = _apply_with_recoverable(msgs, recoverable=True)assert "[↑" in true_out[2]["content"] # explicit recoverable -> folds+++def _five_numbered(digits):+ return "\n".join("1" * digits + f":alpha line {i} of the file contents" for i in range(5))+++def test_long_digit_run_does_not_disable_dedup():+ # int() raises ValueError on 4,301+ digits; that must not skip dedup.+ for digits in (10, 4300, 4301):+ t = _five_numbered(digits)+ out, stats = dedup_blocks([_blk(t, 1), _blk(t, 2)])+ assert stats["spans_folded"] == 1, digits+ assert out[1].text != t+++def test_fold_never_longer_than_span():+ # A 40-char 3-line span would fold into a 43-char pointer: leave it alone.+ t = "a" * 21 + "\n" + "b" * 9 + "\n" + "c" * 8+ assert len(t) == 40+ out, stats = dedup_blocks([_blk(t, 1), _blk(t, 2)])+ assert stats["spans_folded"] == 0+ assert out[1].text == t+ # Byte boundary: the pointer's "↑" is 3 UTF-8 bytes. A 44-char span has a+ # 43-char pointer (shorter in characters) that is 45 bytes, so folding+ # would grow the output; it must stay. A 46-byte span folds.+ t44 = "a" * 25 + "\n" + "b" * 9 + "\n" + "c" * 8+ assert len(t44) == 44+ out, stats = dedup_blocks([_blk(t44, 1), _blk(t44, 2)])+ assert stats["spans_folded"] == 0+ assert out[1].text == t44+ t46 = "a" * 27 + "\n" + "b" * 9 + "\n" + "c" * 8+ out, stats = dedup_blocks([_blk(t46, 1), _blk(t46, 2)])+ assert stats["spans_folded"] == 1+ assert len(out[1].text.encode("utf-8")) < len(t46.encode("utf-8"))+ # A long span still folds.+ long_t = _code("", 5)+ out, stats = dedup_blocks([_blk(long_t, 1), _blk(long_t, 2)])+ assert stats["spans_folded"] == 1+ assert len(out[1].text) < len(long_t)