Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(transforms/cross-turn-dedup): don't renumber-fold zero-padded line prefixes (#2369)
Ehsanf4070c44cbd6
diff --git a/headroom/transforms/cross_turn_dedup.py b/headroom/transforms/cross_turn_dedup.pyindex 92731725c..424816667 100644--- a/headroom/transforms/cross_turn_dedup.py+++ b/headroom/transforms/cross_turn_dedup.py@@ -48,7 +48,17 @@ MAX_ANCHOR_CANDIDATES = 16# NOT match a padded/right-aligned prefix (`` 123<TAB>`` from ``cat -n``): the# line must start with the digit, so ``number + delta`` re-numbers byte-exactly# without touching alignment padding, keeping renumbered folds strictly lossless.-_LINENO_RE = re.compile(r"^(\d+)(:|\t)(.*)$", re.DOTALL)+#+# ``[1-9]\d*`` (not ``\d+``) is load-bearing: a LEADING-ZERO run (``08:00:01`` in+# a timestamped log, ``007:...``) is not an unpadded grep/sed line number, and+# recovery is ``str(int(number) + delta)``, which drops the zero pad — ``int("08")+# + 1`` renders ``"9"``, not ``"09"``. Matching those would fold them under a+# uniform delta and the round-trip would NOT be byte-exact, breaking the strictly+# lossless promise above. Excluding them keeps such lines non-numbered, so they+# fold only on an EXACT match (delta 0) — the "prefer false negatives" posture.+# Real grep -n / sed -n / rg -n numbers never carry a leading zero, so the+# intended renumber-fold feature is unaffected.+_LINENO_RE = re.compile(r"^([1-9]\d*)(:|\t)(.*)$", re.DOTALL)def _num_and_key(line: str) -> tuple[int | None, str, str]:diff --git a/tests/test_cross_turn_dedup.py b/tests/test_cross_turn_dedup.pyindex 4949a5633..53cd9b13a 100644--- a/tests/test_cross_turn_dedup.py+++ b/tests/test_cross_turn_dedup.py@@ -129,6 +129,100 @@ def test_info_preserving_reconstruction_multiref():_reconstruct(blocks, out)+# --------------------------------------------------------------------------+# Numbered (renumber-fold) path: a leading line-number lets the same content+# fold across a uniform shift, with the offset carried in the pointer so the+# original bytes recover as ``str(int(number) + delta)``. That recovery is+# byte-exact ONLY for UNPADDED numbers: a leading-zero prefix (a timestamped+# log row, ``08:00:01``) loses its pad on renumber (``int("08") + 1`` -> ``"9"``,+# not ``"09"``), so it must not fold under a delta. The helper below renumbers+# exactly as the module documents, so the round-trip assertion is faithful.+# --------------------------------------------------------------------------+def _reconstruct_numbered(orig_blocks, out_blocks):+ """Like ``_reconstruct`` but honours a non-zero delta: recover each folded+ span from the referenced msg, renumbering leading line numbers by the stated+ offset (``str(int(number) + delta) + key``), then assert byte-exact bytes."""+ by_turn = {b.turn: b.text.split("\n") for b in orig_blocks}++ def _content(line):+ return _num_and_key(line)[2].strip()++ for orig, out in zip(orig_blocks, out_blocks):+ if orig.protected:+ assert out.text == orig.text+ continue+ rebuilt = []+ for line in out.text.split("\n"):+ m = _FOLD_RE.search(line)+ if m and line.lstrip().startswith("[↑"):+ n, ref = int(m.group(1)), int(m.group(2))+ delta = int(m.group(3)) if m.group(3) else 0+ anchor = m.group(4)+ assert ref < orig.turn, "reference must point to an EARLIER msg"+ core = anchor[:-3] if anchor.endswith("...") else anchor+ ref_lines = by_turn[ref]+ idx = next(i for i, rl in enumerate(ref_lines) if _content(rl).startswith(core))+ for rl in ref_lines[idx : idx + n]:+ num, key, _c = _num_and_key(rl) # key keeps the separator+ rebuilt.append(f"{num + delta}{key}" if (num is not None and delta) else rl)+ else:+ rebuilt.append(line)+ assert "\n".join(rebuilt) == orig.text, f"turn {orig.turn} not faithfully reconstructable"+++def _log(prefixes):+ # Timestamped probe rows: identical content, distinct LEADING-ZERO hour.+ return "\n".join(f"{p}:00:01 probe ok latency=12ms region=us-east-1" for p in prefixes)+++def test_zero_padded_prefix_not_folded_lossily():+ # A leading-zero numeric prefix shifted by a uniform +1 looks like a renumber+ # (keys match, delta is uniform), but recovery via int(number)+delta drops the+ # pad, so the fold would NOT round-trip. It must be left verbatim instead.+ blocks = [+ _blk("probe window A\n" + _log(["06", "07", "08"]) + "\ndone A", 1),+ _blk("probe window B\n" + _log(["07", "08", "09"]) + "\ndone B", 5),+ ]+ out, stats = dedup_blocks(blocks)+ assert stats["spans_folded"] == 0+ assert out[1].text == blocks[1].text and "[↑" not in out[1].text+ _reconstruct_numbered(blocks, out) # trivially exact: nothing folded+++def test_unpadded_renumber_still_folds_and_recovers_exactly():+ # The intended feature: an UNPADDED grep -n / sed -n read re-displayed after an+ # edit shifted every line number by a constant still folds, and the pointer's+ # delta recovers the exact numbered bytes. The fix must not regress this.+ span = [+ " result_0 = compute_overdraft(business_id=0, amount=0)",+ " result_1 = compute_overdraft(business_id=1, amount=100)",+ " result_2 = compute_overdraft(business_id=2, amount=200)",+ ]+ b1 = "read A\n" + "\n".join(f"{10 + i}:{s}" for i, s in enumerate(span)) + "\nend A"+ b2 = "read B\n" + "\n".join(f"{15 + i}:{s}" for i, s in enumerate(span)) + "\nend B"+ blocks = [_blk(b1, 1), _blk(b2, 3)]+ out, stats = dedup_blocks(blocks)+ assert stats["spans_folded"] == 1+ assert "+5L" in out[1].text # uniform +5 shift carried in the pointer+ _reconstruct_numbered(blocks, out)+++def test_padded_content_exact_redisplay_still_folds():+ # Surgical-scope guard: the fix only blocks the LOSSY renumbered fold. The same+ # zero-padded rows re-displayed VERBATIM (delta 0) are still a byte-identical+ # fold and must continue to compress.+ log = _log(["06", "07", "08"])+ blocks = [+ _blk("probe window A\n" + log + "\ndone A", 1),+ _blk("re-check\n" + log + "\ntail", 4),+ ]+ out, stats = dedup_blocks(blocks)+ assert stats["spans_folded"] == 1+ m = _FOLD_RE.search(out[1].text)+ assert m is not None and m.group(3) is None # folded, delta-free (byte-identical)+ _reconstruct_numbered(blocks, out)++# --------------------------------------------------------------------------# Integration: full router.apply() path (content-block tool_result format)# --------------------------------------------------------------------------