Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(dedup): shorter fold pointer + MIN_LINES=3, dedup Anthropic list-… (#1932)

Tejas Chopra10e4829201b4
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 9ec52bf79..f036ad8b5 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -662,6 +662,60 @@ def _responses_input_to_items(input_data: Any) -> list[dict[str, Any]]:
return []
+def _dedup_responses_output_items(
+ items: list[dict[str, Any]],
+ output_types: frozenset[str],
+ count_tokens: Any = None,
+) -> tuple[int, int]:
+ """Cross-turn verbatim de-dup over Responses tool-output items (mutates in place).
+
+ A file re-read on the Responses path (e.g. Codex) comes back as a repeated
+ ``function_call_output`` whose ``output`` string carries the same file body
+ under per-call-varying ``Chunk ID`` / ``Wall time`` headers. Per-unit
+ compression cannot fold that — it is inherently cross-item. This collects the
+ tool-output ``.output`` strings in request order and runs the SAME
+ prefix-monotonic, keep-earliest ``dedup_blocks`` the chat path uses: the
+ earliest copy stays verbatim (it is the in-context reference and lives in the
+ cached prefix), later duplicates fold to a one-line pointer. Longest-span
+ matching folds the identical body and leaves the varying header intact.
+
+ Returns ``(spans_folded, tokens_saved)`` — ``tokens_saved`` is 0 when no
+ ``count_tokens`` callable is given. Never raises on malformed input.
+ """
+ try:
+ from headroom.transforms.cross_turn_dedup import DedupBlock, dedup_blocks
+ except Exception:
+ return 0, 0
+
+ locs: list[int] = []
+ blocks: list[DedupBlock] = []
+ for i, item in enumerate(items):
+ if not isinstance(item, dict) or item.get("type") not in output_types:
+ continue
+ out = item.get("output")
+ if isinstance(out, str) and out:
+ locs.append(i)
+ blocks.append(DedupBlock(text=out, turn=i, protected=False))
+ if len(blocks) < 2:
+ return 0, 0
+
+ deduped, stats = dedup_blocks(blocks)
+ if not stats.get("spans_folded"):
+ return 0, 0
+
+ tokens_saved = 0
+ for idx, orig, new in zip(locs, blocks, deduped):
+ if new.text == orig.text:
+ continue
+ if count_tokens is not None:
+ try:
+ tokens_saved += max(0, int(count_tokens(orig.text)) - int(count_tokens(new.text)))
+ except Exception:
+ pass
+ items[idx]["output"] = new.text
+ return int(stats["spans_folded"]), tokens_saved
+
+
def _openai_responses_to_sse(response: dict[str, Any]) -> list[bytes]:
"""Convert a complete Responses API JSON body into a minimal SSE stream.
@@ -1655,6 +1709,23 @@ class OpenAIHandlerMixin:
if "router:excluded:lossless" not in transforms:
transforms.append("router:excluded:lossless")
+ # Cross-turn dedup over tool-output slots (Codex/Responses re-reads): a
+ # repeated function_call_output.output folds to a one-line pointer to the
+ # earlier copy. Per-unit compression above can't do this — it is
+ # inherently cross-item. Keep-earliest + prefix-monotonic: the earlier
+ # (cached-prefix) copy is never rewritten, so appending a turn does not
+ # change cached bytes. Runs on the post-per-unit forms, mirroring the
+ # chat path (ContentRouter._cross_turn_dedup_messages runs last there too).
+ if getattr(router, "_cross_turn_dedup_enabled", False):
+ dd_folded, dd_saved = _dedup_responses_output_items(
+ updated_items, self.OPENAI_RESPONSES_OUTPUT_TYPES, tokenizer.count_text
+ )
+ if dd_folded:
+ modified = True
+ tokens_saved_total += dd_saved
+ if "router:responses_cross_turn_dedup" not in transforms:
+ transforms.append("router:responses_cross_turn_dedup")
+
_log(
"codex_compression_payload_result",
modified=modified,
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py
index 2b5d73edf..8280b630a 100644
--- a/headroom/transforms/content_router.py
+++ b/headroom/transforms/content_router.py
@@ -4235,8 +4235,26 @@ class ContentRouter(Transform):
try:
from headroom.transforms.cross_turn_dedup import DedupBlock, dedup_blocks
- locs: list[tuple[int, int | None]] = []
+ locs: list[tuple[int, int | None, int | None]] = []
dblocks: list[DedupBlock] = []
+
+ def _is_user_read_observation(idx: int) -> bool:
+ # A file read can land in a plain ``role:user`` STRING (text
+ # harnesses: the assistant emits a fenced ``cat/sed/head …`` and the
+ # output comes back as the next user turn). Fold those too, but ONLY
+ # when the preceding assistant turn actually issued a read command —
+ # never ordinary user prose. Same OUTCOME the router uses to protect
+ # reads, re-derived here on dedup's own array so indices stay
+ # self-consistent (no coupling to pre-scan positional indices).
+ for j in range(idx - 1, -1, -1):
+ rj = messages[j].get("role")
+ if rj == "assistant":
+ cmd = _fenced_shell_command(messages[j].get("content"))
+ return bool(cmd and _is_read_command(cmd))
+ if rj == "user":
+ return False
+ return False
+
for i, msg in enumerate(messages):
content = msg.get("content")
frozen = i < frozen_message_count
@@ -4244,18 +4262,44 @@ class ContentRouter(Transform):
for bidx, block in enumerate(content):
if not isinstance(block, dict) or block.get("type") != "tool_result":
continue
- text = block.get("content")
- if not isinstance(text, str) or not text:
- continue
+ tc = block.get("content")
protected = frozen or ("cache_control" in block)
- locs.append((i, bidx))
- dblocks.append(DedupBlock(text=text, turn=i, protected=protected))
- elif isinstance(content, str) and msg.get("role") == "tool":
- if not content:
- continue
- protected = frozen or ("cache_control" in msg)
- locs.append((i, None))
- dblocks.append(DedupBlock(text=content, turn=i, protected=protected))
+ if isinstance(tc, str) and tc:
+ locs.append((i, bidx, None))
+ dblocks.append(DedupBlock(text=tc, turn=i, protected=protected))
+ elif isinstance(tc, list):
+ # Anthropic tool_result carries LIST content (a `text`
+ # sub-block holds the bash/read output). The string-only
+ # path above skipped these entirely, so reads never
+ # deduped on Anthropic models. Dedup the single text
+ # sub-block (the common form); leave multi-text/mixed
+ # blocks verbatim to stay trivially lossless.
+ text_subs = [
+ (si, sub)
+ for si, sub in enumerate(tc)
+ if isinstance(sub, dict)
+ and sub.get("type") == "text"
+ and isinstance(sub.get("text"), str)
+ and sub.get("text")
+ ]
+ if len(text_subs) == 1:
+ si, sub = text_subs[0]
+ locs.append((i, bidx, si))
+ dblocks.append(
+ DedupBlock(text=sub["text"], turn=i, protected=protected)
+ )
+ elif isinstance(content, str) and content:
+ # Tool output as a STRING under any harness label: OpenAI/Kimi
+ # ``role:tool``, legacy ``role:function``, or a text-harness
+ # ``role:user`` read output. Key off the OUTCOME, not the role —
+ # ``role:user`` is gated to genuine reads so prose never folds.
+ role = msg.get("role")
+ if role in ("tool", "function") or (
+ role == "user" and _is_user_read_observation(i)
+ ):
+ protected = frozen or ("cache_control" in msg)
+ locs.append((i, None, None))
+ dblocks.append(DedupBlock(text=content, turn=i, protected=protected))
if len(dblocks) < 2:
return messages
@@ -4265,7 +4309,7 @@ class ContentRouter(Transform):
new_messages = list(messages)
touched: dict[int, dict[str, Any]] = {}
- for (mi, blk_idx), od, nd in zip(locs, dblocks, deduped):
+ for (mi, blk_idx, sub_idx), od, nd in zip(locs, dblocks, deduped):
if od.protected or nd.text == od.text:
continue
if mi not in touched:
@@ -4280,8 +4324,19 @@ class ContentRouter(Transform):
m = touched[mi]
if blk_idx is None:
m["content"] = nd.text
- else:
+ elif sub_idx is None:
m["content"][blk_idx]["content"] = nd.text
+ else:
+ # List-content tool_result: deep-copy the block + its sub-list
+ # before mutating so the input message is never touched.
+ blk = dict(m["content"][blk_idx])
+ sub_list = [
+ dict(s) if isinstance(s, dict) else s for s in blk.get("content", [])
+ ]
+ sub_list[sub_idx] = dict(sub_list[sub_idx])
+ sub_list[sub_idx]["text"] = nd.text
+ blk["content"] = sub_list
+ m["content"][blk_idx] = blk
if route_counts is not None:
route_counts["cross_turn_dedup"] = (
diff --git a/headroom/transforms/cross_turn_dedup.py b/headroom/transforms/cross_turn_dedup.py
index a8f46b301..92731725c 100644
--- a/headroom/transforms/cross_turn_dedup.py
+++ b/headroom/transforms/cross_turn_dedup.py
@@ -28,6 +28,7 @@ Pure stdlib, deterministic, never raises (returns input unchanged on any error).
from __future__ import annotations
+import re
from dataclasses import dataclass
__all__ = ["DedupBlock", "dedup_blocks", "is_prefix_monotonic"]
@@ -36,12 +37,40 @@ __all__ = ["DedupBlock", "dedup_blocks", "is_prefix_monotonic"]
# pointer. Small dups are left alone (fragmenting context is not worth it) —
# and a larger floor keeps the pointer comfortably shorter than the span it
# replaces, so a fold is always a net byte win.
-DEFAULT_MIN_LINES = 7
-DEFAULT_MIN_CHARS = 120
+DEFAULT_MIN_LINES = 3
+DEFAULT_MIN_CHARS = 40
# Cap anchor candidates examined per line so a hot line (e.g. `` return``)
# can't blow up matching. Deterministic: candidates are kept in first-seen order.
MAX_ANCHOR_CANDIDATES = 16
+# An UNPADDED leading line-number prefix: ``123:...`` or ``123<TAB>...`` at the
+# very start of the line (grep -n / sed -n / ripgrep --heading data rows). We do
+# NOT match a padded/right-aligned prefix (`` 123<TAB>`` from ``cat -n``): the
+# line must start with the digit, so ``number + delta`` re-numbers byte-exactly
+# without touching alignment padding, keeping renumbered folds strictly lossless.
+_LINENO_RE = re.compile(r"^(\d+)(:|\t)(.*)$", re.DOTALL)
+
+
+def _num_and_key(line: str) -> tuple[int | None, str, str]:
+ """Split a leading unpadded line-number.
+
+ Returns ``(number, match_key, content)``:
+ * ``number`` — the leading line number (``int``) or ``None`` if absent.
+ * ``match_key`` — the line with the leading number stripped (separator
+ kept), so two occurrences of the same content at DIFFERENT line numbers
+ share a key (e.g. ``92:foo`` and ``94:foo`` -> ``:foo``). Non-numbered
+ lines key on themselves, so exact matching is unchanged for them.
+ * ``content`` — the text after the separator, for the triviality test.
+
+ This is what makes cross-turn dedup survive an edit that renumbers a file:
+ a re-read whose line numbers all shifted by a constant still folds, and the
+ shift is carried in the pointer so the original bytes stay recoverable.
+ """
+ m = _LINENO_RE.match(line)
+ if m is None:
+ return None, line, line
+ return int(m.group(1)), m.group(2) + m.group(3), m.group(3)
+
@dataclass
class DedupBlock:
@@ -79,20 +108,29 @@ def _is_trivial(line: str) -> bool:
}
-def _pointer(span: list[str], ref_turn: int, ref_line: int) -> str:
+def _pointer(span: list[str], ref_turn: int, delta: int = 0) -> str:
"""A one-line, obviously-a-reference marker naming the in-context original.
Includes a first-line anchor so the model can locate the block it already
saw. Marker-free of any ``hash=`` retrieval token: recovery is in-context
- (the original is physically present earlier in the same request)."""
- anchor = next((ln.strip() for ln in span if ln.strip()), "")
- if len(anchor) > 80:
- anchor = anchor[:77] + "..."
- end_line = ref_line + len(span) - 1
- return (
- f"[headroom: {len(span)} lines identical to output shown earlier "
- f"(turn {ref_turn}, lines {ref_line}-{end_line}) — starts: {anchor!r}]"
- )
+ (the original is physically present earlier in the same request).
+
+ ``delta`` is the uniform line-number offset of THIS span relative to the
+ referenced original (0 for a byte-identical fold). When non-zero the span is
+ the same content re-read after an edit shifted its line numbers; the offset
+ is stated so the original the pointer names + ``delta`` reconstructs this
+ span's exact numbered bytes (recovery stays in-window)."""
+ # Compact form (~35c vs the old ~100c): the ~100c pointer made sub-7-line
+ # folds net-negative, so the abundant short (2-4 line) re-read repeats were
+ # left uncompressed. Trimming the pointer + MIN_LINES=3 lets those pay off
+ # (~4.3% -> ~6% lossless on Opus). Still no hash= token (in-context recovery)
+ # and keeps a short first-line anchor so the model can locate the original.
+ anchor = next((_num_and_key(ln)[2].strip() for ln in span if ln.strip()), "")
+ if len(anchor) > 20:
+ anchor = anchor[:17] + "..."
+ if delta:
+ return f"[↑{len(span)}L same as msg {ref_turn} {delta:+d}L: {anchor!r}]"
+ return f"[↑{len(span)}L same as msg {ref_turn}: {anchor!r}]"
def _index_lines(
@@ -104,11 +142,15 @@ def _index_lines(
Keeps first-seen order and caps the candidate list per line. Only VERBATIM
(surviving) lines should be passed here — never the lines of a span that was
- replaced by a pointer."""
+ replaced by a pointer. Indexed by the line-number-stripped ``match_key`` so a
+ later renumbered re-read of the same content finds this anchor."""
for li, ln in enumerate(lines):
- if ln is None or _is_trivial(ln):
+ if ln is None:
+ continue
+ _, key, content = _num_and_key(ln)
+ if _is_trivial(content):
continue
- bucket = anchor_index.setdefault(ln, [])
+ bucket = anchor_index.setdefault(key, [])
if len(bucket) < MAX_ANCHOR_CANDIDATES:
bucket.append((block_pos, li))
@@ -118,34 +160,57 @@ def _longest_match(
start: int,
anchor_index: dict[str, list[tuple[int, int]]],
corpus: list[list[str | None]],
-) -> tuple[int, int, int] | None:
- """Longest contiguous run in ``cur`` starting at ``start`` that appears
- verbatim inside a single earlier block. Returns (length, block_pos,
- ref_line_idx) or None. ``corpus[block_pos]`` holds that block's VERBATIM
- lines (``None`` where a span was already folded, which breaks contiguity)."""
- anchor = cur[start]
- candidates = anchor_index.get(anchor)
+) -> tuple[int, int, int, int] | None:
+ """Longest contiguous run in ``cur`` starting at ``start`` whose content
+ appears in a single earlier block — allowing a UNIFORM line-number shift.
+
+ Returns ``(length, block_pos, ref_line_idx, delta)`` or ``None``. ``delta``
+ is the constant line-number offset of the run vs the referenced original
+ (0 for a byte-identical fold). ``corpus[block_pos]`` holds that block's
+ VERBATIM lines (``None`` where a span was already folded, which breaks
+ contiguity).
+
+ A run extends while each pair shares a ``match_key`` (content modulo the
+ leading line number) AND every numbered pair has the SAME numeric offset —
+ so an edit that shifts a file's numbers by a constant still folds, but a
+ non-uniform change (edit *inside* the span) ends the run at the divergence.
+ Non-numbered lines must match exactly (they carry no offset)."""
+ _, anchor_key, _ = _num_and_key(cur[start])
+ candidates = anchor_index.get(anchor_key)
if not candidates:
return None
best_len = 0
best_bp = best_li = -1
+ best_delta = 0
for bp, li in candidates:
block_lines = corpus[bp]
k = 0
- while (
- start + k < len(cur)
- and li + k < len(block_lines)
- and block_lines[li + k] is not None
- and cur[start + k] == block_lines[li + k]
- ):
+ delta: int | None = None
+ while start + k < len(cur) and li + k < len(block_lines):
+ ca = cur[start + k]
+ cb = block_lines[li + k]
+ if cb is None: # end of a folded block in the corpus — run ends here
+ break
+ na, ka, _ = _num_and_key(ca)
+ nb, kb, _ = _num_and_key(cb)
+ if ka != kb:
+ break # different content -> run ends
+ if na is not None and nb is not None:
+ d = na - nb
+ if delta is None:
+ delta = d
+ elif delta != d:
+ break # non-uniform shift (edit inside span) -> run ends
+ elif ca != cb:
+ break # non-numbered line must match exactly
k += 1
# Deterministic tie-break: longer wins; on ties keep the earliest
# (smallest block_pos, then line) already held in best_*.
if k > best_len:
- best_len, best_bp, best_li = k, bp, li
+ best_len, best_bp, best_li, best_delta = k, bp, li, (delta or 0)
if best_len == 0:
return None
- return best_len, best_bp, best_li
+ return best_len, best_bp, best_li, best_delta
def dedup_blocks(
@@ -186,7 +251,7 @@ def dedup_blocks(
span_text = "\n".join(span)
if len(span_text) >= min_chars:
ref_turn = blocks[m[1]].turn
- ptr = _pointer(span, ref_turn, m[2])
+ ptr = _pointer(span, ref_turn, m[3])
out.append(ptr)
# Folded span is NOT verbatim in this block's output:
# mark None so it can't seed a later contiguous match,
diff --git a/tests/test_cross_turn_dedup.py b/tests/test_cross_turn_dedup.py
index d4c35b7a4..4949a5633 100644
--- a/tests/test_cross_turn_dedup.py
+++ b/tests/test_cross_turn_dedup.py
@@ -4,11 +4,15 @@ import re
from headroom.transforms.cross_turn_dedup import (
DedupBlock,
+ _num_and_key,
dedup_blocks,
is_prefix_monotonic,
)
-_PTR_RE = re.compile(r"identical to output shown earlier \(turn (\d+), lines (\d+)-(\d+)\)")
+# Compact fold pointer: ``[↑<N>L same as msg <ref>[ <±delta>L]: '<anchor>']`` —
+# span length + referenced msg + optional line-number offset + a truncated
+# first-line anchor (no explicit line range; recovery locates the span by anchor).
+_FOLD_RE = re.compile(r"\[↑(\d+)L same as msg (\d+)(?: ([+-]\d+)L)?: '([^']*)'\]")
def _blk(text, turn, protected=False):
@@ -24,20 +28,33 @@ def _code(prefix, n):
def _reconstruct(orig_blocks, out_blocks):
- """Replace each pointer with the referenced turn's original lines and assert
- it reproduces the original block — proves references are faithful & in-context."""
+ """Replace each fold pointer with the referenced msg's original lines and assert
+ it reproduces the original block — proves references are faithful & in-context.
+
+ The compact pointer names the ref msg + span length + a first-line anchor (not
+ an explicit line range), so recovery locates the span by its anchor in the
+ referenced message and takes ``<N>`` lines. (All test spans are unnumbered, so
+ delta is always absent; a non-zero delta would renumber on the way out.)"""
by_turn = {b.turn: b.text.split("\n") for b in orig_blocks}
+
+ def _content(line):
+ return _num_and_key(line)[2].strip()
+
for orig, out in zip(orig_blocks, out_blocks):
if orig.protected:
assert out.text == orig.text
continue
rebuilt = []
for line in out.text.split("\n"):
- m = _PTR_RE.search(line)
- if m and line.startswith("[headroom:"):
- t, a, b = int(m.group(1)), int(m.group(2)), int(m.group(3))
- assert t < orig.turn, "reference must point to an EARLIER turn"
- rebuilt.extend(by_turn[t][a : b + 1])
+ m = _FOLD_RE.search(line)
+ if m and line.lstrip().startswith("[↑"):
+ assert m.group(3) is None, "unexpected delta for an unnumbered span"
+ n, ref, anchor = int(m.group(1)), int(m.group(2)), m.group(4)
+ assert ref < orig.turn, "reference must point to an EARLIER msg"
+ core = anchor[:-3] if anchor.endswith("...") else anchor
+ ref_lines = by_turn[ref]
+ idx = next(i for i, rl in enumerate(ref_lines) if _content(rl).startswith(core))
+ rebuilt.extend(ref_lines[idx : idx + n])
else:
rebuilt.append(line)
assert "\n".join(rebuilt) == orig.text, f"turn {orig.turn} not faithfully reconstructable"
@@ -48,7 +65,7 @@ def test_verbatim_reread_is_folded_keep_earliest():
blocks = [_blk(f"cat merge.py\n{span}\ntail", 1), _blk(f"sed run\n{span}\nmore", 5)]
out, stats = dedup_blocks(blocks)
assert out[0].text == blocks[0].text # earliest untouched
- assert "[headroom:" in out[1].text # later occurrence folded
+ assert "[↑" in out[1].text # later occurrence folded
assert stats["spans_folded"] == 1
_reconstruct(blocks, out)
@@ -65,7 +82,7 @@ def test_cache_safety_prefix_monotonic():
def test_below_min_lines_not_folded():
- span = _code("", 3) # below min_lines (7)
+ span = _code("", 2) # below min_lines (3)
blocks = [_blk(span, 1), _blk(span, 2)]
out, stats = dedup_blocks(blocks)
assert stats["spans_folded"] == 0
@@ -95,7 +112,7 @@ def test_protected_block_not_rewritten_but_is_reference_target():
]
out, stats = dedup_blocks(blocks)
assert out[0].text == blocks[0].text
- assert "[headroom:" in out[1].text
+ assert "[↑" in out[1].text
_reconstruct(blocks, out)
@@ -159,8 +176,8 @@ def test_apply_dedups_reread_and_keeps_prefix_stable():
# Dedup fired on the later re-read (turn t2), earliest (t1) untouched.
later = out2[-1]["content"][0]["content"]
earlier = out2[2]["content"][0]["content"]
- assert "[headroom:" in later
- assert "[headroom:" not in earlier and span in earlier
+ assert "[↑" in later
+ assert "[↑" not in earlier and span in earlier
# CACHE-SAFETY at the router level: appending turn t2 did NOT change any
# earlier message's emitted bytes → the prompt-cache prefix is stable.
@@ -189,7 +206,7 @@ def test_apply_no_dedup_when_flag_off():
r = ContentRouter(ContentRouterConfig(lossless=True, enable_cross_turn_dedup=False))
out = r.apply(copy.deepcopy(msgs), _mk_tok()).messages
joined = "".join(b["content"] for m in out for b in m["content"] if isinstance(b, dict))
- assert "[headroom:" not in joined
+ assert "[↑" not in joined
def test_apply_dedup_runs_in_ccr_mode_too():
@@ -214,4 +231,77 @@ def test_apply_dedup_runs_in_ccr_mode_too():
for b in m["content"]
if isinstance(b, dict)
)
- assert "[headroom:" in joined # dedup fired despite lossless=False
+ assert "[↑" in joined # dedup fired despite lossless=False
+
+
+# --------------------------------------------------------------------------
+# No dangling reference: dedup folds only against content present in the array
+# it processes (it runs last, over the final sent messages). If compaction
+# already removed the original, there is nothing earlier to reference → verbatim.
+# --------------------------------------------------------------------------
+def test_no_fold_when_original_absent_fallback():
+ span = _code("", 8)
+ # Only the LATER read survives; its original was compacted out of the array.
+ blocks = [_blk("unrelated log\n" + _code("z", 8), 1), _blk(f"sed\n{span}\nmore", 5)]
+ out, stats = dedup_blocks(blocks)
+ assert stats["spans_folded"] == 0
+ assert out[1].text == blocks[1].text and "[↑" not in out[1].text
+
+
+# --------------------------------------------------------------------------
+# Shape-agnostic CODE-READ coverage: a file read gets deduped wherever it lands
+# — role:tool, role:function, or a text-harness role:user string — keyed off the
+# read OUTCOME, never ordinary user prose.
+# --------------------------------------------------------------------------
+def _dedup_only(messages):
+ """Run ONLY the cross-turn dedup pass (no per-block compression) on a raw
+ message array, isolating extraction + fold across message shapes."""
+ from headroom.transforms.content_router import ContentRouter, ContentRouterConfig
+
+ r = ContentRouter(ContentRouterConfig(lossless=True, enable_cross_turn_dedup=True))
+ return r._cross_turn_dedup_messages(messages, 0, [], None)
+
+
+def _readspan():
+ return "\n".join(f" result_{i} = compute_overdraft(business_id={i})" for i in range(10))
+
+
+def test_dedup_folds_user_string_read_observation():
+ # Text-harness shape: the read output arrives as a role:user STRING after an
+ # assistant fenced `cat`. The later identical read folds; earliest stays intact.
+ span = _readspan()
+ asst = {"role": "assistant", "content": "```bash\ncat report.py\n```"}
+ msgs = [
+ asst,
+ {"role": "user", "content": span}, # read #1 (reference target)
+ asst,
+ {"role": "user", "content": span}, # read #2 (duplicate) -> folds
+ ]
+ out = _dedup_only(msgs)
+ assert "[↑" in out[3]["content"]
+ assert out[1]["content"] == span
+
+
+def test_dedup_does_not_fold_plain_user_prose():
+ # A duplicated ORDINARY user message (no preceding read command) must stay
+ # verbatim — user intent is never folded.
+ prose = "\n".join(f"please also make sure case {i} is handled carefully" for i in range(10))
+ msgs = [
+ {"role": "user", "content": prose},
+ {"role": "assistant", "content": "understood"},
+ {"role": "user", "content": prose}, # duplicate prose -> must NOT fold
+ ]
+ out = _dedup_only(msgs)
+ assert "[↑" not in out[2]["content"] and out[2]["content"] == prose
+
+
+def test_dedup_folds_role_function_output():
+ # Legacy OpenAI role:function tool output — same operation, different label.
+ span = _readspan()
+ msgs = [
+ {"role": "function", "name": "read_file", "content": span},
+ {"role": "assistant", "content": "let me re-check"},
+ {"role": "function", "name": "read_file", "content": span}, # dup -> folds
+ ]
+ out = _dedup_only(msgs)
+ assert "[↑" in out[2]["content"]
diff --git a/tests/test_responses_cross_turn_dedup.py b/tests/test_responses_cross_turn_dedup.py
new file mode 100644
index 000000000..276531f61
--- /dev/null
+++ b/tests/test_responses_cross_turn_dedup.py
@@ -0,0 +1,114 @@
+"""Cross-turn dedup on the OpenAI Responses path (Codex ``function_call_output``).
+
+Fixtures mirror a REAL Codex run captured through the headroom proxy: a file read
+returns as ``{"type":"function_call_output","call_id":...,"output":"Chunk ID: …\\n
+Wall time: …\\nProcess exited with code 0\\nOriginal token count: …\\nOutput:\\n
+<FILE BODY>\\n"}``. The ``Chunk ID`` / ``Wall time`` header varies per call, so a
+whole-block match never fires — longest-span matching must fold the identical
+body and leave the varying header verbatim.
+"""
+
+from __future__ import annotations
+
+from headroom.proxy.handlers.openai import (
+ _RESPONSES_OUTPUT_ITEM_TYPES,
+ _dedup_responses_output_items,
+)
+
+BODY = (
+ "def paginate_orders(items, page, page_size):\n"
+ ' """Return one page of orders."""\n'
+ " start = page * page_size\n"
+ " end = start + page_size + 1 # off-by-one: should be start + page_size\n"
+ " return items[start:end]\n"
+ "\n\n"
+ 'SERVICE_TAG = "svc-03e8-tag"\n'
+ "\n\n"
+ "def compute_overdraft(business_id, amount):\n"
+ " fee = amount * 0.05\n"
+ ' return {"business_id": business_id, "fee": fee, "tag": SERVICE_TAG}\n'
+)
+
+
+def _wrap(chunk_id: str, wall: str) -> str:
+ # Codex's exec_command wrapper — the header lines vary call to call.
+ return (
+ f"Chunk ID: {chunk_id}\n"
+ f"Wall time: {wall} seconds\n"
+ "Process exited with code 0\n"
+ "Original token count: 97\n"
+ "Output:\n"
+ )
+
+
+def _read_output(call_id: str, chunk_id: str, wall: str) -> dict:
+ return {
+ "type": "function_call_output",
+ "call_id": call_id,
+ "output": _wrap(chunk_id, wall) + BODY,
+ }
+
+
+def _read_call(call_id: str) -> dict:
+ return {
+ "type": "function_call",
+ "name": "exec_command",
+ "arguments": '{"cmd":"cat buggy.py","workdir":"/tmp"}',
+ "call_id": call_id,
+ }
+
+
+def test_repeated_codex_read_folds_body_keeps_varying_header():
+ items = [
+ {"role": "user", "content": "find the bug"},
+ _read_call("c1"),
+ _read_output("c1", "492f0f", "0.0000"), # read #1 (reference)
+ {
+ "type": "message",
+ "role": "assistant",
+ "content": [{"type": "output_text", "text": "re-reading"}],
+ },
+ _read_call("c2"),
+ _read_output("c2", "a1b2c3", "0.0100"), # read #2 (duplicate) -> body folds
+ ]
+ folded, saved = _dedup_responses_output_items(
+ items, _RESPONSES_OUTPUT_ITEM_TYPES, count_tokens=len
+ )
+
+ assert folded == 1
+ assert saved > 0
+ # earliest read: byte-identical (reference target, sits in the cached prefix)
+ assert items[2]["output"] == _wrap("492f0f", "0.0000") + BODY
+ # later read: identical body folded to a pointer; the per-call header stays verbatim
+ later = items[5]["output"]
+ assert "[↑" in later
+ assert later.startswith("Chunk ID: a1b2c3\nWall time: 0.0100")
+ assert "def paginate_orders" not in later # body folded away
+ # lossless: the folded body is still fully present earlier in the request
+ assert "def paginate_orders" in items[2]["output"]
+
+
+def test_single_read_does_not_fold():
+ items = [_read_call("c1"), _read_output("c1", "492f0f", "0.0000")]
+ folded, saved = _dedup_responses_output_items(
+ items, _RESPONSES_OUTPUT_ITEM_TYPES, count_tokens=len
+ )
+ assert folded == 0 and saved == 0
+ assert items[1]["output"] == _wrap("492f0f", "0.0000") + BODY
+
+
+def test_non_output_items_untouched():
+ # A duplicated MESSAGE (not a tool output) must never fold — only output
+ # items are eligible.
+ msg = {"role": "user", "content": BODY}
+ items = [dict(msg), {"type": "message", "role": "assistant", "content": "ok"}, dict(msg)]
+ folded, _ = _dedup_responses_output_items(items, _RESPONSES_OUTPUT_ITEM_TYPES)
+ assert folded == 0
+ assert items[2]["content"] == BODY
+
+
+def test_never_raises_on_malformed():
+ # Defensive: junk items must not blow up the request path.
+ items = [{"type": "function_call_output"}, {"type": "function_call_output", "output": None}, 42]
+ folded, saved = _dedup_responses_output_items(items, _RESPONSES_OUTPUT_ITEM_TYPES) # type: ignore[arg-type]
+ assert (folded, saved) == (0, 0)