Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(dedup): shorter fold pointer + MIN_LINES=3, dedup Anthropic list-… (#1932)
Tejas Chopra10e4829201b4
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 9ec52bf79..f036ad8b5 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -662,6 +662,60 @@ def _responses_input_to_items(input_data: Any) -> list[dict[str, Any]]:return []+def _dedup_responses_output_items(+ items: list[dict[str, Any]],+ output_types: frozenset[str],+ count_tokens: Any = None,+) -> tuple[int, int]:+ """Cross-turn verbatim de-dup over Responses tool-output items (mutates in place).++ A file re-read on the Responses path (e.g. Codex) comes back as a repeated+ ``function_call_output`` whose ``output`` string carries the same file body+ under per-call-varying ``Chunk ID`` / ``Wall time`` headers. Per-unit+ compression cannot fold that — it is inherently cross-item. This collects the+ tool-output ``.output`` strings in request order and runs the SAME+ prefix-monotonic, keep-earliest ``dedup_blocks`` the chat path uses: the+ earliest copy stays verbatim (it is the in-context reference and lives in the+ cached prefix), later duplicates fold to a one-line pointer. Longest-span+ matching folds the identical body and leaves the varying header intact.++ Returns ``(spans_folded, tokens_saved)`` — ``tokens_saved`` is 0 when no+ ``count_tokens`` callable is given. Never raises on malformed input.+ """+ try:+ from headroom.transforms.cross_turn_dedup import DedupBlock, dedup_blocks+ except Exception:+ return 0, 0++ locs: list[int] = []+ blocks: list[DedupBlock] = []+ for i, item in enumerate(items):+ if not isinstance(item, dict) or item.get("type") not in output_types:+ continue+ out = item.get("output")+ if isinstance(out, str) and out:+ locs.append(i)+ blocks.append(DedupBlock(text=out, turn=i, protected=False))+ if len(blocks) < 2:+ return 0, 0++ deduped, stats = dedup_blocks(blocks)+ if not stats.get("spans_folded"):+ return 0, 0++ tokens_saved = 0+ for idx, orig, new in zip(locs, blocks, deduped):+ if new.text == orig.text:+ continue+ if count_tokens is not None:+ try:+ tokens_saved += max(0, int(count_tokens(orig.text)) - int(count_tokens(new.text)))+ except Exception:+ pass+ items[idx]["output"] = new.text+ return int(stats["spans_folded"]), tokens_saved++def _openai_responses_to_sse(response: dict[str, Any]) -> list[bytes]:"""Convert a complete Responses API JSON body into a minimal SSE stream.@@ -1655,6 +1709,23 @@ class OpenAIHandlerMixin:if "router:excluded:lossless" not in transforms:transforms.append("router:excluded:lossless")+ # Cross-turn dedup over tool-output slots (Codex/Responses re-reads): a+ # repeated function_call_output.output folds to a one-line pointer to the+ # earlier copy. Per-unit compression above can't do this — it is+ # inherently cross-item. Keep-earliest + prefix-monotonic: the earlier+ # (cached-prefix) copy is never rewritten, so appending a turn does not+ # change cached bytes. Runs on the post-per-unit forms, mirroring the+ # chat path (ContentRouter._cross_turn_dedup_messages runs last there too).+ if getattr(router, "_cross_turn_dedup_enabled", False):+ dd_folded, dd_saved = _dedup_responses_output_items(+ updated_items, self.OPENAI_RESPONSES_OUTPUT_TYPES, tokenizer.count_text+ )+ if dd_folded:+ modified = True+ tokens_saved_total += dd_saved+ if "router:responses_cross_turn_dedup" not in transforms:+ transforms.append("router:responses_cross_turn_dedup")+_log("codex_compression_payload_result",modified=modified,diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.pyindex 2b5d73edf..8280b630a 100644--- a/headroom/transforms/content_router.py+++ b/headroom/transforms/content_router.py@@ -4235,8 +4235,26 @@ class ContentRouter(Transform):try:from headroom.transforms.cross_turn_dedup import DedupBlock, dedup_blocks- locs: list[tuple[int, int | None]] = []+ locs: list[tuple[int, int | None, int | None]] = []dblocks: list[DedupBlock] = []++ def _is_user_read_observation(idx: int) -> bool:+ # A file read can land in a plain ``role:user`` STRING (text+ # harnesses: the assistant emits a fenced ``cat/sed/head …`` and the+ # output comes back as the next user turn). Fold those too, but ONLY+ # when the preceding assistant turn actually issued a read command —+ # never ordinary user prose. Same OUTCOME the router uses to protect+ # reads, re-derived here on dedup's own array so indices stay+ # self-consistent (no coupling to pre-scan positional indices).+ for j in range(idx - 1, -1, -1):+ rj = messages[j].get("role")+ if rj == "assistant":+ cmd = _fenced_shell_command(messages[j].get("content"))+ return bool(cmd and _is_read_command(cmd))+ if rj == "user":+ return False+ return False+for i, msg in enumerate(messages):content = msg.get("content")frozen = i < frozen_message_count@@ -4244,18 +4262,44 @@ class ContentRouter(Transform):for bidx, block in enumerate(content):if not isinstance(block, dict) or block.get("type") != "tool_result":continue- text = block.get("content")- if not isinstance(text, str) or not text:- continue+ tc = block.get("content")protected = frozen or ("cache_control" in block)- locs.append((i, bidx))- dblocks.append(DedupBlock(text=text, turn=i, protected=protected))- elif isinstance(content, str) and msg.get("role") == "tool":- if not content:- continue- protected = frozen or ("cache_control" in msg)- locs.append((i, None))- dblocks.append(DedupBlock(text=content, turn=i, protected=protected))+ if isinstance(tc, str) and tc:+ locs.append((i, bidx, None))+ dblocks.append(DedupBlock(text=tc, turn=i, protected=protected))+ elif isinstance(tc, list):+ # Anthropic tool_result carries LIST content (a `text`+ # sub-block holds the bash/read output). The string-only+ # path above skipped these entirely, so reads never+ # deduped on Anthropic models. Dedup the single text+ # sub-block (the common form); leave multi-text/mixed+ # blocks verbatim to stay trivially lossless.+ text_subs = [+ (si, sub)+ for si, sub in enumerate(tc)+ if isinstance(sub, dict)+ and sub.get("type") == "text"+ and isinstance(sub.get("text"), str)+ and sub.get("text")+ ]+ if len(text_subs) == 1:+ si, sub = text_subs[0]+ locs.append((i, bidx, si))+ dblocks.append(+ DedupBlock(text=sub["text"], turn=i, protected=protected)+ )+ elif isinstance(content, str) and content:+ # Tool output as a STRING under any harness label: OpenAI/Kimi+ # ``role:tool``, legacy ``role:function``, or a text-harness+ # ``role:user`` read output. Key off the OUTCOME, not the role —+ # ``role:user`` is gated to genuine reads so prose never folds.+ role = msg.get("role")+ if role in ("tool", "function") or (+ role == "user" and _is_user_read_observation(i)+ ):+ protected = frozen or ("cache_control" in msg)+ locs.append((i, None, None))+ dblocks.append(DedupBlock(text=content, turn=i, protected=protected))if len(dblocks) < 2:return messages@@ -4265,7 +4309,7 @@ class ContentRouter(Transform):new_messages = list(messages)touched: dict[int, dict[str, Any]] = {}- for (mi, blk_idx), od, nd in zip(locs, dblocks, deduped):+ for (mi, blk_idx, sub_idx), od, nd in zip(locs, dblocks, deduped):if od.protected or nd.text == od.text:continueif mi not in touched:@@ -4280,8 +4324,19 @@ class ContentRouter(Transform):m = touched[mi]if blk_idx is None:m["content"] = nd.text- else:+ elif sub_idx is None:m["content"][blk_idx]["content"] = nd.text+ else:+ # List-content tool_result: deep-copy the block + its sub-list+ # before mutating so the input message is never touched.+ blk = dict(m["content"][blk_idx])+ sub_list = [+ dict(s) if isinstance(s, dict) else s for s in blk.get("content", [])+ ]+ sub_list[sub_idx] = dict(sub_list[sub_idx])+ sub_list[sub_idx]["text"] = nd.text+ blk["content"] = sub_list+ m["content"][blk_idx] = blkif route_counts is not None:route_counts["cross_turn_dedup"] = (diff --git a/headroom/transforms/cross_turn_dedup.py b/headroom/transforms/cross_turn_dedup.pyindex a8f46b301..92731725c 100644--- a/headroom/transforms/cross_turn_dedup.py+++ b/headroom/transforms/cross_turn_dedup.py@@ -28,6 +28,7 @@ Pure stdlib, deterministic, never raises (returns input unchanged on any error).from __future__ import annotations+import refrom dataclasses import dataclass__all__ = ["DedupBlock", "dedup_blocks", "is_prefix_monotonic"]@@ -36,12 +37,40 @@ __all__ = ["DedupBlock", "dedup_blocks", "is_prefix_monotonic"]# pointer. Small dups are left alone (fragmenting context is not worth it) —# and a larger floor keeps the pointer comfortably shorter than the span it# replaces, so a fold is always a net byte win.-DEFAULT_MIN_LINES = 7-DEFAULT_MIN_CHARS = 120+DEFAULT_MIN_LINES = 3+DEFAULT_MIN_CHARS = 40# Cap anchor candidates examined per line so a hot line (e.g. `` return``)# can't blow up matching. Deterministic: candidates are kept in first-seen order.MAX_ANCHOR_CANDIDATES = 16+# An UNPADDED leading line-number prefix: ``123:...`` or ``123<TAB>...`` at the+# very start of the line (grep -n / sed -n / ripgrep --heading data rows). We do+# NOT match a padded/right-aligned prefix (`` 123<TAB>`` from ``cat -n``): the+# line must start with the digit, so ``number + delta`` re-numbers byte-exactly+# without touching alignment padding, keeping renumbered folds strictly lossless.+_LINENO_RE = re.compile(r"^(\d+)(:|\t)(.*)$", re.DOTALL)+++def _num_and_key(line: str) -> tuple[int | None, str, str]:+ """Split a leading unpadded line-number.++ Returns ``(number, match_key, content)``:+ * ``number`` — the leading line number (``int``) or ``None`` if absent.+ * ``match_key`` — the line with the leading number stripped (separator+ kept), so two occurrences of the same content at DIFFERENT line numbers+ share a key (e.g. ``92:foo`` and ``94:foo`` -> ``:foo``). Non-numbered+ lines key on themselves, so exact matching is unchanged for them.+ * ``content`` — the text after the separator, for the triviality test.++ This is what makes cross-turn dedup survive an edit that renumbers a file:+ a re-read whose line numbers all shifted by a constant still folds, and the+ shift is carried in the pointer so the original bytes stay recoverable.+ """+ m = _LINENO_RE.match(line)+ if m is None:+ return None, line, line+ return int(m.group(1)), m.group(2) + m.group(3), m.group(3)+@dataclassclass DedupBlock:@@ -79,20 +108,29 @@ def _is_trivial(line: str) -> bool:}-def _pointer(span: list[str], ref_turn: int, ref_line: int) -> str:+def _pointer(span: list[str], ref_turn: int, delta: int = 0) -> str:"""A one-line, obviously-a-reference marker naming the in-context original.Includes a first-line anchor so the model can locate the block it alreadysaw. Marker-free of any ``hash=`` retrieval token: recovery is in-context- (the original is physically present earlier in the same request)."""- anchor = next((ln.strip() for ln in span if ln.strip()), "")- if len(anchor) > 80:- anchor = anchor[:77] + "..."- end_line = ref_line + len(span) - 1- return (- f"[headroom: {len(span)} lines identical to output shown earlier "- f"(turn {ref_turn}, lines {ref_line}-{end_line}) — starts: {anchor!r}]"- )+ (the original is physically present earlier in the same request).++ ``delta`` is the uniform line-number offset of THIS span relative to the+ referenced original (0 for a byte-identical fold). When non-zero the span is+ the same content re-read after an edit shifted its line numbers; the offset+ is stated so the original the pointer names + ``delta`` reconstructs this+ span's exact numbered bytes (recovery stays in-window)."""+ # Compact form (~35c vs the old ~100c): the ~100c pointer made sub-7-line+ # folds net-negative, so the abundant short (2-4 line) re-read repeats were+ # left uncompressed. Trimming the pointer + MIN_LINES=3 lets those pay off+ # (~4.3% -> ~6% lossless on Opus). Still no hash= token (in-context recovery)+ # and keeps a short first-line anchor so the model can locate the original.+ anchor = next((_num_and_key(ln)[2].strip() for ln in span if ln.strip()), "")+ if len(anchor) > 20:+ anchor = anchor[:17] + "..."+ if delta:+ return f"[↑{len(span)}L same as msg {ref_turn} {delta:+d}L: {anchor!r}]"+ return f"[↑{len(span)}L same as msg {ref_turn}: {anchor!r}]"def _index_lines(@@ -104,11 +142,15 @@ def _index_lines(Keeps first-seen order and caps the candidate list per line. Only VERBATIM(surviving) lines should be passed here — never the lines of a span that was- replaced by a pointer."""+ replaced by a pointer. Indexed by the line-number-stripped ``match_key`` so a+ later renumbered re-read of the same content finds this anchor."""for li, ln in enumerate(lines):- if ln is None or _is_trivial(ln):+ if ln is None:+ continue+ _, key, content = _num_and_key(ln)+ if _is_trivial(content):continue- bucket = anchor_index.setdefault(ln, [])+ bucket = anchor_index.setdefault(key, [])if len(bucket) < MAX_ANCHOR_CANDIDATES:bucket.append((block_pos, li))@@ -118,34 +160,57 @@ def _longest_match(start: int,anchor_index: dict[str, list[tuple[int, int]]],corpus: list[list[str | None]],-) -> tuple[int, int, int] | None:- """Longest contiguous run in ``cur`` starting at ``start`` that appears- verbatim inside a single earlier block. Returns (length, block_pos,- ref_line_idx) or None. ``corpus[block_pos]`` holds that block's VERBATIM- lines (``None`` where a span was already folded, which breaks contiguity)."""- anchor = cur[start]- candidates = anchor_index.get(anchor)+) -> tuple[int, int, int, int] | None:+ """Longest contiguous run in ``cur`` starting at ``start`` whose content+ appears in a single earlier block — allowing a UNIFORM line-number shift.++ Returns ``(length, block_pos, ref_line_idx, delta)`` or ``None``. ``delta``+ is the constant line-number offset of the run vs the referenced original+ (0 for a byte-identical fold). ``corpus[block_pos]`` holds that block's+ VERBATIM lines (``None`` where a span was already folded, which breaks+ contiguity).++ A run extends while each pair shares a ``match_key`` (content modulo the+ leading line number) AND every numbered pair has the SAME numeric offset —+ so an edit that shifts a file's numbers by a constant still folds, but a+ non-uniform change (edit *inside* the span) ends the run at the divergence.+ Non-numbered lines must match exactly (they carry no offset)."""+ _, anchor_key, _ = _num_and_key(cur[start])+ candidates = anchor_index.get(anchor_key)if not candidates:return Nonebest_len = 0best_bp = best_li = -1+ best_delta = 0for bp, li in candidates:block_lines = corpus[bp]k = 0- while (- start + k < len(cur)- and li + k < len(block_lines)- and block_lines[li + k] is not None- and cur[start + k] == block_lines[li + k]- ):+ delta: int | None = None+ while start + k < len(cur) and li + k < len(block_lines):+ ca = cur[start + k]+ cb = block_lines[li + k]+ if cb is None: # end of a folded block in the corpus — run ends here+ break+ na, ka, _ = _num_and_key(ca)+ nb, kb, _ = _num_and_key(cb)+ if ka != kb:+ break # different content -> run ends+ if na is not None and nb is not None:+ d = na - nb+ if delta is None:+ delta = d+ elif delta != d:+ break # non-uniform shift (edit inside span) -> run ends+ elif ca != cb:+ break # non-numbered line must match exactlyk += 1# Deterministic tie-break: longer wins; on ties keep the earliest# (smallest block_pos, then line) already held in best_*.if k > best_len:- best_len, best_bp, best_li = k, bp, li+ best_len, best_bp, best_li, best_delta = k, bp, li, (delta or 0)if best_len == 0:return None- return best_len, best_bp, best_li+ return best_len, best_bp, best_li, best_deltadef dedup_blocks(@@ -186,7 +251,7 @@ def dedup_blocks(span_text = "\n".join(span)if len(span_text) >= min_chars:ref_turn = blocks[m[1]].turn- ptr = _pointer(span, ref_turn, m[2])+ ptr = _pointer(span, ref_turn, m[3])out.append(ptr)# Folded span is NOT verbatim in this block's output:# mark None so it can't seed a later contiguous match,diff --git a/tests/test_cross_turn_dedup.py b/tests/test_cross_turn_dedup.pyindex d4c35b7a4..4949a5633 100644--- a/tests/test_cross_turn_dedup.py+++ b/tests/test_cross_turn_dedup.py@@ -4,11 +4,15 @@ import refrom headroom.transforms.cross_turn_dedup import (DedupBlock,+ _num_and_key,dedup_blocks,is_prefix_monotonic,)-_PTR_RE = re.compile(r"identical to output shown earlier \(turn (\d+), lines (\d+)-(\d+)\)")+# Compact fold pointer: ``[↑<N>L same as msg <ref>[ <±delta>L]: '<anchor>']`` —+# span length + referenced msg + optional line-number offset + a truncated+# first-line anchor (no explicit line range; recovery locates the span by anchor).+_FOLD_RE = re.compile(r"\[↑(\d+)L same as msg (\d+)(?: ([+-]\d+)L)?: '([^']*)'\]")def _blk(text, turn, protected=False):@@ -24,20 +28,33 @@ def _code(prefix, n):def _reconstruct(orig_blocks, out_blocks):- """Replace each pointer with the referenced turn's original lines and assert- it reproduces the original block — proves references are faithful & in-context."""+ """Replace each fold pointer with the referenced msg's original lines and assert+ it reproduces the original block — proves references are faithful & in-context.++ The compact pointer names the ref msg + span length + a first-line anchor (not+ an explicit line range), so recovery locates the span by its anchor in the+ referenced message and takes ``<N>`` lines. (All test spans are unnumbered, so+ delta is always absent; a non-zero delta would renumber on the way out.)"""by_turn = {b.turn: b.text.split("\n") for b in orig_blocks}++ def _content(line):+ return _num_and_key(line)[2].strip()+for orig, out in zip(orig_blocks, out_blocks):if orig.protected:assert out.text == orig.textcontinuerebuilt = []for line in out.text.split("\n"):- m = _PTR_RE.search(line)- if m and line.startswith("[headroom:"):- t, a, b = int(m.group(1)), int(m.group(2)), int(m.group(3))- assert t < orig.turn, "reference must point to an EARLIER turn"- rebuilt.extend(by_turn[t][a : b + 1])+ m = _FOLD_RE.search(line)+ if m and line.lstrip().startswith("[↑"):+ assert m.group(3) is None, "unexpected delta for an unnumbered span"+ n, ref, anchor = int(m.group(1)), int(m.group(2)), m.group(4)+ assert ref < orig.turn, "reference must point to an EARLIER msg"+ core = anchor[:-3] if anchor.endswith("...") else anchor+ ref_lines = by_turn[ref]+ idx = next(i for i, rl in enumerate(ref_lines) if _content(rl).startswith(core))+ rebuilt.extend(ref_lines[idx : idx + n])else:rebuilt.append(line)assert "\n".join(rebuilt) == orig.text, f"turn {orig.turn} not faithfully reconstructable"@@ -48,7 +65,7 @@ def test_verbatim_reread_is_folded_keep_earliest():blocks = [_blk(f"cat merge.py\n{span}\ntail", 1), _blk(f"sed run\n{span}\nmore", 5)]out, stats = dedup_blocks(blocks)assert out[0].text == blocks[0].text # earliest untouched- assert "[headroom:" in out[1].text # later occurrence folded+ assert "[↑" in out[1].text # later occurrence foldedassert stats["spans_folded"] == 1_reconstruct(blocks, out)@@ -65,7 +82,7 @@ def test_cache_safety_prefix_monotonic():def test_below_min_lines_not_folded():- span = _code("", 3) # below min_lines (7)+ span = _code("", 2) # below min_lines (3)blocks = [_blk(span, 1), _blk(span, 2)]out, stats = dedup_blocks(blocks)assert stats["spans_folded"] == 0@@ -95,7 +112,7 @@ def test_protected_block_not_rewritten_but_is_reference_target():]out, stats = dedup_blocks(blocks)assert out[0].text == blocks[0].text- assert "[headroom:" in out[1].text+ assert "[↑" in out[1].text_reconstruct(blocks, out)@@ -159,8 +176,8 @@ def test_apply_dedups_reread_and_keeps_prefix_stable():# Dedup fired on the later re-read (turn t2), earliest (t1) untouched.later = out2[-1]["content"][0]["content"]earlier = out2[2]["content"][0]["content"]- assert "[headroom:" in later- assert "[headroom:" not in earlier and span in earlier+ assert "[↑" in later+ assert "[↑" not in earlier and span in earlier# CACHE-SAFETY at the router level: appending turn t2 did NOT change any# earlier message's emitted bytes → the prompt-cache prefix is stable.@@ -189,7 +206,7 @@ def test_apply_no_dedup_when_flag_off():r = ContentRouter(ContentRouterConfig(lossless=True, enable_cross_turn_dedup=False))out = r.apply(copy.deepcopy(msgs), _mk_tok()).messagesjoined = "".join(b["content"] for m in out for b in m["content"] if isinstance(b, dict))- assert "[headroom:" not in joined+ assert "[↑" not in joineddef test_apply_dedup_runs_in_ccr_mode_too():@@ -214,4 +231,77 @@ def test_apply_dedup_runs_in_ccr_mode_too():for b in m["content"]if isinstance(b, dict))- assert "[headroom:" in joined # dedup fired despite lossless=False+ assert "[↑" in joined # dedup fired despite lossless=False+++# --------------------------------------------------------------------------+# No dangling reference: dedup folds only against content present in the array+# it processes (it runs last, over the final sent messages). If compaction+# already removed the original, there is nothing earlier to reference → verbatim.+# --------------------------------------------------------------------------+def test_no_fold_when_original_absent_fallback():+ span = _code("", 8)+ # Only the LATER read survives; its original was compacted out of the array.+ blocks = [_blk("unrelated log\n" + _code("z", 8), 1), _blk(f"sed\n{span}\nmore", 5)]+ out, stats = dedup_blocks(blocks)+ assert stats["spans_folded"] == 0+ assert out[1].text == blocks[1].text and "[↑" not in out[1].text+++# --------------------------------------------------------------------------+# Shape-agnostic CODE-READ coverage: a file read gets deduped wherever it lands+# — role:tool, role:function, or a text-harness role:user string — keyed off the+# read OUTCOME, never ordinary user prose.+# --------------------------------------------------------------------------+def _dedup_only(messages):+ """Run ONLY the cross-turn dedup pass (no per-block compression) on a raw+ message array, isolating extraction + fold across message shapes."""+ from headroom.transforms.content_router import ContentRouter, ContentRouterConfig++ r = ContentRouter(ContentRouterConfig(lossless=True, enable_cross_turn_dedup=True))+ return r._cross_turn_dedup_messages(messages, 0, [], None)+++def _readspan():+ return "\n".join(f" result_{i} = compute_overdraft(business_id={i})" for i in range(10))+++def test_dedup_folds_user_string_read_observation():+ # Text-harness shape: the read output arrives as a role:user STRING after an+ # assistant fenced `cat`. The later identical read folds; earliest stays intact.+ span = _readspan()+ asst = {"role": "assistant", "content": "```bash\ncat report.py\n```"}+ msgs = [+ asst,+ {"role": "user", "content": span}, # read #1 (reference target)+ asst,+ {"role": "user", "content": span}, # read #2 (duplicate) -> folds+ ]+ out = _dedup_only(msgs)+ assert "[↑" in out[3]["content"]+ assert out[1]["content"] == span+++def test_dedup_does_not_fold_plain_user_prose():+ # A duplicated ORDINARY user message (no preceding read command) must stay+ # verbatim — user intent is never folded.+ prose = "\n".join(f"please also make sure case {i} is handled carefully" for i in range(10))+ msgs = [+ {"role": "user", "content": prose},+ {"role": "assistant", "content": "understood"},+ {"role": "user", "content": prose}, # duplicate prose -> must NOT fold+ ]+ out = _dedup_only(msgs)+ assert "[↑" not in out[2]["content"] and out[2]["content"] == prose+++def test_dedup_folds_role_function_output():+ # Legacy OpenAI role:function tool output — same operation, different label.+ span = _readspan()+ msgs = [+ {"role": "function", "name": "read_file", "content": span},+ {"role": "assistant", "content": "let me re-check"},+ {"role": "function", "name": "read_file", "content": span}, # dup -> folds+ ]+ out = _dedup_only(msgs)+ assert "[↑" in out[2]["content"]diff --git a/tests/test_responses_cross_turn_dedup.py b/tests/test_responses_cross_turn_dedup.pynew file mode 100644index 000000000..276531f61--- /dev/null+++ b/tests/test_responses_cross_turn_dedup.py@@ -0,0 +1,114 @@+"""Cross-turn dedup on the OpenAI Responses path (Codex ``function_call_output``).++Fixtures mirror a REAL Codex run captured through the headroom proxy: a file read+returns as ``{"type":"function_call_output","call_id":...,"output":"Chunk ID: …\\n+Wall time: …\\nProcess exited with code 0\\nOriginal token count: …\\nOutput:\\n+<FILE BODY>\\n"}``. The ``Chunk ID`` / ``Wall time`` header varies per call, so a+whole-block match never fires — longest-span matching must fold the identical+body and leave the varying header verbatim.+"""++from __future__ import annotations++from headroom.proxy.handlers.openai import (+ _RESPONSES_OUTPUT_ITEM_TYPES,+ _dedup_responses_output_items,+)++BODY = (+ "def paginate_orders(items, page, page_size):\n"+ ' """Return one page of orders."""\n'+ " start = page * page_size\n"+ " end = start + page_size + 1 # off-by-one: should be start + page_size\n"+ " return items[start:end]\n"+ "\n\n"+ 'SERVICE_TAG = "svc-03e8-tag"\n'+ "\n\n"+ "def compute_overdraft(business_id, amount):\n"+ " fee = amount * 0.05\n"+ ' return {"business_id": business_id, "fee": fee, "tag": SERVICE_TAG}\n'+)+++def _wrap(chunk_id: str, wall: str) -> str:+ # Codex's exec_command wrapper — the header lines vary call to call.+ return (+ f"Chunk ID: {chunk_id}\n"+ f"Wall time: {wall} seconds\n"+ "Process exited with code 0\n"+ "Original token count: 97\n"+ "Output:\n"+ )+++def _read_output(call_id: str, chunk_id: str, wall: str) -> dict:+ return {+ "type": "function_call_output",+ "call_id": call_id,+ "output": _wrap(chunk_id, wall) + BODY,+ }+++def _read_call(call_id: str) -> dict:+ return {+ "type": "function_call",+ "name": "exec_command",+ "arguments": '{"cmd":"cat buggy.py","workdir":"/tmp"}',+ "call_id": call_id,+ }+++def test_repeated_codex_read_folds_body_keeps_varying_header():+ items = [+ {"role": "user", "content": "find the bug"},+ _read_call("c1"),+ _read_output("c1", "492f0f", "0.0000"), # read #1 (reference)+ {+ "type": "message",+ "role": "assistant",+ "content": [{"type": "output_text", "text": "re-reading"}],+ },+ _read_call("c2"),+ _read_output("c2", "a1b2c3", "0.0100"), # read #2 (duplicate) -> body folds+ ]+ folded, saved = _dedup_responses_output_items(+ items, _RESPONSES_OUTPUT_ITEM_TYPES, count_tokens=len+ )++ assert folded == 1+ assert saved > 0+ # earliest read: byte-identical (reference target, sits in the cached prefix)+ assert items[2]["output"] == _wrap("492f0f", "0.0000") + BODY+ # later read: identical body folded to a pointer; the per-call header stays verbatim+ later = items[5]["output"]+ assert "[↑" in later+ assert later.startswith("Chunk ID: a1b2c3\nWall time: 0.0100")+ assert "def paginate_orders" not in later # body folded away+ # lossless: the folded body is still fully present earlier in the request+ assert "def paginate_orders" in items[2]["output"]+++def test_single_read_does_not_fold():+ items = [_read_call("c1"), _read_output("c1", "492f0f", "0.0000")]+ folded, saved = _dedup_responses_output_items(+ items, _RESPONSES_OUTPUT_ITEM_TYPES, count_tokens=len+ )+ assert folded == 0 and saved == 0+ assert items[1]["output"] == _wrap("492f0f", "0.0000") + BODY+++def test_non_output_items_untouched():+ # A duplicated MESSAGE (not a tool output) must never fold — only output+ # items are eligible.+ msg = {"role": "user", "content": BODY}+ items = [dict(msg), {"type": "message", "role": "assistant", "content": "ok"}, dict(msg)]+ folded, _ = _dedup_responses_output_items(items, _RESPONSES_OUTPUT_ITEM_TYPES)+ assert folded == 0+ assert items[2]["content"] == BODY+++def test_never_raises_on_malformed():+ # Defensive: junk items must not blow up the request path.+ items = [{"type": "function_call_output"}, {"type": "function_call_output", "output": None}, 42]+ folded, saved = _dedup_responses_output_items(items, _RESPONSES_OUTPUT_ITEM_TYPES) # type: ignore[arg-type]+ assert (folded, saved) == (0, 0)