Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): repair tool-search history in place so signed thinking blocks keep their coordinates (#3457)
Eyal Mizrachi68359c056cc9
diff --git a/headroom/proxy/handlers/_debug_dump.py b/headroom/proxy/handlers/_debug_dump.pyindex 6fbc46fa3..b04459fb3 100644--- a/headroom/proxy/handlers/_debug_dump.py+++ b/headroom/proxy/handlers/_debug_dump.py@@ -8,8 +8,15 @@ OpenAI handlers so the gating stays consistent across providers.from __future__ import annotations+import json+import loggingimport os+from datetime import datetime+from pathlib import Pathfrom typing import Any+from urllib.parse import urlsplit, urlunsplit++logger = logging.getLogger(__name__)def _debug_dump_mode(config: Any) -> str:@@ -45,3 +52,90 @@ def _redact_debug_value(value: Any, _max_len: int = 80) -> Any:if isinstance(value, list):return [_redact_debug_value(v, _max_len) for v in value]return value+++def _redact_url(url: str) -> str:+ """Drop the query string and fragment from ``url``.++ Some upstreams carry credentials in the query (Gemini's ``?key=``), and the+ query is never needed to debug a rejected request, so it is elided in every+ mode — ``full`` opts in to prompt content, not to credentials.+ """+ try:+ parts = urlsplit(url)+ except ValueError:+ return "<unparseable url>"+ if not parts.query and not parts.fragment:+ return url+ redacted = urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))+ return f"{redacted}?<redacted>" if parts.query else redacted+++def _decode_sent_body(body: Any) -> Any:+ """Decode the exact bytes sent upstream back to JSON for the dump."""+ if not isinstance(body, (bytes, bytearray)):+ return body+ try:+ return json.loads(body)+ except ValueError:+ return f"<{len(body)} bytes, not JSON>"+++def write_upstream_error_dump(+ config: Any,+ *,+ request_id: str,+ url: str,+ status: int,+ provider: str | None = None,+ model: str | None = None,+ body: Any = None,+ body_source: str | None = None,+ transforms: Any = None,+ stream: bool = False,+) -> Path | None:+ """Write a diagnostic dump of an erroring (>=400) upstream request.++ ``body`` may be the raw bytes that went on the wire; they are decoded here,+ after the off-by-default gate, so a disabled dump costs nothing.+ ``body_source`` records which body was forwarded (for example+ ``"passthrough"`` when the proxy's mutations were dropped), since that is+ often the whole explanation for a 400.++ Returns the path written, or ``None`` when the dump is disabled (the+ default, and always in stateless mode) or could not be written. Never+ raises: a diagnostic must not turn an upstream error into a proxy error.+ """+ mode = _debug_dump_mode(config)+ if mode == "off":+ return None+ try:+ from headroom import paths as _hr_paths++ dump_dir = _hr_paths.debug_400_dir()+ dump_dir.mkdir(mode=0o700, parents=True, exist_ok=True)+ dump_path = dump_dir / f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_{request_id}.json"+ sent_body = _decode_sent_body(body)+ payload = json.dumps(+ {+ "request_id": request_id,+ "url": _redact_url(url),+ "status": status,+ "provider": provider,+ "model": model,+ "stream": stream,+ "transforms": transforms,+ "body_source": body_source,+ "body": sent_body if mode == "full" else _redact_debug_value(sent_body),+ },+ indent=2,+ default=str,+ )+ # The dump can hold prompt content: keep it readable by the owner only.+ fd = os.open(dump_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)+ with os.fdopen(fd, "w", encoding="utf-8") as fh:+ fh.write(payload)+ return dump_path+ except Exception:+ logger.debug("upstream error debug dump skipped", exc_info=True)+ return Nonediff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex dc88815ff..b648ba0a1 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -3140,8 +3140,9 @@ class AnthropicHandlerMixin:body_mutation_tracker.mark_mutated("tool_search_history_repair")transforms_applied.append(f"router:tool_search_repair:{_ts_stripped}blocks")logger.info(- "[%s] Tool search: dropped %d unsupportable history block(s) "- "(tools array cannot resolve their tool_reference entries)",+ "[%s] Tool search: repaired %d unsupportable history block(s) "+ "(replaced with text in place; tools array cannot resolve their "+ "tool_reference entries)",request_id,_ts_stripped,)diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.pyindex 66e89fae3..b364c3c7a 100644--- a/headroom/proxy/handlers/streaming.py+++ b/headroom/proxy/handlers/streaming.py@@ -13,6 +13,7 @@ import timefrom typing import TYPE_CHECKING, Anyfrom headroom.proxy.auth_mode import classify_client, supports_mid_turn_coalescing+from headroom.proxy.handlers._debug_dump import write_upstream_error_dumpfrom headroom.proxy.helpers import (RETRYABLE_OVERLOAD_STATUSES,jitter_delay_ms,@@ -1463,6 +1464,27 @@ class StreamingMixin:upstream_response.status_code,url,)+ # Diagnostic dump of the erroring request — parity with the+ # non-streaming handlers, which dump on >=400 but never fire for a+ # streaming turn (Claude Code streams every request, so the most+ # common 400s were invisible). Same gating: OFF by default, never+ # in stateless mode, content redacted unless HEADROOM_DEBUG_DUMP=full.+ # Dump the bytes that went on the wire, not ``body``: when the edits+ # are dropped (source="passthrough") ``body`` shows a request that+ # never left the proxy.+ write_upstream_error_dump(+ getattr(self, "config", None),+ request_id=request_id,+ url=url,+ status=upstream_response.status_code,+ provider=provider,+ model=model,+ body=outbound_bytes,+ body_source=outbound_source,+ transforms=transforms_applied,+ stream=True,+ )+response_headers = dict(upstream_response.headers)response_headers.pop("content-length", None)response_headers.pop("transfer-encoding", None)diff --git a/headroom/proxy/helpers.py b/headroom/proxy/helpers.pyindex 8e8766bda..2231f9ae0 100644--- a/headroom/proxy/helpers.py+++ b/headroom/proxy/helpers.py@@ -3285,17 +3285,38 @@ def _tool_search_reference_names(content: Any) -> list[str]:return names+# Stand-in for a tool-search block the outbound tools array cannot support. Text+# so it is inert to every validator, short so it costs ~10 tokens, and constant so+# the repaired prefix stays byte-stable across turns (the provider cache needs the+# same bytes every time).+_TOOL_SEARCH_PLACEHOLDER_BLOCK: dict[str, Any] = {+ "type": "text",+ "text": "[tool search omitted: unavailable in this request]",+}++def strip_unsupported_tool_search_blocks(messages: Any, tools: Any) -> tuple[Any, int]:- """Drop tool-search blocks this request's ``tools`` array cannot support.+ """Neutralize tool-search blocks this request's ``tools`` array cannot support.A block pair is unsupportable when the request carries no ``tool_search_tool_*``tool, or when a ``tool_reference`` names a tool absent from ``tools`` — the twoshapes Anthropic rejects. Both the ``tool_search_tool_result`` and its paired- ``server_tool_use`` are removed (an orphan of either 400s on its own), and a- message left with no content blocks is dropped rather than sent empty.-- Returns ``(messages, blocks_removed)``, and the ORIGINAL ``messages`` object- when nothing was removed — callers rely on identity to skip the write-back.+ ``server_tool_use`` are handled (an orphan of either 400s on its own).++ Replace in place rather than remove (#3456). The block indexes of a message+ are load-bearing: ``thinking_block_fingerprint`` keys a signed thinking block+ by ``(message_index, block_index)``, so deleting a block that sits BEFORE a+ thinking block in the same message — or deleting a whole message ahead of one —+ moves that block, ``thinking_blocks_survived_mutation`` reports False, and+ ``select_outbound_body`` then forwards the client's ORIGINAL bytes and discards+ every mutation, this repair included. The request that needed repairing is+ exactly the one that loses it, and upstream 400s on the reference we had+ already found. Swapping each block for a short text block keeps every thinking+ block at its original coordinates, so the repair survives to the wire. Same+ reasoning as the CCR sibling below.++ Returns ``(messages, blocks_repaired)``, and the ORIGINAL ``messages`` object+ when nothing changed — callers rely on identity to skip the write-back."""if not isinstance(messages, list):return messages, 0@@ -3328,7 +3349,7 @@ def strip_unsupported_tool_search_blocks(messages: Any, tools: Any) -> tuple[Anyout.append(message)continue- drop_indexes: set[int] = set()+ neutralize_indexes: set[int] = set()orphaned_ids: set[str] = set()for index, block in enumerate(content):if not isinstance(block, dict) or block.get("type") != _TOOL_SEARCH_RESULT_TYPE:@@ -3336,7 +3357,7 @@ def strip_unsupported_tool_search_blocks(messages: Any, tools: Any) -> tuple[Anynames = _tool_search_reference_names(block.get("content"))if has_search_tool and all(name in available for name in names):continue- drop_indexes.add(index)+ neutralize_indexes.add(index)use_id = block.get("tool_use_id")if use_id:orphaned_ids.add(str(use_id))@@ -3348,19 +3369,19 @@ def strip_unsupported_tool_search_blocks(messages: Any, tools: Any) -> tuple[Anycontinueis_search_call = str(block.get("name", "")).startswith(_TOOL_SEARCH_TOOL_TYPE_PREFIX)if str(block.get("id", "")) in orphaned_ids or (is_search_call and not has_search_tool):- drop_indexes.add(index)+ neutralize_indexes.add(index)- if not drop_indexes:+ if not neutralize_indexes:out.append(message)continuechanged = True- removed += len(drop_indexes)- kept = [block for index, block in enumerate(content) if index not in drop_indexes]- if not kept:- continue # the whole turn was tool-search bookkeeping+ removed += len(neutralize_indexes)repaired = dict(message)- repaired["content"] = kept+ repaired["content"] = [+ dict(_TOOL_SEARCH_PLACEHOLDER_BLOCK) if index in neutralize_indexes else block+ for index, block in enumerate(content)+ ]out.append(repaired)return (out, removed) if changed else (messages, 0)diff --git a/tests/test_debug_dump_gating.py b/tests/test_debug_dump_gating.pyindex 3bc85bbbe..924c9d3fa 100644--- a/tests/test_debug_dump_gating.py+++ b/tests/test_debug_dump_gating.py@@ -8,6 +8,8 @@ operator explicitly opts in to full content.from __future__ import annotationsimport inspect+import json+import osfrom types import SimpleNamespaceimport pytest@@ -94,3 +96,143 @@ def test_both_handlers_gate_the_dump(module_name):assert 'if dump_mode != "off":' in src, (f"{module_name} debug dump is not guarded by an off-by-default check")+++# ---------------------------------------------------------------------------+# write_upstream_error_dump: the shared writer used by the streaming handler+# ---------------------------------------------------------------------------+++def dump_dir(tmp_path, monkeypatch):+ """Point ``paths.debug_400_dir()`` at a temp dir for the writer tests."""+ from headroom import paths++ target = tmp_path / "debug_400"+ monkeypatch.setattr(paths, "debug_400_dir", lambda: target)+ return target+++def _write(**kwargs):+ from headroom.proxy.handlers._debug_dump import write_upstream_error_dump++ params = {+ "request_id": "req_1",+ "url": "https://api.anthropic.com/v1/messages",+ "status": 400,+ "provider": "anthropic",+ "model": "claude-opus-5",+ "body": {"messages": [{"role": "user", "content": "secret prompt content " * 20}]},+ "transforms": ["tool_search_history_repair"],+ "stream": True,+ }+ params.update(kwargs)+ config = params.pop("config", _config())+ return write_upstream_error_dump(config, **params)+++def test_upstream_dump_off_by_default_writes_nothing(dump_dir, monkeypatch):+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)+ assert _write() is None+ assert not dump_dir.exists()+++def test_upstream_dump_stateless_writes_nothing(dump_dir, monkeypatch):+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")+ assert _write(config=_config(stateless=True)) is None+ assert not dump_dir.exists()+++def test_upstream_dump_full_records_request(dump_dir, monkeypatch):+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")+ path = _write()+ assert path is not None and path.parent == dump_dir+ payload = json.loads(path.read_text())+ assert payload["request_id"] == "req_1"+ assert payload["status"] == 400+ assert payload["provider"] == "anthropic"+ assert payload["model"] == "claude-opus-5"+ assert payload["stream"] is True+ assert payload["transforms"] == ["tool_search_history_repair"]+ assert "secret prompt content" in payload["body"]["messages"][0]["content"]+++def test_upstream_dump_redacted_elides_content(dump_dir, monkeypatch):+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "1")+ path = _write()+ assert path is not None+ payload = json.loads(path.read_text())+ content = payload["body"]["messages"][0]["content"]+ assert content.startswith("<redacted:") and "secret prompt" not in content+ # Structure is still there to debug with:+ assert payload["body"]["messages"][0]["role"] == "user"+++def test_upstream_dump_serializes_non_json_values(dump_dir, monkeypatch):+ # ``default=str`` must keep an exotic body from raising mid-dump.+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")+ path = _write(body={"when": object()})+ assert path is not None+ assert "object object" in json.loads(path.read_text())["body"]["when"]++[email protected]("mode", ["1", "full"])+def test_upstream_dump_never_writes_url_credentials(dump_dir, monkeypatch, mode):+ # Gemini streaming URLs carry the API key as ``?key=``; ``full`` opts in to+ # prompt content, not to credentials, so the query is dropped in every mode.+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", mode)+ url = (+ "https://generativelanguage.googleapis.com/v1beta/models/gemini:streamGenerateContent"+ "?alt=sse&key=AIzaSECRET"+ )+ path = _write(url=url, provider="gemini")+ assert path is not None+ text = path.read_text()+ assert "AIzaSECRET" not in text+ assert json.loads(text)["url"] == (+ "https://generativelanguage.googleapis.com/v1beta/models/gemini:streamGenerateContent"+ "?<redacted>"+ )+++def test_upstream_dump_keeps_query_free_url(dump_dir, monkeypatch):+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")+ path = _write()+ assert json.loads(path.read_text())["url"] == "https://api.anthropic.com/v1/messages"+++def test_upstream_dump_records_the_bytes_actually_sent(dump_dir, monkeypatch):+ # When the proxy's edits are dropped (passthrough), the dump must show the+ # body that went on the wire, not the edited one that never left.+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")+ sent = json.dumps({"messages": [{"role": "user", "content": "as sent"}]}).encode()+ path = _write(body=sent, body_source="passthrough")+ payload = json.loads(path.read_text())+ assert payload["body"]["messages"][0]["content"] == "as sent"+ assert payload["body_source"] == "passthrough"+++def test_upstream_dump_tolerates_non_json_bytes(dump_dir, monkeypatch):+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")+ path = _write(body=b"\x00\x01not json")+ assert json.loads(path.read_text())["body"] == "<10 bytes, not JSON>"++[email protected](os.name == "nt", reason="POSIX permission bits")+def test_upstream_dump_is_owner_only(dump_dir, monkeypatch):+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")+ path = _write()+ assert path.stat().st_mode & 0o777 == 0o600+++def test_upstream_dump_never_raises_when_write_fails(monkeypatch):+ # A diagnostic must not turn an upstream error into a proxy error.+ from headroom import paths++ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")++ def _boom():+ raise OSError("read-only filesystem")++ monkeypatch.setattr(paths, "debug_400_dir", _boom)+ assert _write() is Nonediff --git a/tests/test_issue_746_tool_search.py b/tests/test_issue_746_tool_search.pyindex b3321a010..6ee4a70ae 100644--- a/tests/test_issue_746_tool_search.py+++ b/tests/test_issue_746_tool_search.py@@ -433,7 +433,7 @@ def _poisoned_transcript() -> list[dict]:]-def test_repair_drops_blocks_the_hook_evaluator_cannot_resolve() -> None:+def test_repair_neutralizes_blocks_the_hook_evaluator_cannot_resolve() -> None:# The Stop hook evaluator replays the transcript with a small tools array# that has neither the search tool nor AskUserQuestion -> upstream 400.messages, removed = strip_unsupported_tool_search_blocks(@@ -441,7 +441,11 @@ def test_repair_drops_blocks_the_hook_evaluator_cannot_resolve() -> None:)assert removed == 2 # server_tool_use + tool_search_tool_resultkinds = [b["type"] for b in messages[1]["content"]]- assert kinds == ["text", "text"] # surrounding assistant text survives+ assert kinds == ["text", "text", "text", "text"] # both swapped for text in place+ assert messages[1]["content"][0]["text"] == "Searching for a tool."+ assert messages[1]["content"][3]["text"] == "Found it." # index preserved+ assert "tool search omitted" in messages[1]["content"][1]["text"]+ assert "tool search omitted" in messages[1]["content"][2]["text"]assert messages[0]["content"][0]["text"] == "ask the user"@@ -458,15 +462,18 @@ def test_repair_is_noop_on_the_main_loop() -> None:assert messages is transcript-def test_repair_drops_a_turn_left_with_no_blocks() -> None:- # An assistant turn that was ONLY the search round-trip must be removed, not- # forwarded with an empty content array (which Anthropic also rejects).+def test_repair_keeps_a_turn_that_was_only_bookkeeping() -> None:+ # An assistant turn that was ONLY the search round-trip keeps its message+ # slot (an all-text turn is valid, an empty content array is not). Dropping+ # the message would shift every later message index and so move any signed+ # thinking block, which makes select_outbound_body discard the repair (#3456).transcript = _poisoned_transcript()transcript[1]["content"] = transcript[1]["content"][1:3]messages, removed = strip_unsupported_tool_search_blocks(transcript, [])assert removed == 2- assert len(messages) == 1- assert messages[0]["role"] == "user"+ assert len(messages) == 2+ assert messages[1]["role"] == "assistant"+ assert [b["type"] for b in messages[1]["content"]] == ["text", "text"]def test_repair_leaves_other_server_tools_alone() -> None:@@ -613,10 +620,10 @@ def test_repair_drops_search_tool_self_reference_when_inject_ran() -> None:{"name": "mcp_tool_x", "input_schema": {}, "defer_loading": True},]messages, removed = strip_unsupported_tool_search_blocks(transcript, tools)- assert removed == 2 # server_tool_use + tool_search_tool_result both dropped- # The assistant turn is entirely stripped (only search blocks were present).- assert len(messages) == 1- assert messages[0]["role"] == "user"+ assert removed == 2 # server_tool_use + tool_search_tool_result both repaired+ # The assistant turn keeps its slot; both search blocks became text.+ assert len(messages) == 2+ assert [b["type"] for b in messages[1]["content"]] == ["text", "text"]def test_repair_noop_when_referenced_tool_is_regular_deferred_tool() -> None:@@ -645,3 +652,56 @@ def test_repair_drops_when_referenced_tool_absent_despite_search_tool_present()]messages, removed = strip_unsupported_tool_search_blocks(transcript, tools)assert removed == 2+++def test_repair_does_not_move_signed_thinking_blocks() -> None:+ # Production failure (#3456): the unsupportable block sat in the SAME+ # assistant message as signed thinking blocks. Removing it moved the+ # thinking block that followed it, thinking_blocks_survived_mutation went+ # False, and select_outbound_body forwarded the client's ORIGINAL bytes --+ # discarding the repair, so upstream 400'd on the very reference we found.+ # Repairing in place must leave the thinking fingerprint byte-identical.+ from headroom.proxy.body_forwarding import thinking_block_fingerprint++ transcript = [+ {"role": "user", "content": [{"type": "text", "text": "go"}]},+ {+ "role": "assistant",+ "content": [+ {"type": "thinking", "thinking": "first", "signature": "sig-1"},+ {"type": "text", "text": "Checking."},+ {+ "type": "server_tool_use",+ "id": "srvtoolu_01ABC",+ "name": _TOOL_SEARCH_DEFAULT_NAME,+ "input": {},+ },+ {+ "type": "tool_search_tool_result",+ "tool_use_id": "srvtoolu_01ABC",+ "content": {+ "type": "tool_search_tool_search_result",+ "tool_references": [+ {"type": "tool_reference", "tool_name": "mcp__gone__tool"}+ ],+ },+ },+ {"type": "thinking", "thinking": "second", "signature": "sig-2"},+ ],+ },+ {"role": "user", "content": [{"type": "text", "text": "next"}]},+ ]+ before = thinking_block_fingerprint({"messages": transcript})++ messages, removed = strip_unsupported_tool_search_blocks(+ transcript, [_SEARCH_TOOL, {"name": "Bash", "input_schema": {}}]+ )++ assert removed == 2+ assert thinking_block_fingerprint({"messages": messages}) == before+ assert not [+ block+ for message in messages+ for block in message["content"]+ if block["type"] in ("tool_search_tool_result", "server_tool_use")+ ]