Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): repair tool-search history in place so signed thinking blocks keep their coordinates (#3457)

Eyal Mizrachi68359c056cc9
diff --git a/headroom/proxy/handlers/_debug_dump.py b/headroom/proxy/handlers/_debug_dump.py
index 6fbc46fa3..b04459fb3 100644
--- a/headroom/proxy/handlers/_debug_dump.py
+++ b/headroom/proxy/handlers/_debug_dump.py
@@ -8,8 +8,15 @@ OpenAI handlers so the gating stays consistent across providers.
from __future__ import annotations
+import json
+import logging
import os
+from datetime import datetime
+from pathlib import Path
from typing import Any
+from urllib.parse import urlsplit, urlunsplit
+
+logger = logging.getLogger(__name__)
def _debug_dump_mode(config: Any) -> str:
@@ -45,3 +52,90 @@ def _redact_debug_value(value: Any, _max_len: int = 80) -> Any:
if isinstance(value, list):
return [_redact_debug_value(v, _max_len) for v in value]
return value
+
+
+def _redact_url(url: str) -> str:
+ """Drop the query string and fragment from ``url``.
+
+ Some upstreams carry credentials in the query (Gemini's ``?key=``), and the
+ query is never needed to debug a rejected request, so it is elided in every
+ mode — ``full`` opts in to prompt content, not to credentials.
+ """
+ try:
+ parts = urlsplit(url)
+ except ValueError:
+ return "<unparseable url>"
+ if not parts.query and not parts.fragment:
+ return url
+ redacted = urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
+ return f"{redacted}?<redacted>" if parts.query else redacted
+
+
+def _decode_sent_body(body: Any) -> Any:
+ """Decode the exact bytes sent upstream back to JSON for the dump."""
+ if not isinstance(body, (bytes, bytearray)):
+ return body
+ try:
+ return json.loads(body)
+ except ValueError:
+ return f"<{len(body)} bytes, not JSON>"
+
+
+def write_upstream_error_dump(
+ config: Any,
+ *,
+ request_id: str,
+ url: str,
+ status: int,
+ provider: str | None = None,
+ model: str | None = None,
+ body: Any = None,
+ body_source: str | None = None,
+ transforms: Any = None,
+ stream: bool = False,
+) -> Path | None:
+ """Write a diagnostic dump of an erroring (>=400) upstream request.
+
+ ``body`` may be the raw bytes that went on the wire; they are decoded here,
+ after the off-by-default gate, so a disabled dump costs nothing.
+ ``body_source`` records which body was forwarded (for example
+ ``"passthrough"`` when the proxy's mutations were dropped), since that is
+ often the whole explanation for a 400.
+
+ Returns the path written, or ``None`` when the dump is disabled (the
+ default, and always in stateless mode) or could not be written. Never
+ raises: a diagnostic must not turn an upstream error into a proxy error.
+ """
+ mode = _debug_dump_mode(config)
+ if mode == "off":
+ return None
+ try:
+ from headroom import paths as _hr_paths
+
+ dump_dir = _hr_paths.debug_400_dir()
+ dump_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
+ dump_path = dump_dir / f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_{request_id}.json"
+ sent_body = _decode_sent_body(body)
+ payload = json.dumps(
+ {
+ "request_id": request_id,
+ "url": _redact_url(url),
+ "status": status,
+ "provider": provider,
+ "model": model,
+ "stream": stream,
+ "transforms": transforms,
+ "body_source": body_source,
+ "body": sent_body if mode == "full" else _redact_debug_value(sent_body),
+ },
+ indent=2,
+ default=str,
+ )
+ # The dump can hold prompt content: keep it readable by the owner only.
+ fd = os.open(dump_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+ with os.fdopen(fd, "w", encoding="utf-8") as fh:
+ fh.write(payload)
+ return dump_path
+ except Exception:
+ logger.debug("upstream error debug dump skipped", exc_info=True)
+ return None
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index dc88815ff..b648ba0a1 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -3140,8 +3140,9 @@ class AnthropicHandlerMixin:
body_mutation_tracker.mark_mutated("tool_search_history_repair")
transforms_applied.append(f"router:tool_search_repair:{_ts_stripped}blocks")
logger.info(
- "[%s] Tool search: dropped %d unsupportable history block(s) "
- "(tools array cannot resolve their tool_reference entries)",
+ "[%s] Tool search: repaired %d unsupportable history block(s) "
+ "(replaced with text in place; tools array cannot resolve their "
+ "tool_reference entries)",
request_id,
_ts_stripped,
)
diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.py
index 66e89fae3..b364c3c7a 100644
--- a/headroom/proxy/handlers/streaming.py
+++ b/headroom/proxy/handlers/streaming.py
@@ -13,6 +13,7 @@ import time
from typing import TYPE_CHECKING, Any
from headroom.proxy.auth_mode import classify_client, supports_mid_turn_coalescing
+from headroom.proxy.handlers._debug_dump import write_upstream_error_dump
from headroom.proxy.helpers import (
RETRYABLE_OVERLOAD_STATUSES,
jitter_delay_ms,
@@ -1463,6 +1464,27 @@ class StreamingMixin:
upstream_response.status_code,
url,
)
+ # Diagnostic dump of the erroring request — parity with the
+ # non-streaming handlers, which dump on >=400 but never fire for a
+ # streaming turn (Claude Code streams every request, so the most
+ # common 400s were invisible). Same gating: OFF by default, never
+ # in stateless mode, content redacted unless HEADROOM_DEBUG_DUMP=full.
+ # Dump the bytes that went on the wire, not ``body``: when the edits
+ # are dropped (source="passthrough") ``body`` shows a request that
+ # never left the proxy.
+ write_upstream_error_dump(
+ getattr(self, "config", None),
+ request_id=request_id,
+ url=url,
+ status=upstream_response.status_code,
+ provider=provider,
+ model=model,
+ body=outbound_bytes,
+ body_source=outbound_source,
+ transforms=transforms_applied,
+ stream=True,
+ )
+
response_headers = dict(upstream_response.headers)
response_headers.pop("content-length", None)
response_headers.pop("transfer-encoding", None)
diff --git a/headroom/proxy/helpers.py b/headroom/proxy/helpers.py
index 8e8766bda..2231f9ae0 100644
--- a/headroom/proxy/helpers.py
+++ b/headroom/proxy/helpers.py
@@ -3285,17 +3285,38 @@ def _tool_search_reference_names(content: Any) -> list[str]:
return names
+# Stand-in for a tool-search block the outbound tools array cannot support. Text
+# so it is inert to every validator, short so it costs ~10 tokens, and constant so
+# the repaired prefix stays byte-stable across turns (the provider cache needs the
+# same bytes every time).
+_TOOL_SEARCH_PLACEHOLDER_BLOCK: dict[str, Any] = {
+ "type": "text",
+ "text": "[tool search omitted: unavailable in this request]",
+}
+
+
def strip_unsupported_tool_search_blocks(messages: Any, tools: Any) -> tuple[Any, int]:
- """Drop tool-search blocks this request's ``tools`` array cannot support.
+ """Neutralize tool-search blocks this request's ``tools`` array cannot support.
A block pair is unsupportable when the request carries no ``tool_search_tool_*``
tool, or when a ``tool_reference`` names a tool absent from ``tools`` — the two
shapes Anthropic rejects. Both the ``tool_search_tool_result`` and its paired
- ``server_tool_use`` are removed (an orphan of either 400s on its own), and a
- message left with no content blocks is dropped rather than sent empty.
-
- Returns ``(messages, blocks_removed)``, and the ORIGINAL ``messages`` object
- when nothing was removed — callers rely on identity to skip the write-back.
+ ``server_tool_use`` are handled (an orphan of either 400s on its own).
+
+ Replace in place rather than remove (#3456). The block indexes of a message
+ are load-bearing: ``thinking_block_fingerprint`` keys a signed thinking block
+ by ``(message_index, block_index)``, so deleting a block that sits BEFORE a
+ thinking block in the same message — or deleting a whole message ahead of one —
+ moves that block, ``thinking_blocks_survived_mutation`` reports False, and
+ ``select_outbound_body`` then forwards the client's ORIGINAL bytes and discards
+ every mutation, this repair included. The request that needed repairing is
+ exactly the one that loses it, and upstream 400s on the reference we had
+ already found. Swapping each block for a short text block keeps every thinking
+ block at its original coordinates, so the repair survives to the wire. Same
+ reasoning as the CCR sibling below.
+
+ Returns ``(messages, blocks_repaired)``, and the ORIGINAL ``messages`` object
+ when nothing changed — callers rely on identity to skip the write-back.
"""
if not isinstance(messages, list):
return messages, 0
@@ -3328,7 +3349,7 @@ def strip_unsupported_tool_search_blocks(messages: Any, tools: Any) -> tuple[Any
out.append(message)
continue
- drop_indexes: set[int] = set()
+ neutralize_indexes: set[int] = set()
orphaned_ids: set[str] = set()
for index, block in enumerate(content):
if not isinstance(block, dict) or block.get("type") != _TOOL_SEARCH_RESULT_TYPE:
@@ -3336,7 +3357,7 @@ def strip_unsupported_tool_search_blocks(messages: Any, tools: Any) -> tuple[Any
names = _tool_search_reference_names(block.get("content"))
if has_search_tool and all(name in available for name in names):
continue
- drop_indexes.add(index)
+ neutralize_indexes.add(index)
use_id = block.get("tool_use_id")
if use_id:
orphaned_ids.add(str(use_id))
@@ -3348,19 +3369,19 @@ def strip_unsupported_tool_search_blocks(messages: Any, tools: Any) -> tuple[Any
continue
is_search_call = str(block.get("name", "")).startswith(_TOOL_SEARCH_TOOL_TYPE_PREFIX)
if str(block.get("id", "")) in orphaned_ids or (is_search_call and not has_search_tool):
- drop_indexes.add(index)
+ neutralize_indexes.add(index)
- if not drop_indexes:
+ if not neutralize_indexes:
out.append(message)
continue
changed = True
- removed += len(drop_indexes)
- kept = [block for index, block in enumerate(content) if index not in drop_indexes]
- if not kept:
- continue # the whole turn was tool-search bookkeeping
+ removed += len(neutralize_indexes)
repaired = dict(message)
- repaired["content"] = kept
+ repaired["content"] = [
+ dict(_TOOL_SEARCH_PLACEHOLDER_BLOCK) if index in neutralize_indexes else block
+ for index, block in enumerate(content)
+ ]
out.append(repaired)
return (out, removed) if changed else (messages, 0)
diff --git a/tests/test_debug_dump_gating.py b/tests/test_debug_dump_gating.py
index 3bc85bbbe..924c9d3fa 100644
--- a/tests/test_debug_dump_gating.py
+++ b/tests/test_debug_dump_gating.py
@@ -8,6 +8,8 @@ operator explicitly opts in to full content.
from __future__ import annotations
import inspect
+import json
+import os
from types import SimpleNamespace
import pytest
@@ -94,3 +96,143 @@ def test_both_handlers_gate_the_dump(module_name):
assert 'if dump_mode != "off":' in src, (
f"{module_name} debug dump is not guarded by an off-by-default check"
)
+
+
+# ---------------------------------------------------------------------------
+# write_upstream_error_dump: the shared writer used by the streaming handler
+# ---------------------------------------------------------------------------
+
+
+def dump_dir(tmp_path, monkeypatch):
+ """Point ``paths.debug_400_dir()`` at a temp dir for the writer tests."""
+ from headroom import paths
+
+ target = tmp_path / "debug_400"
+ monkeypatch.setattr(paths, "debug_400_dir", lambda: target)
+ return target
+
+
+def _write(**kwargs):
+ from headroom.proxy.handlers._debug_dump import write_upstream_error_dump
+
+ params = {
+ "request_id": "req_1",
+ "url": "https://api.anthropic.com/v1/messages",
+ "status": 400,
+ "provider": "anthropic",
+ "model": "claude-opus-5",
+ "body": {"messages": [{"role": "user", "content": "secret prompt content " * 20}]},
+ "transforms": ["tool_search_history_repair"],
+ "stream": True,
+ }
+ params.update(kwargs)
+ config = params.pop("config", _config())
+ return write_upstream_error_dump(config, **params)
+
+
+def test_upstream_dump_off_by_default_writes_nothing(dump_dir, monkeypatch):
+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
+ assert _write() is None
+ assert not dump_dir.exists()
+
+
+def test_upstream_dump_stateless_writes_nothing(dump_dir, monkeypatch):
+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")
+ assert _write(config=_config(stateless=True)) is None
+ assert not dump_dir.exists()
+
+
+def test_upstream_dump_full_records_request(dump_dir, monkeypatch):
+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")
+ path = _write()
+ assert path is not None and path.parent == dump_dir
+ payload = json.loads(path.read_text())
+ assert payload["request_id"] == "req_1"
+ assert payload["status"] == 400
+ assert payload["provider"] == "anthropic"
+ assert payload["model"] == "claude-opus-5"
+ assert payload["stream"] is True
+ assert payload["transforms"] == ["tool_search_history_repair"]
+ assert "secret prompt content" in payload["body"]["messages"][0]["content"]
+
+
+def test_upstream_dump_redacted_elides_content(dump_dir, monkeypatch):
+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "1")
+ path = _write()
+ assert path is not None
+ payload = json.loads(path.read_text())
+ content = payload["body"]["messages"][0]["content"]
+ assert content.startswith("<redacted:") and "secret prompt" not in content
+ # Structure is still there to debug with:
+ assert payload["body"]["messages"][0]["role"] == "user"
+
+
+def test_upstream_dump_serializes_non_json_values(dump_dir, monkeypatch):
+ # ``default=str`` must keep an exotic body from raising mid-dump.
+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")
+ path = _write(body={"when": object()})
+ assert path is not None
+ assert "object object" in json.loads(path.read_text())["body"]["when"]
+
+
[email protected]("mode", ["1", "full"])
+def test_upstream_dump_never_writes_url_credentials(dump_dir, monkeypatch, mode):
+ # Gemini streaming URLs carry the API key as ``?key=``; ``full`` opts in to
+ # prompt content, not to credentials, so the query is dropped in every mode.
+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", mode)
+ url = (
+ "https://generativelanguage.googleapis.com/v1beta/models/gemini:streamGenerateContent"
+ "?alt=sse&key=AIzaSECRET"
+ )
+ path = _write(url=url, provider="gemini")
+ assert path is not None
+ text = path.read_text()
+ assert "AIzaSECRET" not in text
+ assert json.loads(text)["url"] == (
+ "https://generativelanguage.googleapis.com/v1beta/models/gemini:streamGenerateContent"
+ "?<redacted>"
+ )
+
+
+def test_upstream_dump_keeps_query_free_url(dump_dir, monkeypatch):
+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")
+ path = _write()
+ assert json.loads(path.read_text())["url"] == "https://api.anthropic.com/v1/messages"
+
+
+def test_upstream_dump_records_the_bytes_actually_sent(dump_dir, monkeypatch):
+ # When the proxy's edits are dropped (passthrough), the dump must show the
+ # body that went on the wire, not the edited one that never left.
+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")
+ sent = json.dumps({"messages": [{"role": "user", "content": "as sent"}]}).encode()
+ path = _write(body=sent, body_source="passthrough")
+ payload = json.loads(path.read_text())
+ assert payload["body"]["messages"][0]["content"] == "as sent"
+ assert payload["body_source"] == "passthrough"
+
+
+def test_upstream_dump_tolerates_non_json_bytes(dump_dir, monkeypatch):
+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")
+ path = _write(body=b"\x00\x01not json")
+ assert json.loads(path.read_text())["body"] == "<10 bytes, not JSON>"
+
+
[email protected](os.name == "nt", reason="POSIX permission bits")
+def test_upstream_dump_is_owner_only(dump_dir, monkeypatch):
+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")
+ path = _write()
+ assert path.stat().st_mode & 0o777 == 0o600
+
+
+def test_upstream_dump_never_raises_when_write_fails(monkeypatch):
+ # A diagnostic must not turn an upstream error into a proxy error.
+ from headroom import paths
+
+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")
+
+ def _boom():
+ raise OSError("read-only filesystem")
+
+ monkeypatch.setattr(paths, "debug_400_dir", _boom)
+ assert _write() is None
diff --git a/tests/test_issue_746_tool_search.py b/tests/test_issue_746_tool_search.py
index b3321a010..6ee4a70ae 100644
--- a/tests/test_issue_746_tool_search.py
+++ b/tests/test_issue_746_tool_search.py
@@ -433,7 +433,7 @@ def _poisoned_transcript() -> list[dict]:
]
-def test_repair_drops_blocks_the_hook_evaluator_cannot_resolve() -> None:
+def test_repair_neutralizes_blocks_the_hook_evaluator_cannot_resolve() -> None:
# The Stop hook evaluator replays the transcript with a small tools array
# that has neither the search tool nor AskUserQuestion -> upstream 400.
messages, removed = strip_unsupported_tool_search_blocks(
@@ -441,7 +441,11 @@ def test_repair_drops_blocks_the_hook_evaluator_cannot_resolve() -> None:
)
assert removed == 2 # server_tool_use + tool_search_tool_result
kinds = [b["type"] for b in messages[1]["content"]]
- assert kinds == ["text", "text"] # surrounding assistant text survives
+ assert kinds == ["text", "text", "text", "text"] # both swapped for text in place
+ assert messages[1]["content"][0]["text"] == "Searching for a tool."
+ assert messages[1]["content"][3]["text"] == "Found it." # index preserved
+ assert "tool search omitted" in messages[1]["content"][1]["text"]
+ assert "tool search omitted" in messages[1]["content"][2]["text"]
assert messages[0]["content"][0]["text"] == "ask the user"
@@ -458,15 +462,18 @@ def test_repair_is_noop_on_the_main_loop() -> None:
assert messages is transcript
-def test_repair_drops_a_turn_left_with_no_blocks() -> None:
- # An assistant turn that was ONLY the search round-trip must be removed, not
- # forwarded with an empty content array (which Anthropic also rejects).
+def test_repair_keeps_a_turn_that_was_only_bookkeeping() -> None:
+ # An assistant turn that was ONLY the search round-trip keeps its message
+ # slot (an all-text turn is valid, an empty content array is not). Dropping
+ # the message would shift every later message index and so move any signed
+ # thinking block, which makes select_outbound_body discard the repair (#3456).
transcript = _poisoned_transcript()
transcript[1]["content"] = transcript[1]["content"][1:3]
messages, removed = strip_unsupported_tool_search_blocks(transcript, [])
assert removed == 2
- assert len(messages) == 1
- assert messages[0]["role"] == "user"
+ assert len(messages) == 2
+ assert messages[1]["role"] == "assistant"
+ assert [b["type"] for b in messages[1]["content"]] == ["text", "text"]
def test_repair_leaves_other_server_tools_alone() -> None:
@@ -613,10 +620,10 @@ def test_repair_drops_search_tool_self_reference_when_inject_ran() -> None:
{"name": "mcp_tool_x", "input_schema": {}, "defer_loading": True},
]
messages, removed = strip_unsupported_tool_search_blocks(transcript, tools)
- assert removed == 2 # server_tool_use + tool_search_tool_result both dropped
- # The assistant turn is entirely stripped (only search blocks were present).
- assert len(messages) == 1
- assert messages[0]["role"] == "user"
+ assert removed == 2 # server_tool_use + tool_search_tool_result both repaired
+ # The assistant turn keeps its slot; both search blocks became text.
+ assert len(messages) == 2
+ assert [b["type"] for b in messages[1]["content"]] == ["text", "text"]
def test_repair_noop_when_referenced_tool_is_regular_deferred_tool() -> None:
@@ -645,3 +652,56 @@ def test_repair_drops_when_referenced_tool_absent_despite_search_tool_present()
]
messages, removed = strip_unsupported_tool_search_blocks(transcript, tools)
assert removed == 2
+
+
+def test_repair_does_not_move_signed_thinking_blocks() -> None:
+ # Production failure (#3456): the unsupportable block sat in the SAME
+ # assistant message as signed thinking blocks. Removing it moved the
+ # thinking block that followed it, thinking_blocks_survived_mutation went
+ # False, and select_outbound_body forwarded the client's ORIGINAL bytes --
+ # discarding the repair, so upstream 400'd on the very reference we found.
+ # Repairing in place must leave the thinking fingerprint byte-identical.
+ from headroom.proxy.body_forwarding import thinking_block_fingerprint
+
+ transcript = [
+ {"role": "user", "content": [{"type": "text", "text": "go"}]},
+ {
+ "role": "assistant",
+ "content": [
+ {"type": "thinking", "thinking": "first", "signature": "sig-1"},
+ {"type": "text", "text": "Checking."},
+ {
+ "type": "server_tool_use",
+ "id": "srvtoolu_01ABC",
+ "name": _TOOL_SEARCH_DEFAULT_NAME,
+ "input": {},
+ },
+ {
+ "type": "tool_search_tool_result",
+ "tool_use_id": "srvtoolu_01ABC",
+ "content": {
+ "type": "tool_search_tool_search_result",
+ "tool_references": [
+ {"type": "tool_reference", "tool_name": "mcp__gone__tool"}
+ ],
+ },
+ },
+ {"type": "thinking", "thinking": "second", "signature": "sig-2"},
+ ],
+ },
+ {"role": "user", "content": [{"type": "text", "text": "next"}]},
+ ]
+ before = thinking_block_fingerprint({"messages": transcript})
+
+ messages, removed = strip_unsupported_tool_search_blocks(
+ transcript, [_SEARCH_TOOL, {"name": "Bash", "input_schema": {}}]
+ )
+
+ assert removed == 2
+ assert thinking_block_fingerprint({"messages": messages}) == before
+ assert not [
+ block
+ for message in messages
+ for block in message["content"]
+ if block["type"] in ("tool_search_tool_result", "server_tool_use")
+ ]