Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(tool-search): detect every wire shape of client-side deferral (#3746)
Tejas Chopra665b73df1b25
diff --git a/headroom/proxy/helpers.py b/headroom/proxy/helpers.pyindex 74835d7ab..722df823a 100644--- a/headroom/proxy/helpers.py+++ b/headroom/proxy/helpers.py@@ -18,7 +18,7 @@ import reimport threadingimport timefrom collections import OrderedDict-from collections.abc import Callable+from collections.abc import Callable, Iteratorfrom dataclasses import dataclassfrom pathlib import Pathfrom typing import TYPE_CHECKING, Any, Literal, cast@@ -3142,6 +3142,26 @@ def compute_turn_id(# ---------------------------------------------------------------------------_TOOL_SEARCH_TOOL_TYPE_PREFIX = "tool_search_tool_"+# Every wire spelling of "this client is already deferring its tool schemas".+# Deferral is no longer an Anthropic-only feature: Codex and the Responses API+# use a bare ``tool_search`` type that carries no ``name`` at all, GitHub+# Copilot CLI ships ``tool_search_tool``, VS Code Copilot and Kiro use+# ``tool_search``, and Codebuff uses ``composio_search_tools``. Matching only+# Anthropic's versioned ``tool_search_tool_*`` prefix misses all of them --+# including the exact string ``tool_search_tool``, which does not start with+# ``tool_search_tool_``.+_RESPONSES_TOOL_SEARCH_TYPE = "tool_search"+_TOOL_SEARCH_META_TOOL_NAMES = frozenset(+ {+ _RESPONSES_TOOL_SEARCH_TYPE,+ "tool_search_tool",+ "tool_search_tool_regex",+ "composio_search_tools",+ }+)+# Responses lets a client group tools under one entry that carries the real+# tools in a nested ``tools`` array; Codex ships its MCP servers that way.+_NAMESPACE_TOOL_TYPE = "namespace"# Substrings of the ``anthropic-beta`` tokens that gate tool search:# ``advanced-tool-use-2025-11-20`` (firstParty/foundry) and# ``tool-search-tool-2025-10-19`` (vertex/bedrock/mantle/gateway).@@ -3507,15 +3527,72 @@ def _client_tool_search_names() -> frozenset[str]:}+def iter_tool_entries(tools: Any) -> Iterator[dict[str, Any]]:+ """Yield every tool dict in ``tools``, descending into namespace groups.++ A Responses client may group tools under+ ``{"type": "namespace", "name": ..., "tools": [...]}`` and Codex ships its+ MCP servers that way. A scan that reads only top-level ``tools[].name``+ sees the wrapper and none of the tools inside it, so every nested tool is+ invisible to both deferral detection and the resident/defer decision --+ and a namespace is exactly where a large MCP catalog lives.++ One level is what the API defines, and that is all this descends; a nested+ ``tools`` value that is not a list is skipped rather than trusted.+ """++ if not isinstance(tools, list):+ return+ for tool in tools:+ if not isinstance(tool, dict):+ continue+ yield tool+ if tool.get("type") != _NAMESPACE_TOOL_TYPE:+ continue+ for nested in tool.get("tools") or []:+ if isinstance(nested, dict):+ yield nested+++def _is_tool_search_meta_tool(tool: dict[str, Any]) -> bool:+ """Whether one tool entry is a tool-search meta-tool, in any known spelling."""++ ttype = str(tool.get("type", ""))+ if ttype.startswith(_TOOL_SEARCH_TOOL_TYPE_PREFIX) or ttype in _TOOL_SEARCH_META_TOOL_NAMES:+ return True+ name = _tool_search_resident_key(tool.get("name"))+ return (+ name.startswith(_TOOL_SEARCH_TOOL_TYPE_PREFIX)+ or name in _TOOL_SEARCH_META_TOOL_NAMES+ or name in _client_tool_search_names()+ )++def request_already_defers_tools(tools: Any) -> bool:"""Whether ``tools`` shows the CLIENT is already deferring tool schemas.- True for the Messages API server-side shape (``tool_search_tool_*`` as- either ``type`` or ``name``), which only appears when deferral is actually- on. NOT true for a bare client-side tool name such as ``ToolSearch``, which- a client may send whether or not it is deferring — see+ Keyed on wire shape, never on a client allowlist, so it generalizes to+ harnesses we have not seen. Three independent signals, any of which is+ conclusive:++ * a tool-search meta-tool in any known spelling -- Anthropic's versioned+ ``tool_search_tool_*``, the Responses/Codex bare ``tool_search`` type+ (which carries no ``name`` at all), Copilot's ``tool_search_tool``;+ * ``defer_loading`` already set on any tool, which is the client marking+ its own catalog deferred and is shape-independent;+ * either of the above on a tool nested inside a ``namespace`` entry.++ NOT true for a bare client-side tool name such as ``ToolSearch``, which a+ client may send whether or not it is deferring -- see``_CLIENT_TOOL_SEARCH_NAMES``.+ Deferring on top of a client that already defers suppresses ITS mechanism+ and inlines the catalog we were trying to keep out, so a miss here costs+ real tokens. That is no longer a rare case: Codex, GitHub Copilot CLI, VS+ Code Copilot, Kiro and Codebuff all ship their own tool search, and on+ Copilot's Anthropic path the provider's server-side search can be a third+ mechanism in the same request.+Upstream-independent on purpose: the question is what the client is doing,not where the request is forwarded, so the same answer holds for first-partyAnthropic, Bedrock, Vertex and gateways. That matters because through Kong@@ -3523,19 +3600,65 @@ def request_already_defers_tools(tools: Any) -> bool:deferral is the only one available."""- if not isinstance(tools, list):- return False- for tool in tools:- if not isinstance(tool, dict):- continue- if str(tool.get("type", "")).startswith(_TOOL_SEARCH_TOOL_TYPE_PREFIX):- return True- name = _tool_search_resident_key(tool.get("name"))- if name.startswith(_TOOL_SEARCH_TOOL_TYPE_PREFIX) or name in _client_tool_search_names():+ deferred_without_search_tool = False+ for tool in iter_tool_entries(tools):+ if _is_tool_search_meta_tool(tool):return True+ if tool.get("defer_loading") is True:+ deferred_without_search_tool = True+ if deferred_without_search_tool:+ # Tools marked deferred with no meta-tool we recognize that could+ # resolve them. Two very different causes, and the operator wants to+ # know which: a harness whose search tool is spelled in a way this+ # survey missed (benign -- standing down is right, and the log names the+ # tools so the spelling can be added), or an intermediary that stripped+ # the search tool and left the marks behind, which upstream will reject.+ # Headroom's own third-party strip clears both halves precisely so it+ # cannot be the cause, so seeing this points outside us.+ _warn_deferred_without_search_tool(tools)+ return Truereturn False+_DEFERRED_ORPHAN_WARN_INTERVAL_S = 3600.0+_deferred_orphan_lock = threading.Lock()+_deferred_orphan_last: float | None = None+++def reset_deferred_orphan_warn_state() -> None:+ """Re-arm the orphaned-deferral warning. For tests."""+ global _deferred_orphan_last+ with _deferred_orphan_lock:+ _deferred_orphan_last = None+++def _warn_deferred_without_search_tool(tools: Any) -> None:+ """Warn, at most hourly, that deferred tools have nothing to resolve them."""++ global _deferred_orphan_last+ now = time.monotonic()+ with _deferred_orphan_lock:+ last = _deferred_orphan_last+ if last is not None and (now - last) < _DEFERRED_ORPHAN_WARN_INTERVAL_S:+ return+ _deferred_orphan_last = now+ names = [+ str(t.get("name") or t.get("type") or "?")+ for t in iter_tool_entries(tools)+ if t.get("defer_loading") is True+ ]+ logger.warning(+ "event=tool_search_deferred_orphan deferred=%d names=%s hint=%s",+ len(names),+ ",".join(sorted(names)[:8]),+ "tools are marked defer_loading but no recognized tool-search tool can "+ "resolve them; Headroom is standing down. Either this harness spells its "+ "search tool in a way Headroom does not know (set "+ "HEADROOM_CLIENT_TOOL_SEARCH_NAMES to teach it) or an intermediary "+ "stripped the search tool and left the marks, which the upstream will reject",+ )++def anthropic_first_party_tool_search_supported(api_base_url: str | None) -> bool:"""Return whether Anthropic server-side tool search is valid for this upstream."""from headroom.providers.claude.runtime import is_custom_anthropic_base_url@@ -3967,7 +4090,7 @@ def strip_unsupported_ccr_retrieve_blocks(messages: Any, tools: Any) -> tuple[An# * No ``cache_control`` (OpenAI caches automatically), so no breakpoint move.# ----------------------------------------------------------------------------_OPENAI_TOOL_SEARCH_TYPE = "tool_search"+_OPENAI_TOOL_SEARCH_TYPE = _RESPONSES_TOOL_SEARCH_TYPE_OPENAI_TOOL_SEARCH_MIN_TOOLS = 12_OPENAI_TOOL_SEARCH_RESIDENT_NAMES = frozenset({"terminal"})_OPENAI_TOOL_SEARCH_UNSUPPORTED_CLIENTS = frozenset({"codex", "opencode"})@@ -4034,16 +4157,19 @@ def inject_tool_search_deferral_openai(return toolsif not _model_supports_openai_tool_search(model):return tools- if not isinstance(tools, list) or len(tools) < _OPENAI_TOOL_SEARCH_MIN_TOOLS:+ if not isinstance(tools, list):return tools- if any(- isinstance(tool, dict) and tool.get("type") == _OPENAI_TOOL_SEARCH_TYPE for tool in tools- ) or request_already_defers_tools(tools):- # Client already defers — its own Responses tool_search, or a- # client-side tool such as Claude Code's ToolSearch. Deferring on top of+ # Count nested namespace members too. A client that groups a 30-tool MCP+ # catalog under one namespace entry presents a handful of top-level entries,+ # so a top-level-only count reads it as a small tool surface and skips+ # exactly the request with the most schema to save.+ if sum(1 for _ in iter_tool_entries(tools)) < _OPENAI_TOOL_SEARCH_MIN_TOOLS:+ return tools+ if request_already_defers_tools(tools):+ # Client already defers — its own Responses tool_search, Copilot's+ # tool_search_tool, or defer_loading it set itself. Deferring on top of# a client that is already deferring suppresses ITS mechanism and- # inlines the catalog we were trying to keep out. Shape-based, not a- # client allowlist, so it generalizes to harnesses we have not seen.+ # inlines the catalog we were trying to keep out.return toolsout: list[Any] = [{"type": _OPENAI_TOOL_SEARCH_TYPE}]diff --git a/tests/test_tool_search_client_deferral.py b/tests/test_tool_search_client_deferral.pyindex 9592cef20..80b048cd3 100644--- a/tests/test_tool_search_client_deferral.py+++ b/tests/test_tool_search_client_deferral.py@@ -5,7 +5,13 @@ Two mechanisms exist at two layers:* **Server-side (Messages API / Responses).** ``tool_search_tool_regex`` /``tool_search_tool_bm25`` types paired with ``defer_loading: true``, orOpenAI's ``{"type": "tool_search"}``. This shape appears ONLY when deferral- is actually on, so it is an unambiguous signal.+ is actually on, so it is an unambiguous signal. It has no single spelling:+ a survey of shipped harnesses found tool search of their own in Codex+ (``tool_search``, a type with no ``name`` at all), GitHub Copilot CLI+ (``tool_search_tool``), VS Code Copilot and Kiro (``tool_search``) and+ Codebuff (``composio_search_tools``) — so the client already deferring is+ the common case, not the exception, and ``defer_loading`` on any tool is+ the one signal that needs no agreement about names.* **Client-side.** Claude Code carries a tool named ``ToolSearch``. It resolvestools the client keeps in a local registry (TaskCreate, WebFetch, ...) andreal traffic shows it riding alongside a fully inline MCP catalog. Its@@ -25,6 +31,7 @@ So: key on the server-side shape, never on the bare client-side name.from __future__ import annotations+import loggingfrom typing import Anyimport pytest@@ -34,7 +41,9 @@ from headroom.proxy.helpers import (claude_code_tool_search_inactive,inject_tool_search_deferral,inject_tool_search_deferral_openai,+ iter_tool_entries,request_already_defers_tools,+ reset_deferred_orphan_warn_state,resolved_core_tools,strip_first_party_tool_search_tools_for_third_party_upstream,)@@ -110,7 +119,6 @@ def test_server_side_shape_means_the_client_defers(marker: dict[str, Any]) -> Nopytest.param(None, id="not-a-list"),pytest.param([{"name": "mcp__search__tools"}], id="mcp-tool-named-search"),pytest.param([{"name": "SearchTool"}], id="reversed-words"),- pytest.param([{"name": "tool_search"}], id="prefix-but-incomplete"),pytest.param([{"type": "web_search"}], id="unrelated-server-tool"),pytest.param(["not-a-dict", 7], id="non-dict-entries"),],@@ -119,6 +127,112 @@ def test_not_a_deferral_signal(tools: Any) -> None:assert request_already_defers_tools(tools) is False+# ---------------------------------------------------------------------------+# Deferral is not an Anthropic-only feature. A survey of shipped harnesses+# found their own tool search in Codex, GitHub Copilot CLI, VS Code Copilot,+# Kiro and Codebuff, each with a different wire spelling. Matching only+# Anthropic's versioned ``tool_search_tool_*`` prefix stood down for none of+# them, so Headroom deferred on top of a client that was already deferring --+# which suppresses the client's mechanism and inlines the very catalog we were+# keeping out.+# ---------------------------------------------------------------------------+++ "marker",+ [+ # Codex: codex-rs/tools/src/tool_spec.rs. A union variant carrying no+ # ``name`` field at all, so a name-only scan can never see it.+ pytest.param({"type": "tool_search"}, id="codex-responses-bare-type"),+ # VS Code Copilot: ToolName.tool_search; Kiro CLI uses the same name.+ pytest.param({"name": "tool_search"}, id="vscode-copilot-kiro-name"),+ # GitHub Copilot CLI: the exact string ``tool_search_tool``, which does+ # NOT start with ``tool_search_tool_`` -- the off-by-one that let the+ # prefix test miss a shipped harness.+ pytest.param({"name": "tool_search_tool"}, id="copilot-cli-exact"),+ pytest.param({"name": "tool_search_tool_regex"}, id="copilot-cli-regex"),+ # Codebuff delegates to an external catalog under its own name.+ pytest.param({"name": "composio_search_tools"}, id="codebuff"),+ ],+)+def test_every_known_meta_tool_spelling_means_the_client_defers(marker: dict[str, Any]) -> None:+ assert request_already_defers_tools([marker, *_mcp(3)]) is True+++def test_defer_loading_alone_means_the_client_defers() -> None:+ """Shape-independent: the client marking its own catalog deferred is enough.++ Worth matching on its own because it needs no agreement about what the+ meta-tool is called, so it still holds for a harness this survey missed.+ """+ tools = [{"name": "mcp__github__list_issues", "defer_loading": True}, *_mcp(3)]++ assert request_already_defers_tools(tools) is True+++def test_defer_loading_false_is_not_a_signal() -> None:+ """Present-and-false is a client that considered deferral and declined."""+ assert request_already_defers_tools([{"name": "read", "defer_loading": False}]) is False+++# --- namespace nesting -----------------------------------------------------+++def _namespace(name: str, *tools: dict[str, Any]) -> dict[str, Any]:+ """A Responses namespace group, the shape Codex ships MCP servers in."""+ return {"type": "namespace", "name": name, "tools": list(tools)}+++def test_a_meta_tool_nested_in_a_namespace_is_found() -> None:+ """The wrapper is visible to a top-level scan; its contents are not."""+ tools = [_namespace("functions", {"type": "function", "name": "tool_search_tool"})]++ assert request_already_defers_tools(tools) is True+++def test_defer_loading_nested_in_a_namespace_is_found() -> None:+ tools = [_namespace("github", {"type": "function", "name": "x", "defer_loading": True})]++ assert request_already_defers_tools(tools) is True+++def test_a_plain_namespace_is_not_a_deferral_signal() -> None:+ """Grouping tools is not deferring them."""+ tools = [_namespace("github", {"type": "function", "name": "list_issues"})]++ assert request_already_defers_tools(tools) is False+++def test_iter_tool_entries_yields_the_wrapper_and_its_members() -> None:+ nested = {"type": "function", "name": "inner"}+ group = _namespace("ns", nested)++ names = [t.get("name") for t in iter_tool_entries([{"name": "top"}, group])]++ assert names == ["top", "ns", "inner"]+++ "tools",+ [+ pytest.param(None, id="not-a-list"),+ pytest.param([{"type": "namespace"}], id="namespace-without-tools"),+ pytest.param([{"type": "namespace", "tools": "nope"}], id="tools-not-a-list"),+ pytest.param([{"type": "namespace", "tools": [7, "x"]}], id="non-dict-members"),+ ],+)+def test_iter_tool_entries_tolerates_malformed_input(tools: Any) -> None:+ """A malformed namespace must not raise on the request path."""+ assert all(isinstance(t, dict) for t in iter_tool_entries(tools))+++def test_nesting_does_not_recurse_past_one_level() -> None:+ """One level is what the API defines; deeper input is not trusted."""+ deep = _namespace("outer", _namespace("inner", {"name": "buried"}))++ assert "buried" not in [t.get("name") for t in iter_tool_entries([deep])]++def test_extra_names_can_be_configured(monkeypatch: pytest.MonkeyPatch) -> None:"""Escape hatch for a harness that does signal deferral by name."""tools = [{"name": "MyHarnessSearch"}, *_mcp(3)]@@ -327,6 +441,41 @@ def test_openai_still_defers_for_an_eager_client() -> None:assert len(_deferred(out)) == 14+def test_openai_counts_namespace_members_toward_the_threshold() -> None:+ """A big catalog grouped under one namespace is still a big catalog.++ Counting only top-level entries reads a 20-tool MCP server as a single+ tool and skips the request, which is precisely the request with the most+ schema to save.+ """+ top_level = [_fn(f"jira_{i}") for i in range(4)]+ tools = [+ *top_level,+ {"type": "namespace", "name": "slack", "tools": [_fn(f"slack_{i}") for i in range(20)]},+ ]++ assert len(tools) < _TOOL_SEARCH_MIN_TOOLS # a top-level count would bail here+ out = inject_tool_search_deferral_openai(tools, "gpt-5.5")++ # The namespace itself stays untouched; the top-level tools it made us+ # look at in the first place are what get deferred.+ assert [t["name"] for t in _deferred(out)] == [t["name"] for t in top_level]+++def test_openai_leaves_namespace_groups_resident() -> None:+ """We do not guess a wire shape for deferring inside a namespace.++ Marking nested members ``defer_loading`` is unverified against the API, and+ an unknown field in the wrong place is a 400. Counting them is safe;+ rewriting them is not, so the group is forwarded untouched.+ """+ group = {"type": "namespace", "name": "slack", "tools": [_fn(f"slack_{i}") for i in range(20)]}+ out = inject_tool_search_deferral_openai([_fn("bash"), group], "gpt-5.5")++ assert group in out+ assert _deferred(out) == []++def test_openai_toolsearch_name_alone_does_not_stand_us_down() -> None:tools = [_fn("ToolSearch"), *[_fn(f"slack_{i}") for i in range(14)]]@@ -505,3 +654,52 @@ def test_the_scan_gate_closes_even_when_the_hint_does_not_fire(assert H.tool_search_hint_pending() is False # ...and the gate closed anywayfinally:H.reset_tool_search_hint_state()+++# --- orphaned deferrals ----------------------------------------------------+++def test_deferred_tools_with_no_search_tool_warn(caplog: pytest.LogCaptureFixture) -> None:+ """Standing down here is right but ambiguous, so it must not be silent.++ Either the harness spells its search tool in a way we do not know, or an+ intermediary stripped the search tool and left the marks behind. The first+ is benign and the second is fatal upstream, and only the operator can tell+ them apart -- so name the tools and say what to do.+ """+ reset_deferred_orphan_warn_state()+ tools = [{"name": "mcp__github__list_issues", "defer_loading": True}, *_mcp(3)]++ with caplog.at_level(logging.WARNING, logger="headroom.proxy"):+ assert request_already_defers_tools(tools) is True++ lines = [+ r.getMessage() for r in caplog.records if "tool_search_deferred_orphan" in r.getMessage()+ ]+ assert len(lines) == 1+ assert "mcp__github__list_issues" in lines[0]+ assert "HEADROOM_CLIENT_TOOL_SEARCH_NAMES" in lines[0]+++def test_a_recognized_search_tool_is_not_an_orphan(caplog: pytest.LogCaptureFixture) -> None:+ """The normal deferring client must stay quiet."""+ reset_deferred_orphan_warn_state()+ tools = [ANTHROPIC_SEARCH_REGEX, {"name": "mcp__github__x", "defer_loading": True}]++ with caplog.at_level(logging.WARNING, logger="headroom.proxy"):+ assert request_already_defers_tools(tools) is True++ assert [r for r in caplog.records if "tool_search_deferred_orphan" in r.getMessage()] == []+++def test_the_orphan_warning_is_throttled(caplog: pytest.LogCaptureFixture) -> None:+ """It is evaluated per request; one line per request would be a flood."""+ reset_deferred_orphan_warn_state()+ tools = [{"name": "x", "defer_loading": True}]++ with caplog.at_level(logging.WARNING, logger="headroom.proxy"):+ for _ in range(50):+ request_already_defers_tools(tools)++ lines = [r for r in caplog.records if "tool_search_deferred_orphan" in r.getMessage()]+ assert len(lines) == 1