Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(tool-search): detect every wire shape of client-side deferral (#3746)

Tejas Chopra665b73df1b25
diff --git a/headroom/proxy/helpers.py b/headroom/proxy/helpers.py
index 74835d7ab..722df823a 100644
--- a/headroom/proxy/helpers.py
+++ b/headroom/proxy/helpers.py
@@ -18,7 +18,7 @@ import re
import threading
import time
from collections import OrderedDict
-from collections.abc import Callable
+from collections.abc import Callable, Iterator
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING, Any, Literal, cast
@@ -3142,6 +3142,26 @@ def compute_turn_id(
# ---------------------------------------------------------------------------
_TOOL_SEARCH_TOOL_TYPE_PREFIX = "tool_search_tool_"
+# Every wire spelling of "this client is already deferring its tool schemas".
+# Deferral is no longer an Anthropic-only feature: Codex and the Responses API
+# use a bare ``tool_search`` type that carries no ``name`` at all, GitHub
+# Copilot CLI ships ``tool_search_tool``, VS Code Copilot and Kiro use
+# ``tool_search``, and Codebuff uses ``composio_search_tools``. Matching only
+# Anthropic's versioned ``tool_search_tool_*`` prefix misses all of them --
+# including the exact string ``tool_search_tool``, which does not start with
+# ``tool_search_tool_``.
+_RESPONSES_TOOL_SEARCH_TYPE = "tool_search"
+_TOOL_SEARCH_META_TOOL_NAMES = frozenset(
+ {
+ _RESPONSES_TOOL_SEARCH_TYPE,
+ "tool_search_tool",
+ "tool_search_tool_regex",
+ "composio_search_tools",
+ }
+)
+# Responses lets a client group tools under one entry that carries the real
+# tools in a nested ``tools`` array; Codex ships its MCP servers that way.
+_NAMESPACE_TOOL_TYPE = "namespace"
# Substrings of the ``anthropic-beta`` tokens that gate tool search:
# ``advanced-tool-use-2025-11-20`` (firstParty/foundry) and
# ``tool-search-tool-2025-10-19`` (vertex/bedrock/mantle/gateway).
@@ -3507,15 +3527,72 @@ def _client_tool_search_names() -> frozenset[str]:
}
+def iter_tool_entries(tools: Any) -> Iterator[dict[str, Any]]:
+ """Yield every tool dict in ``tools``, descending into namespace groups.
+
+ A Responses client may group tools under
+ ``{"type": "namespace", "name": ..., "tools": [...]}`` and Codex ships its
+ MCP servers that way. A scan that reads only top-level ``tools[].name``
+ sees the wrapper and none of the tools inside it, so every nested tool is
+ invisible to both deferral detection and the resident/defer decision --
+ and a namespace is exactly where a large MCP catalog lives.
+
+ One level is what the API defines, and that is all this descends; a nested
+ ``tools`` value that is not a list is skipped rather than trusted.
+ """
+
+ if not isinstance(tools, list):
+ return
+ for tool in tools:
+ if not isinstance(tool, dict):
+ continue
+ yield tool
+ if tool.get("type") != _NAMESPACE_TOOL_TYPE:
+ continue
+ for nested in tool.get("tools") or []:
+ if isinstance(nested, dict):
+ yield nested
+
+
+def _is_tool_search_meta_tool(tool: dict[str, Any]) -> bool:
+ """Whether one tool entry is a tool-search meta-tool, in any known spelling."""
+
+ ttype = str(tool.get("type", ""))
+ if ttype.startswith(_TOOL_SEARCH_TOOL_TYPE_PREFIX) or ttype in _TOOL_SEARCH_META_TOOL_NAMES:
+ return True
+ name = _tool_search_resident_key(tool.get("name"))
+ return (
+ name.startswith(_TOOL_SEARCH_TOOL_TYPE_PREFIX)
+ or name in _TOOL_SEARCH_META_TOOL_NAMES
+ or name in _client_tool_search_names()
+ )
+
+
def request_already_defers_tools(tools: Any) -> bool:
"""Whether ``tools`` shows the CLIENT is already deferring tool schemas.
- True for the Messages API server-side shape (``tool_search_tool_*`` as
- either ``type`` or ``name``), which only appears when deferral is actually
- on. NOT true for a bare client-side tool name such as ``ToolSearch``, which
- a client may send whether or not it is deferring — see
+ Keyed on wire shape, never on a client allowlist, so it generalizes to
+ harnesses we have not seen. Three independent signals, any of which is
+ conclusive:
+
+ * a tool-search meta-tool in any known spelling -- Anthropic's versioned
+ ``tool_search_tool_*``, the Responses/Codex bare ``tool_search`` type
+ (which carries no ``name`` at all), Copilot's ``tool_search_tool``;
+ * ``defer_loading`` already set on any tool, which is the client marking
+ its own catalog deferred and is shape-independent;
+ * either of the above on a tool nested inside a ``namespace`` entry.
+
+ NOT true for a bare client-side tool name such as ``ToolSearch``, which a
+ client may send whether or not it is deferring -- see
``_CLIENT_TOOL_SEARCH_NAMES``.
+ Deferring on top of a client that already defers suppresses ITS mechanism
+ and inlines the catalog we were trying to keep out, so a miss here costs
+ real tokens. That is no longer a rare case: Codex, GitHub Copilot CLI, VS
+ Code Copilot, Kiro and Codebuff all ship their own tool search, and on
+ Copilot's Anthropic path the provider's server-side search can be a third
+ mechanism in the same request.
+
Upstream-independent on purpose: the question is what the client is doing,
not where the request is forwarded, so the same answer holds for first-party
Anthropic, Bedrock, Vertex and gateways. That matters because through Kong
@@ -3523,19 +3600,65 @@ def request_already_defers_tools(tools: Any) -> bool:
deferral is the only one available.
"""
- if not isinstance(tools, list):
- return False
- for tool in tools:
- if not isinstance(tool, dict):
- continue
- if str(tool.get("type", "")).startswith(_TOOL_SEARCH_TOOL_TYPE_PREFIX):
- return True
- name = _tool_search_resident_key(tool.get("name"))
- if name.startswith(_TOOL_SEARCH_TOOL_TYPE_PREFIX) or name in _client_tool_search_names():
+ deferred_without_search_tool = False
+ for tool in iter_tool_entries(tools):
+ if _is_tool_search_meta_tool(tool):
return True
+ if tool.get("defer_loading") is True:
+ deferred_without_search_tool = True
+ if deferred_without_search_tool:
+ # Tools marked deferred with no meta-tool we recognize that could
+ # resolve them. Two very different causes, and the operator wants to
+ # know which: a harness whose search tool is spelled in a way this
+ # survey missed (benign -- standing down is right, and the log names the
+ # tools so the spelling can be added), or an intermediary that stripped
+ # the search tool and left the marks behind, which upstream will reject.
+ # Headroom's own third-party strip clears both halves precisely so it
+ # cannot be the cause, so seeing this points outside us.
+ _warn_deferred_without_search_tool(tools)
+ return True
return False
+_DEFERRED_ORPHAN_WARN_INTERVAL_S = 3600.0
+_deferred_orphan_lock = threading.Lock()
+_deferred_orphan_last: float | None = None
+
+
+def reset_deferred_orphan_warn_state() -> None:
+ """Re-arm the orphaned-deferral warning. For tests."""
+ global _deferred_orphan_last
+ with _deferred_orphan_lock:
+ _deferred_orphan_last = None
+
+
+def _warn_deferred_without_search_tool(tools: Any) -> None:
+ """Warn, at most hourly, that deferred tools have nothing to resolve them."""
+
+ global _deferred_orphan_last
+ now = time.monotonic()
+ with _deferred_orphan_lock:
+ last = _deferred_orphan_last
+ if last is not None and (now - last) < _DEFERRED_ORPHAN_WARN_INTERVAL_S:
+ return
+ _deferred_orphan_last = now
+ names = [
+ str(t.get("name") or t.get("type") or "?")
+ for t in iter_tool_entries(tools)
+ if t.get("defer_loading") is True
+ ]
+ logger.warning(
+ "event=tool_search_deferred_orphan deferred=%d names=%s hint=%s",
+ len(names),
+ ",".join(sorted(names)[:8]),
+ "tools are marked defer_loading but no recognized tool-search tool can "
+ "resolve them; Headroom is standing down. Either this harness spells its "
+ "search tool in a way Headroom does not know (set "
+ "HEADROOM_CLIENT_TOOL_SEARCH_NAMES to teach it) or an intermediary "
+ "stripped the search tool and left the marks, which the upstream will reject",
+ )
+
+
def anthropic_first_party_tool_search_supported(api_base_url: str | None) -> bool:
"""Return whether Anthropic server-side tool search is valid for this upstream."""
from headroom.providers.claude.runtime import is_custom_anthropic_base_url
@@ -3967,7 +4090,7 @@ def strip_unsupported_ccr_retrieve_blocks(messages: Any, tools: Any) -> tuple[An
# * No ``cache_control`` (OpenAI caches automatically), so no breakpoint move.
# ---------------------------------------------------------------------------
-_OPENAI_TOOL_SEARCH_TYPE = "tool_search"
+_OPENAI_TOOL_SEARCH_TYPE = _RESPONSES_TOOL_SEARCH_TYPE
_OPENAI_TOOL_SEARCH_MIN_TOOLS = 12
_OPENAI_TOOL_SEARCH_RESIDENT_NAMES = frozenset({"terminal"})
_OPENAI_TOOL_SEARCH_UNSUPPORTED_CLIENTS = frozenset({"codex", "opencode"})
@@ -4034,16 +4157,19 @@ def inject_tool_search_deferral_openai(
return tools
if not _model_supports_openai_tool_search(model):
return tools
- if not isinstance(tools, list) or len(tools) < _OPENAI_TOOL_SEARCH_MIN_TOOLS:
+ if not isinstance(tools, list):
return tools
- if any(
- isinstance(tool, dict) and tool.get("type") == _OPENAI_TOOL_SEARCH_TYPE for tool in tools
- ) or request_already_defers_tools(tools):
- # Client already defers — its own Responses tool_search, or a
- # client-side tool such as Claude Code's ToolSearch. Deferring on top of
+ # Count nested namespace members too. A client that groups a 30-tool MCP
+ # catalog under one namespace entry presents a handful of top-level entries,
+ # so a top-level-only count reads it as a small tool surface and skips
+ # exactly the request with the most schema to save.
+ if sum(1 for _ in iter_tool_entries(tools)) < _OPENAI_TOOL_SEARCH_MIN_TOOLS:
+ return tools
+ if request_already_defers_tools(tools):
+ # Client already defers — its own Responses tool_search, Copilot's
+ # tool_search_tool, or defer_loading it set itself. Deferring on top of
# a client that is already deferring suppresses ITS mechanism and
- # inlines the catalog we were trying to keep out. Shape-based, not a
- # client allowlist, so it generalizes to harnesses we have not seen.
+ # inlines the catalog we were trying to keep out.
return tools
out: list[Any] = [{"type": _OPENAI_TOOL_SEARCH_TYPE}]
diff --git a/tests/test_tool_search_client_deferral.py b/tests/test_tool_search_client_deferral.py
index 9592cef20..80b048cd3 100644
--- a/tests/test_tool_search_client_deferral.py
+++ b/tests/test_tool_search_client_deferral.py
@@ -5,7 +5,13 @@ Two mechanisms exist at two layers:
* **Server-side (Messages API / Responses).** ``tool_search_tool_regex`` /
``tool_search_tool_bm25`` types paired with ``defer_loading: true``, or
OpenAI's ``{"type": "tool_search"}``. This shape appears ONLY when deferral
- is actually on, so it is an unambiguous signal.
+ is actually on, so it is an unambiguous signal. It has no single spelling:
+ a survey of shipped harnesses found tool search of their own in Codex
+ (``tool_search``, a type with no ``name`` at all), GitHub Copilot CLI
+ (``tool_search_tool``), VS Code Copilot and Kiro (``tool_search``) and
+ Codebuff (``composio_search_tools``) — so the client already deferring is
+ the common case, not the exception, and ``defer_loading`` on any tool is
+ the one signal that needs no agreement about names.
* **Client-side.** Claude Code carries a tool named ``ToolSearch``. It resolves
tools the client keeps in a local registry (TaskCreate, WebFetch, ...) and
real traffic shows it riding alongside a fully inline MCP catalog. Its
@@ -25,6 +31,7 @@ So: key on the server-side shape, never on the bare client-side name.
from __future__ import annotations
+import logging
from typing import Any
import pytest
@@ -34,7 +41,9 @@ from headroom.proxy.helpers import (
claude_code_tool_search_inactive,
inject_tool_search_deferral,
inject_tool_search_deferral_openai,
+ iter_tool_entries,
request_already_defers_tools,
+ reset_deferred_orphan_warn_state,
resolved_core_tools,
strip_first_party_tool_search_tools_for_third_party_upstream,
)
@@ -110,7 +119,6 @@ def test_server_side_shape_means_the_client_defers(marker: dict[str, Any]) -> No
pytest.param(None, id="not-a-list"),
pytest.param([{"name": "mcp__search__tools"}], id="mcp-tool-named-search"),
pytest.param([{"name": "SearchTool"}], id="reversed-words"),
- pytest.param([{"name": "tool_search"}], id="prefix-but-incomplete"),
pytest.param([{"type": "web_search"}], id="unrelated-server-tool"),
pytest.param(["not-a-dict", 7], id="non-dict-entries"),
],
@@ -119,6 +127,112 @@ def test_not_a_deferral_signal(tools: Any) -> None:
assert request_already_defers_tools(tools) is False
+# ---------------------------------------------------------------------------
+# Deferral is not an Anthropic-only feature. A survey of shipped harnesses
+# found their own tool search in Codex, GitHub Copilot CLI, VS Code Copilot,
+# Kiro and Codebuff, each with a different wire spelling. Matching only
+# Anthropic's versioned ``tool_search_tool_*`` prefix stood down for none of
+# them, so Headroom deferred on top of a client that was already deferring --
+# which suppresses the client's mechanism and inlines the very catalog we were
+# keeping out.
+# ---------------------------------------------------------------------------
+
+
+ "marker",
+ [
+ # Codex: codex-rs/tools/src/tool_spec.rs. A union variant carrying no
+ # ``name`` field at all, so a name-only scan can never see it.
+ pytest.param({"type": "tool_search"}, id="codex-responses-bare-type"),
+ # VS Code Copilot: ToolName.tool_search; Kiro CLI uses the same name.
+ pytest.param({"name": "tool_search"}, id="vscode-copilot-kiro-name"),
+ # GitHub Copilot CLI: the exact string ``tool_search_tool``, which does
+ # NOT start with ``tool_search_tool_`` -- the off-by-one that let the
+ # prefix test miss a shipped harness.
+ pytest.param({"name": "tool_search_tool"}, id="copilot-cli-exact"),
+ pytest.param({"name": "tool_search_tool_regex"}, id="copilot-cli-regex"),
+ # Codebuff delegates to an external catalog under its own name.
+ pytest.param({"name": "composio_search_tools"}, id="codebuff"),
+ ],
+)
+def test_every_known_meta_tool_spelling_means_the_client_defers(marker: dict[str, Any]) -> None:
+ assert request_already_defers_tools([marker, *_mcp(3)]) is True
+
+
+def test_defer_loading_alone_means_the_client_defers() -> None:
+ """Shape-independent: the client marking its own catalog deferred is enough.
+
+ Worth matching on its own because it needs no agreement about what the
+ meta-tool is called, so it still holds for a harness this survey missed.
+ """
+ tools = [{"name": "mcp__github__list_issues", "defer_loading": True}, *_mcp(3)]
+
+ assert request_already_defers_tools(tools) is True
+
+
+def test_defer_loading_false_is_not_a_signal() -> None:
+ """Present-and-false is a client that considered deferral and declined."""
+ assert request_already_defers_tools([{"name": "read", "defer_loading": False}]) is False
+
+
+# --- namespace nesting -----------------------------------------------------
+
+
+def _namespace(name: str, *tools: dict[str, Any]) -> dict[str, Any]:
+ """A Responses namespace group, the shape Codex ships MCP servers in."""
+ return {"type": "namespace", "name": name, "tools": list(tools)}
+
+
+def test_a_meta_tool_nested_in_a_namespace_is_found() -> None:
+ """The wrapper is visible to a top-level scan; its contents are not."""
+ tools = [_namespace("functions", {"type": "function", "name": "tool_search_tool"})]
+
+ assert request_already_defers_tools(tools) is True
+
+
+def test_defer_loading_nested_in_a_namespace_is_found() -> None:
+ tools = [_namespace("github", {"type": "function", "name": "x", "defer_loading": True})]
+
+ assert request_already_defers_tools(tools) is True
+
+
+def test_a_plain_namespace_is_not_a_deferral_signal() -> None:
+ """Grouping tools is not deferring them."""
+ tools = [_namespace("github", {"type": "function", "name": "list_issues"})]
+
+ assert request_already_defers_tools(tools) is False
+
+
+def test_iter_tool_entries_yields_the_wrapper_and_its_members() -> None:
+ nested = {"type": "function", "name": "inner"}
+ group = _namespace("ns", nested)
+
+ names = [t.get("name") for t in iter_tool_entries([{"name": "top"}, group])]
+
+ assert names == ["top", "ns", "inner"]
+
+
+ "tools",
+ [
+ pytest.param(None, id="not-a-list"),
+ pytest.param([{"type": "namespace"}], id="namespace-without-tools"),
+ pytest.param([{"type": "namespace", "tools": "nope"}], id="tools-not-a-list"),
+ pytest.param([{"type": "namespace", "tools": [7, "x"]}], id="non-dict-members"),
+ ],
+)
+def test_iter_tool_entries_tolerates_malformed_input(tools: Any) -> None:
+ """A malformed namespace must not raise on the request path."""
+ assert all(isinstance(t, dict) for t in iter_tool_entries(tools))
+
+
+def test_nesting_does_not_recurse_past_one_level() -> None:
+ """One level is what the API defines; deeper input is not trusted."""
+ deep = _namespace("outer", _namespace("inner", {"name": "buried"}))
+
+ assert "buried" not in [t.get("name") for t in iter_tool_entries([deep])]
+
+
def test_extra_names_can_be_configured(monkeypatch: pytest.MonkeyPatch) -> None:
"""Escape hatch for a harness that does signal deferral by name."""
tools = [{"name": "MyHarnessSearch"}, *_mcp(3)]
@@ -327,6 +441,41 @@ def test_openai_still_defers_for_an_eager_client() -> None:
assert len(_deferred(out)) == 14
+def test_openai_counts_namespace_members_toward_the_threshold() -> None:
+ """A big catalog grouped under one namespace is still a big catalog.
+
+ Counting only top-level entries reads a 20-tool MCP server as a single
+ tool and skips the request, which is precisely the request with the most
+ schema to save.
+ """
+ top_level = [_fn(f"jira_{i}") for i in range(4)]
+ tools = [
+ *top_level,
+ {"type": "namespace", "name": "slack", "tools": [_fn(f"slack_{i}") for i in range(20)]},
+ ]
+
+ assert len(tools) < _TOOL_SEARCH_MIN_TOOLS # a top-level count would bail here
+ out = inject_tool_search_deferral_openai(tools, "gpt-5.5")
+
+ # The namespace itself stays untouched; the top-level tools it made us
+ # look at in the first place are what get deferred.
+ assert [t["name"] for t in _deferred(out)] == [t["name"] for t in top_level]
+
+
+def test_openai_leaves_namespace_groups_resident() -> None:
+ """We do not guess a wire shape for deferring inside a namespace.
+
+ Marking nested members ``defer_loading`` is unverified against the API, and
+ an unknown field in the wrong place is a 400. Counting them is safe;
+ rewriting them is not, so the group is forwarded untouched.
+ """
+ group = {"type": "namespace", "name": "slack", "tools": [_fn(f"slack_{i}") for i in range(20)]}
+ out = inject_tool_search_deferral_openai([_fn("bash"), group], "gpt-5.5")
+
+ assert group in out
+ assert _deferred(out) == []
+
+
def test_openai_toolsearch_name_alone_does_not_stand_us_down() -> None:
tools = [_fn("ToolSearch"), *[_fn(f"slack_{i}") for i in range(14)]]
@@ -505,3 +654,52 @@ def test_the_scan_gate_closes_even_when_the_hint_does_not_fire(
assert H.tool_search_hint_pending() is False # ...and the gate closed anyway
finally:
H.reset_tool_search_hint_state()
+
+
+# --- orphaned deferrals ----------------------------------------------------
+
+
+def test_deferred_tools_with_no_search_tool_warn(caplog: pytest.LogCaptureFixture) -> None:
+ """Standing down here is right but ambiguous, so it must not be silent.
+
+ Either the harness spells its search tool in a way we do not know, or an
+ intermediary stripped the search tool and left the marks behind. The first
+ is benign and the second is fatal upstream, and only the operator can tell
+ them apart -- so name the tools and say what to do.
+ """
+ reset_deferred_orphan_warn_state()
+ tools = [{"name": "mcp__github__list_issues", "defer_loading": True}, *_mcp(3)]
+
+ with caplog.at_level(logging.WARNING, logger="headroom.proxy"):
+ assert request_already_defers_tools(tools) is True
+
+ lines = [
+ r.getMessage() for r in caplog.records if "tool_search_deferred_orphan" in r.getMessage()
+ ]
+ assert len(lines) == 1
+ assert "mcp__github__list_issues" in lines[0]
+ assert "HEADROOM_CLIENT_TOOL_SEARCH_NAMES" in lines[0]
+
+
+def test_a_recognized_search_tool_is_not_an_orphan(caplog: pytest.LogCaptureFixture) -> None:
+ """The normal deferring client must stay quiet."""
+ reset_deferred_orphan_warn_state()
+ tools = [ANTHROPIC_SEARCH_REGEX, {"name": "mcp__github__x", "defer_loading": True}]
+
+ with caplog.at_level(logging.WARNING, logger="headroom.proxy"):
+ assert request_already_defers_tools(tools) is True
+
+ assert [r for r in caplog.records if "tool_search_deferred_orphan" in r.getMessage()] == []
+
+
+def test_the_orphan_warning_is_throttled(caplog: pytest.LogCaptureFixture) -> None:
+ """It is evaluated per request; one line per request would be a flood."""
+ reset_deferred_orphan_warn_state()
+ tools = [{"name": "x", "defer_loading": True}]
+
+ with caplog.at_level(logging.WARNING, logger="headroom.proxy"):
+ for _ in range(50):
+ request_already_defers_tools(tools)
+
+ lines = [r for r in caplog.records if "tool_search_deferred_orphan" in r.getMessage()]
+ assert len(lines) == 1