Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): skip memory tool injection on streamed OpenAI requests (#4451)

danieldutu5f308e1cc8b4
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 00499e80c..6607d97fe 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -4392,23 +4392,27 @@ class OpenAIHandlerMixin:
apply_session_sticky_memory_tools as _apply_sticky_mem_tools,
)
- memory_tool_defs = (
- self.memory_handler.compute_memory_tool_definitions("openai")
- if self.memory_handler.config.inject_tools
- else []
- )
- tools, mem_tools_injected = _apply_sticky_mem_tools(
- provider="openai",
- session_id=openai_session_id,
- request_id=request_id,
- existing_tools=tools,
- memory_tools_to_inject=memory_tool_defs,
- inject_this_turn=bool(self.memory_handler.config.inject_tools),
- client_declared_tools=bool(_original_tools),
- )
- if mem_tools_injected:
- memory_tools_injected = True
- logger.info(f"[{request_id}] Memory: Injected memory tools (openai)")
+ # Only the Anthropic stream path executes memory tool calls
+ # (streaming.py `memory_enabled`). Skip the whole call on a
+ # streamed turn: the sticky replay would re-add golden tools.
+ if not stream:
+ memory_tool_defs = (
+ self.memory_handler.compute_memory_tool_definitions("openai")
+ if self.memory_handler.config.inject_tools
+ else []
+ )
+ tools, mem_tools_injected = _apply_sticky_mem_tools(
+ provider="openai",
+ session_id=openai_session_id,
+ request_id=request_id,
+ existing_tools=tools,
+ memory_tools_to_inject=memory_tool_defs,
+ inject_this_turn=bool(self.memory_handler.config.inject_tools),
+ client_declared_tools=bool(_original_tools),
+ )
+ if mem_tools_injected:
+ memory_tools_injected = True
+ logger.info(f"[{request_id}] Memory: Injected memory tools (openai)")
except Exception as e:
logger.warning(f"[{request_id}] Memory injection failed: {e}")
@@ -6019,7 +6023,10 @@ class OpenAIHandlerMixin:
) or _client_can_receive_memory_tools(memory_client)
if _ensure_chatgpt_responses_store_false(body, is_chatgpt_auth=is_chatgpt_auth):
logger.info(f"[{request_id}] Responses: forced store=false for ChatGPT auth")
- responses_memory_tools_allowed = _allow_responses_memory_tools(is_chatgpt_auth)
+ # Streamed Responses turns do not execute memory tool calls; keep tools off.
+ responses_memory_tools_allowed = (
+ _allow_responses_memory_tools(is_chatgpt_auth) and not stream
+ )
# PR-A6 (P5-50, preps P0-6): session-sticky `OpenAI-Beta` merge
# for /v1/responses. Compute a session_id off the same store the
diff --git a/tests/test_memory_tool_stream.py b/tests/test_memory_tool_stream.py
index d2e6e45a4..93af46805 100644
--- a/tests/test_memory_tool_stream.py
+++ b/tests/test_memory_tool_stream.py
@@ -801,7 +801,7 @@ class TestHandlerPassesServerMemoryTools:
captured: dict[str, Any] = {}
async def fake_stream_response(*args: Any, **kwargs: Any) -> StreamingResponse:
- captured.update(kwargs)
+ captured.update(kwargs, body=args[2])
async def gen() -> Any:
yield b""
@@ -838,6 +838,85 @@ class TestHandlerPassesServerMemoryTools:
)
assert captured["server_memory_tool_names"] == MEMORY_TOOLS
+ assert MEMORY_TOOLS <= {t["name"] for t in captured["body"]["tools"]}
+
+ @pytest.mark.parametrize("after_buffered_turn", [False, True])
+ def test_streamed_openai_chat_gets_no_memory_tools(self, after_buffered_turn: bool) -> None:
+ from types import SimpleNamespace
+
+ from fastapi.responses import StreamingResponse
+ from fastapi.testclient import TestClient
+
+ from headroom.proxy.server import ProxyConfig, create_app
+
+ config = ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ ccr_inject_tool=False,
+ ccr_handle_responses=False,
+ ccr_context_tracking=False,
+ image_optimize=False,
+ )
+ captured: dict[str, Any] = {}
+
+ async def fake_stream_response(url: str, headers: Any, body: Any, *a: Any, **kw: Any):
+ captured["body"] = body
+
+ async def gen() -> Any:
+ yield b""
+
+ return StreamingResponse(gen(), media_type="text/event-stream")
+
+ memory_tools = [
+ {
+ "type": "function",
+ "function": {"name": name, "description": name, "parameters": {}},
+ }
+ for name in sorted(MEMORY_TOOLS)
+ ]
+ from headroom.proxy.helpers import (
+ _reset_session_tool_tracker_for_test,
+ apply_session_sticky_memory_tools,
+ )
+
+ _reset_session_tool_tracker_for_test()
+ with TestClient(create_app(config)) as client:
+ proxy = client.app.state.proxy
+ proxy.memory_handler = SimpleNamespace(
+ config=SimpleNamespace(inject_context=False, inject_tools=True),
+ compute_memory_tool_definitions=lambda provider: memory_tools,
+ get_beta_headers=lambda: {},
+ has_memory_tool_calls=lambda resp, provider: False,
+ )
+ proxy._stream_response = fake_stream_response
+ proxy.session_tracker_store.compute_session_id = lambda *a, **k: "sess-mixed"
+ if after_buffered_turn:
+ # A buffered turn records the golden memory tools for the session.
+ apply_session_sticky_memory_tools(
+ provider="openai",
+ session_id="sess-mixed",
+ request_id="r0",
+ existing_tools=[],
+ memory_tools_to_inject=memory_tools,
+ inject_this_turn=True,
+ )
+ client.post(
+ "/v1/chat/completions",
+ headers={"authorization": "Bearer sk-test", "x-headroom-user-id": "u1"},
+ json={
+ "model": "gpt-4o-mini",
+ "stream": True,
+ "tools": [{"type": "function", "function": {"name": "Bash", "parameters": {}}}],
+ "messages": [{"role": "user", "content": "remember the deploy region"}],
+ },
+ )
+
+ _reset_session_tool_tracker_for_test()
+ names = {t["function"]["name"] for t in captured["body"]["tools"]}
+ assert names == {"Bash"}
class TestStreamingRetainedByteLimit:
diff --git a/tests/test_openai_codex_routing.py b/tests/test_openai_codex_routing.py
index 838891dd6..a83720a34 100644
--- a/tests/test_openai_codex_routing.py
+++ b/tests/test_openai_codex_routing.py
@@ -581,6 +581,30 @@ def test_handle_openai_responses_api_auth_store_false_injects_stateless_memory_t
assert memory_handler.compute_calls == 1
+def test_handle_openai_responses_stream_skips_memory_tools(monkeypatch):
+ request = _build_request(
+ {"model": "gpt-4o-mini", "input": "hello", "store": False, "stream": True},
+ {
+ "Authorization": "Bearer sk-test",
+ "User-Agent": "codex-cli/0.5",
+ "x-headroom-user-id": "user-1",
+ },
+ )
+ handler = _DummyOpenAIHandler()
+ handler.memory_handler = _MemoryToolsOnlyHandler()
+ handler.session_tracker_store = SimpleNamespace(
+ compute_session_id=lambda *a, **k: "sess-api-memory-tools-stream",
+ )
+
+ monkeypatch.setattr("headroom.tokenizers.get_tokenizer", lambda model: _DummyTokenizer())
+
+ anyio.run(handler.handle_openai_responses, request)
+
+ assert handler.captured_stream_request is not None
+ assert "memory_search" not in json.dumps(handler.captured_stream_request[2].get("tools", []))
+ assert handler.memory_handler.compute_calls == 0
+
+
@pytest.mark.parametrize("store", [pytest.param(None, id="omitted"), True, False])
@pytest.mark.parametrize(
"include",