Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): skip memory tool injection on streamed OpenAI requests (#4451)
danieldutu5f308e1cc8b4
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 00499e80c..6607d97fe 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -4392,23 +4392,27 @@ class OpenAIHandlerMixin:apply_session_sticky_memory_tools as _apply_sticky_mem_tools,)- memory_tool_defs = (- self.memory_handler.compute_memory_tool_definitions("openai")- if self.memory_handler.config.inject_tools- else []- )- tools, mem_tools_injected = _apply_sticky_mem_tools(- provider="openai",- session_id=openai_session_id,- request_id=request_id,- existing_tools=tools,- memory_tools_to_inject=memory_tool_defs,- inject_this_turn=bool(self.memory_handler.config.inject_tools),- client_declared_tools=bool(_original_tools),- )- if mem_tools_injected:- memory_tools_injected = True- logger.info(f"[{request_id}] Memory: Injected memory tools (openai)")+ # Only the Anthropic stream path executes memory tool calls+ # (streaming.py `memory_enabled`). Skip the whole call on a+ # streamed turn: the sticky replay would re-add golden tools.+ if not stream:+ memory_tool_defs = (+ self.memory_handler.compute_memory_tool_definitions("openai")+ if self.memory_handler.config.inject_tools+ else []+ )+ tools, mem_tools_injected = _apply_sticky_mem_tools(+ provider="openai",+ session_id=openai_session_id,+ request_id=request_id,+ existing_tools=tools,+ memory_tools_to_inject=memory_tool_defs,+ inject_this_turn=bool(self.memory_handler.config.inject_tools),+ client_declared_tools=bool(_original_tools),+ )+ if mem_tools_injected:+ memory_tools_injected = True+ logger.info(f"[{request_id}] Memory: Injected memory tools (openai)")except Exception as e:logger.warning(f"[{request_id}] Memory injection failed: {e}")@@ -6019,7 +6023,10 @@ class OpenAIHandlerMixin:) or _client_can_receive_memory_tools(memory_client)if _ensure_chatgpt_responses_store_false(body, is_chatgpt_auth=is_chatgpt_auth):logger.info(f"[{request_id}] Responses: forced store=false for ChatGPT auth")- responses_memory_tools_allowed = _allow_responses_memory_tools(is_chatgpt_auth)+ # Streamed Responses turns do not execute memory tool calls; keep tools off.+ responses_memory_tools_allowed = (+ _allow_responses_memory_tools(is_chatgpt_auth) and not stream+ )# PR-A6 (P5-50, preps P0-6): session-sticky `OpenAI-Beta` merge# for /v1/responses. Compute a session_id off the same store thediff --git a/tests/test_memory_tool_stream.py b/tests/test_memory_tool_stream.pyindex d2e6e45a4..93af46805 100644--- a/tests/test_memory_tool_stream.py+++ b/tests/test_memory_tool_stream.py@@ -801,7 +801,7 @@ class TestHandlerPassesServerMemoryTools:captured: dict[str, Any] = {}async def fake_stream_response(*args: Any, **kwargs: Any) -> StreamingResponse:- captured.update(kwargs)+ captured.update(kwargs, body=args[2])async def gen() -> Any:yield b""@@ -838,6 +838,85 @@ class TestHandlerPassesServerMemoryTools:)assert captured["server_memory_tool_names"] == MEMORY_TOOLS+ assert MEMORY_TOOLS <= {t["name"] for t in captured["body"]["tools"]}++ @pytest.mark.parametrize("after_buffered_turn", [False, True])+ def test_streamed_openai_chat_gets_no_memory_tools(self, after_buffered_turn: bool) -> None:+ from types import SimpleNamespace++ from fastapi.responses import StreamingResponse+ from fastapi.testclient import TestClient++ from headroom.proxy.server import ProxyConfig, create_app++ config = ProxyConfig(+ optimize=False,+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ log_requests=False,+ ccr_inject_tool=False,+ ccr_handle_responses=False,+ ccr_context_tracking=False,+ image_optimize=False,+ )+ captured: dict[str, Any] = {}++ async def fake_stream_response(url: str, headers: Any, body: Any, *a: Any, **kw: Any):+ captured["body"] = body++ async def gen() -> Any:+ yield b""++ return StreamingResponse(gen(), media_type="text/event-stream")++ memory_tools = [+ {+ "type": "function",+ "function": {"name": name, "description": name, "parameters": {}},+ }+ for name in sorted(MEMORY_TOOLS)+ ]+ from headroom.proxy.helpers import (+ _reset_session_tool_tracker_for_test,+ apply_session_sticky_memory_tools,+ )++ _reset_session_tool_tracker_for_test()+ with TestClient(create_app(config)) as client:+ proxy = client.app.state.proxy+ proxy.memory_handler = SimpleNamespace(+ config=SimpleNamespace(inject_context=False, inject_tools=True),+ compute_memory_tool_definitions=lambda provider: memory_tools,+ get_beta_headers=lambda: {},+ has_memory_tool_calls=lambda resp, provider: False,+ )+ proxy._stream_response = fake_stream_response+ proxy.session_tracker_store.compute_session_id = lambda *a, **k: "sess-mixed"+ if after_buffered_turn:+ # A buffered turn records the golden memory tools for the session.+ apply_session_sticky_memory_tools(+ provider="openai",+ session_id="sess-mixed",+ request_id="r0",+ existing_tools=[],+ memory_tools_to_inject=memory_tools,+ inject_this_turn=True,+ )+ client.post(+ "/v1/chat/completions",+ headers={"authorization": "Bearer sk-test", "x-headroom-user-id": "u1"},+ json={+ "model": "gpt-4o-mini",+ "stream": True,+ "tools": [{"type": "function", "function": {"name": "Bash", "parameters": {}}}],+ "messages": [{"role": "user", "content": "remember the deploy region"}],+ },+ )++ _reset_session_tool_tracker_for_test()+ names = {t["function"]["name"] for t in captured["body"]["tools"]}+ assert names == {"Bash"}class TestStreamingRetainedByteLimit:diff --git a/tests/test_openai_codex_routing.py b/tests/test_openai_codex_routing.pyindex 838891dd6..a83720a34 100644--- a/tests/test_openai_codex_routing.py+++ b/tests/test_openai_codex_routing.py@@ -581,6 +581,30 @@ def test_handle_openai_responses_api_auth_store_false_injects_stateless_memory_tassert memory_handler.compute_calls == 1+def test_handle_openai_responses_stream_skips_memory_tools(monkeypatch):+ request = _build_request(+ {"model": "gpt-4o-mini", "input": "hello", "store": False, "stream": True},+ {+ "Authorization": "Bearer sk-test",+ "User-Agent": "codex-cli/0.5",+ "x-headroom-user-id": "user-1",+ },+ )+ handler = _DummyOpenAIHandler()+ handler.memory_handler = _MemoryToolsOnlyHandler()+ handler.session_tracker_store = SimpleNamespace(+ compute_session_id=lambda *a, **k: "sess-api-memory-tools-stream",+ )++ monkeypatch.setattr("headroom.tokenizers.get_tokenizer", lambda model: _DummyTokenizer())++ anyio.run(handler.handle_openai_responses, request)++ assert handler.captured_stream_request is not None+ assert "memory_search" not in json.dumps(handler.captured_stream_request[2].get("tools", []))+ assert handler.memory_handler.compute_calls == 0++@pytest.mark.parametrize("store", [pytest.param(None, id="omitted"), True, False])@pytest.mark.parametrize("include",