Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): skip memory-context search in cache mode on OpenAI routes (#4632)
AKSHITH BHARADWAJ D L T365f6fe5233c
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 4d0d30700..14eec5c01 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -4372,7 +4372,24 @@ class OpenAIHandlerMixin:# /v1/responses) — pre-PR-this site was the only chat# path without one.try:- if self.memory_handler.config.inject_context:+ if self.memory_handler.config.inject_context and is_cache_mode(self.config.mode):+ # Cache mode never injects context, so the next-turn prefix+ # bytes stay byte-equal to this turn's bytes. Skip the search+ # too: its result could only be discarded (#4600, #4622).+ from headroom.proxy.helpers import log_memory_injection+ from headroom.utils import extract_user_query++ if tags is not None:+ tags["memory_skip_reason"] = "cache_mode"+ log_memory_injection(+ request_id=request_id,+ session_id=None,+ decision="skipped_cache_mode",+ bytes_injected=0,+ query=extract_user_query(optimized_messages) or "",+ tags=tags,+ )+ elif self.memory_handler.config.inject_context:memory_context = await asyncio.wait_for(self.memory_handler.search_and_format_context(memory_user_id,@@ -6211,7 +6228,24 @@ class OpenAIHandlerMixin:# (latest non-frozen user item). Instructions are part of the# cache hot zone and must never be mutated — invariant I2.# See REALIGNMENT/03-phase-A-lockdown.md PR-A2.- if self.memory_handler.config.inject_context:+ if self.memory_handler.config.inject_context and is_cache_mode(self.config.mode):+ # Cache mode never injects context, so the next-turn prefix+ # bytes stay byte-equal to this turn's bytes. Skip the search+ # too: its result could only be discarded (#4600, #4622).+ from headroom.proxy.helpers import log_memory_injection+ from headroom.utils import extract_user_query++ if tags is not None:+ tags["memory_skip_reason"] = "cache_mode"+ log_memory_injection(+ request_id=request_id,+ session_id=None,+ decision="skipped_cache_mode",+ bytes_injected=0,+ query=extract_user_query(optimized_messages) or "",+ tags=tags,+ )+ elif self.memory_handler.config.inject_context:try:memory_context = await asyncio.wait_for(self.memory_handler.search_and_format_context(@@ -8276,7 +8310,24 @@ class OpenAIHandlerMixin:)# Inject memory context into instructions- if self.memory_handler.config.inject_context:+ if self.memory_handler.config.inject_context and is_cache_mode(+ self.config.mode+ ):+ # Cache mode never injects context, so the next-turn prefix+ # bytes stay byte-equal to this turn's bytes. Skip the search+ # too: its result could only be discarded (#4600, #4622).+ if ws_tags is not None:+ ws_tags["memory_skip_reason"] = "cache_mode"+ ws_input = ws_response_body.get("input", "")+ log_memory_injection(+ request_id=request_id,+ session_id=session_id,+ decision="skipped_cache_mode",+ bytes_injected=0,+ query=ws_input if isinstance(ws_input, str) else "",+ tags=ws_tags,+ )+ elif self.memory_handler.config.inject_context:ws_input = ws_response_body.get("input", "")ws_instructions = ws_response_body.get("instructions")ws_msgs: list[dict[str, Any]] = []diff --git a/tests/test_codex_ws_per_frame_memory.py b/tests/test_codex_ws_per_frame_memory.pyindex 2c385092c..a705e96f4 100644--- a/tests/test_codex_ws_per_frame_memory.py+++ b/tests/test_codex_ws_per_frame_memory.py@@ -196,6 +196,50 @@ async def test_memory_lookup_runs_for_each_issue_artifact_frame_and_preserves_noassert forwarded_turns[0]["response"]["tools"] == forwarded_turns[1]["response"]["tools"]+async def test_cache_mode_skips_context_on_each_frame_and_preserves_tools():+ upstream = _FakeUpstream(+ [+ json.dumps({"type": "response.created", "response": {"id": "r_1"}}),+ json.dumps({"type": "response.completed", "response": {"id": "r_1"}}),+ ],+ hold_after_events=True,+ )+ first, later = _issue_2059_inputs()+ cancel = json.dumps({"type": "response.cancel"})+ frames = [+ _turn_with_tools(first, _client_response_tools()),+ cancel,+ _turn_with_tools(later, _client_response_tools()),+ ]+ client_ws = _FakeWebSocket(frames=frames)+ handler = _DummyOpenAIHandler()+ handler.config.mode = "cache"+ memory = _MemoryHandler()+ handler.memory_handler = memory++ with (+ patch.dict(sys.modules, {"websockets": _make_fake_websockets_module(upstream)}),+ patch("headroom.proxy.helpers.log_memory_injection") as log_injection,+ ):+ await handler.handle_openai_responses_ws(client_ws)++ assert memory.queries == []+ assert upstream.sent[1] == cancel+ forwarded = [json.loads(upstream.sent[i])["response"] for i in (0, 2)]+ assert [response["input"] for response in forwarded] == [first, later]+ expected_tools = [*_client_response_tools(), *_expected_memory_response_tools()]+ assert all(response["tools"] == expected_tools for response in forwarded)+ skips = [+ call.kwargs+ for call in log_injection.call_args_list+ if call.kwargs.get("decision") == "skipped_cache_mode"+ ]+ assert len(skips) == 2+ assert all(call["bytes_injected"] == 0 for call in skips)+ assert all(call["tags"]["memory_skip_reason"] == "cache_mode" for call in skips)++@pytest.mark.asyncio@pytest.mark.parametrize("tools", [None, []], ids=["omitted", "empty"])async def test_ws_responses_do_not_inject_memory_tools_without_client_tools(tools):diff --git a/tests/test_openai_codex_ws_lifecycle.py b/tests/test_openai_codex_ws_lifecycle.pyindex f468183ed..39ee352e6 100644--- a/tests/test_openai_codex_ws_lifecycle.py+++ b/tests/test_openai_codex_ws_lifecycle.py@@ -129,6 +129,7 @@ class _DummyOpenAIHandler(OpenAIHandlerMixin):self.metrics = _DummyMetrics()self.config = SimpleNamespace(optimize=False,+ mode="token",retry_max_attempts=1,retry_base_delay_ms=1,retry_max_delay_ms=1,diff --git a/tests/test_proxy_system_prompt_immutable.py b/tests/test_proxy_system_prompt_immutable.pyindex cb619018b..bb5a435f2 100644--- a/tests/test_proxy_system_prompt_immutable.py+++ b/tests/test_proxy_system_prompt_immutable.py@@ -21,6 +21,11 @@ from fastapi.testclient import TestClientfrom headroom.proxy.server import ProxyConfig, create_app[email protected](autouse=True)+def _stub_env(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("HEADROOM_REQUIRE_RUST_CORE", "false")++class _FakePrefixTracker:def __init__(self, frozen_count: int):self._frozen_count = frozen_count@@ -474,3 +479,107 @@ def test_memory_context_appears_in_responses_api_input_tail() -> None:assert any("MEMCTX" in t for t in text_parts)elif isinstance(input_field, str):assert input_field.endswith("MEMCTX")+++def test_cache_mode_skips_memory_context_search_openai_chat() -> None:+ """Cache mode never injects memory context on OpenAI chat, so it must not search for it."""+ captured: dict[str, object] = {}+ with _make_proxy_client() as client:+ proxy = client.app.state.proxy+ proxy.config.mode = "cache"+ proxy.config.optimize = False+ proxy.config.image_optimize = False+ proxy.config.ccr_proactive_expansion = False++ _install_session_tracker(proxy, frozen_count=1)+ _install_memory_handler(proxy)++ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001+ captured["body"] = body+ return httpx.Response(+ 200,+ json={+ "id": "chatcmpl_1",+ "object": "chat.completion",+ "choices": [+ {+ "index": 0,+ "message": {"role": "assistant", "content": "ack"},+ "finish_reason": "stop",+ }+ ],+ "usage": {"prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12},+ },+ )++ proxy._retry_request = _fake_retry++ response = client.post(+ "/v1/chat/completions",+ headers={+ "authorization": "Bearer sk-test",+ "x-headroom-user-id": "u1",+ },+ json={+ "model": "gpt-4o",+ "messages": [+ {"role": "system", "content": "base system"},+ {"role": "user", "content": "frozen prefix"},+ {"role": "assistant", "content": "ack"},+ {"role": "user", "content": "latest user"},+ ],+ },+ )++ assert response.status_code == 200+ proxy.memory_handler.search_and_format_context.assert_not_awaited()+ sent = captured["body"]+ assert isinstance(sent, dict)+ assert sent["messages"][3]["content"] == "latest user"+++def test_cache_mode_skips_memory_context_search_openai_responses() -> None:+ """Cache mode never injects memory context on OpenAI responses, so it must not search for it."""+ captured: dict[str, object] = {}+ with _make_responses_proxy_client() as client:+ proxy = client.app.state.proxy+ proxy.config.mode = "cache"+ proxy.config.optimize = False+ proxy.config.image_optimize = False+ _install_memory_handler(proxy)++ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001+ captured["body"] = body+ return httpx.Response(+ 200,+ json={+ "id": "resp_1",+ "object": "response",+ "model": "gpt-5.4",+ "output": [],+ "usage": {"input_tokens": 5, "output_tokens": 1, "total_tokens": 6},+ },+ )++ proxy._retry_request = _fake_retry++ response = client.post(+ "/v1/responses",+ headers={+ "authorization": "Bearer sk-test",+ "x-headroom-user-id": "u1",+ },+ json={+ "model": "gpt-5.4",+ "instructions": "You are a Codex assistant.",+ "input": "latest user string input",+ },+ )++ assert response.status_code == 200+ proxy.memory_handler.search_and_format_context.assert_not_awaited()+ assert "body" in captured+ sent = captured["body"]+ assert isinstance(sent, dict)+ assert sent.get("input") == "latest user string input"+ assert sent.get("instructions") == "You are a Codex assistant."