Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): skip memory-context search in cache mode on OpenAI routes (#4632)

AKSHITH BHARADWAJ D L T365f6fe5233c
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 4d0d30700..14eec5c01 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -4372,7 +4372,24 @@ class OpenAIHandlerMixin:
# /v1/responses) — pre-PR-this site was the only chat
# path without one.
try:
- if self.memory_handler.config.inject_context:
+ if self.memory_handler.config.inject_context and is_cache_mode(self.config.mode):
+ # Cache mode never injects context, so the next-turn prefix
+ # bytes stay byte-equal to this turn's bytes. Skip the search
+ # too: its result could only be discarded (#4600, #4622).
+ from headroom.proxy.helpers import log_memory_injection
+ from headroom.utils import extract_user_query
+
+ if tags is not None:
+ tags["memory_skip_reason"] = "cache_mode"
+ log_memory_injection(
+ request_id=request_id,
+ session_id=None,
+ decision="skipped_cache_mode",
+ bytes_injected=0,
+ query=extract_user_query(optimized_messages) or "",
+ tags=tags,
+ )
+ elif self.memory_handler.config.inject_context:
memory_context = await asyncio.wait_for(
self.memory_handler.search_and_format_context(
memory_user_id,
@@ -6211,7 +6228,24 @@ class OpenAIHandlerMixin:
# (latest non-frozen user item). Instructions are part of the
# cache hot zone and must never be mutated — invariant I2.
# See REALIGNMENT/03-phase-A-lockdown.md PR-A2.
- if self.memory_handler.config.inject_context:
+ if self.memory_handler.config.inject_context and is_cache_mode(self.config.mode):
+ # Cache mode never injects context, so the next-turn prefix
+ # bytes stay byte-equal to this turn's bytes. Skip the search
+ # too: its result could only be discarded (#4600, #4622).
+ from headroom.proxy.helpers import log_memory_injection
+ from headroom.utils import extract_user_query
+
+ if tags is not None:
+ tags["memory_skip_reason"] = "cache_mode"
+ log_memory_injection(
+ request_id=request_id,
+ session_id=None,
+ decision="skipped_cache_mode",
+ bytes_injected=0,
+ query=extract_user_query(optimized_messages) or "",
+ tags=tags,
+ )
+ elif self.memory_handler.config.inject_context:
try:
memory_context = await asyncio.wait_for(
self.memory_handler.search_and_format_context(
@@ -8276,7 +8310,24 @@ class OpenAIHandlerMixin:
)
# Inject memory context into instructions
- if self.memory_handler.config.inject_context:
+ if self.memory_handler.config.inject_context and is_cache_mode(
+ self.config.mode
+ ):
+ # Cache mode never injects context, so the next-turn prefix
+ # bytes stay byte-equal to this turn's bytes. Skip the search
+ # too: its result could only be discarded (#4600, #4622).
+ if ws_tags is not None:
+ ws_tags["memory_skip_reason"] = "cache_mode"
+ ws_input = ws_response_body.get("input", "")
+ log_memory_injection(
+ request_id=request_id,
+ session_id=session_id,
+ decision="skipped_cache_mode",
+ bytes_injected=0,
+ query=ws_input if isinstance(ws_input, str) else "",
+ tags=ws_tags,
+ )
+ elif self.memory_handler.config.inject_context:
ws_input = ws_response_body.get("input", "")
ws_instructions = ws_response_body.get("instructions")
ws_msgs: list[dict[str, Any]] = []
diff --git a/tests/test_codex_ws_per_frame_memory.py b/tests/test_codex_ws_per_frame_memory.py
index 2c385092c..a705e96f4 100644
--- a/tests/test_codex_ws_per_frame_memory.py
+++ b/tests/test_codex_ws_per_frame_memory.py
@@ -196,6 +196,50 @@ async def test_memory_lookup_runs_for_each_issue_artifact_frame_and_preserves_no
assert forwarded_turns[0]["response"]["tools"] == forwarded_turns[1]["response"]["tools"]
+async def test_cache_mode_skips_context_on_each_frame_and_preserves_tools():
+ upstream = _FakeUpstream(
+ [
+ json.dumps({"type": "response.created", "response": {"id": "r_1"}}),
+ json.dumps({"type": "response.completed", "response": {"id": "r_1"}}),
+ ],
+ hold_after_events=True,
+ )
+ first, later = _issue_2059_inputs()
+ cancel = json.dumps({"type": "response.cancel"})
+ frames = [
+ _turn_with_tools(first, _client_response_tools()),
+ cancel,
+ _turn_with_tools(later, _client_response_tools()),
+ ]
+ client_ws = _FakeWebSocket(frames=frames)
+ handler = _DummyOpenAIHandler()
+ handler.config.mode = "cache"
+ memory = _MemoryHandler()
+ handler.memory_handler = memory
+
+ with (
+ patch.dict(sys.modules, {"websockets": _make_fake_websockets_module(upstream)}),
+ patch("headroom.proxy.helpers.log_memory_injection") as log_injection,
+ ):
+ await handler.handle_openai_responses_ws(client_ws)
+
+ assert memory.queries == []
+ assert upstream.sent[1] == cancel
+ forwarded = [json.loads(upstream.sent[i])["response"] for i in (0, 2)]
+ assert [response["input"] for response in forwarded] == [first, later]
+ expected_tools = [*_client_response_tools(), *_expected_memory_response_tools()]
+ assert all(response["tools"] == expected_tools for response in forwarded)
+ skips = [
+ call.kwargs
+ for call in log_injection.call_args_list
+ if call.kwargs.get("decision") == "skipped_cache_mode"
+ ]
+ assert len(skips) == 2
+ assert all(call["bytes_injected"] == 0 for call in skips)
+ assert all(call["tags"]["memory_skip_reason"] == "cache_mode" for call in skips)
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize("tools", [None, []], ids=["omitted", "empty"])
async def test_ws_responses_do_not_inject_memory_tools_without_client_tools(tools):
diff --git a/tests/test_openai_codex_ws_lifecycle.py b/tests/test_openai_codex_ws_lifecycle.py
index f468183ed..39ee352e6 100644
--- a/tests/test_openai_codex_ws_lifecycle.py
+++ b/tests/test_openai_codex_ws_lifecycle.py
@@ -129,6 +129,7 @@ class _DummyOpenAIHandler(OpenAIHandlerMixin):
self.metrics = _DummyMetrics()
self.config = SimpleNamespace(
optimize=False,
+ mode="token",
retry_max_attempts=1,
retry_base_delay_ms=1,
retry_max_delay_ms=1,
diff --git a/tests/test_proxy_system_prompt_immutable.py b/tests/test_proxy_system_prompt_immutable.py
index cb619018b..bb5a435f2 100644
--- a/tests/test_proxy_system_prompt_immutable.py
+++ b/tests/test_proxy_system_prompt_immutable.py
@@ -21,6 +21,11 @@ from fastapi.testclient import TestClient
from headroom.proxy.server import ProxyConfig, create_app
[email protected](autouse=True)
+def _stub_env(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("HEADROOM_REQUIRE_RUST_CORE", "false")
+
+
class _FakePrefixTracker:
def __init__(self, frozen_count: int):
self._frozen_count = frozen_count
@@ -474,3 +479,107 @@ def test_memory_context_appears_in_responses_api_input_tail() -> None:
assert any("MEMCTX" in t for t in text_parts)
elif isinstance(input_field, str):
assert input_field.endswith("MEMCTX")
+
+
+def test_cache_mode_skips_memory_context_search_openai_chat() -> None:
+ """Cache mode never injects memory context on OpenAI chat, so it must not search for it."""
+ captured: dict[str, object] = {}
+ with _make_proxy_client() as client:
+ proxy = client.app.state.proxy
+ proxy.config.mode = "cache"
+ proxy.config.optimize = False
+ proxy.config.image_optimize = False
+ proxy.config.ccr_proactive_expansion = False
+
+ _install_session_tracker(proxy, frozen_count=1)
+ _install_memory_handler(proxy)
+
+ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
+ captured["body"] = body
+ return httpx.Response(
+ 200,
+ json={
+ "id": "chatcmpl_1",
+ "object": "chat.completion",
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": "ack"},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": {"prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12},
+ },
+ )
+
+ proxy._retry_request = _fake_retry
+
+ response = client.post(
+ "/v1/chat/completions",
+ headers={
+ "authorization": "Bearer sk-test",
+ "x-headroom-user-id": "u1",
+ },
+ json={
+ "model": "gpt-4o",
+ "messages": [
+ {"role": "system", "content": "base system"},
+ {"role": "user", "content": "frozen prefix"},
+ {"role": "assistant", "content": "ack"},
+ {"role": "user", "content": "latest user"},
+ ],
+ },
+ )
+
+ assert response.status_code == 200
+ proxy.memory_handler.search_and_format_context.assert_not_awaited()
+ sent = captured["body"]
+ assert isinstance(sent, dict)
+ assert sent["messages"][3]["content"] == "latest user"
+
+
+def test_cache_mode_skips_memory_context_search_openai_responses() -> None:
+ """Cache mode never injects memory context on OpenAI responses, so it must not search for it."""
+ captured: dict[str, object] = {}
+ with _make_responses_proxy_client() as client:
+ proxy = client.app.state.proxy
+ proxy.config.mode = "cache"
+ proxy.config.optimize = False
+ proxy.config.image_optimize = False
+ _install_memory_handler(proxy)
+
+ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
+ captured["body"] = body
+ return httpx.Response(
+ 200,
+ json={
+ "id": "resp_1",
+ "object": "response",
+ "model": "gpt-5.4",
+ "output": [],
+ "usage": {"input_tokens": 5, "output_tokens": 1, "total_tokens": 6},
+ },
+ )
+
+ proxy._retry_request = _fake_retry
+
+ response = client.post(
+ "/v1/responses",
+ headers={
+ "authorization": "Bearer sk-test",
+ "x-headroom-user-id": "u1",
+ },
+ json={
+ "model": "gpt-5.4",
+ "instructions": "You are a Codex assistant.",
+ "input": "latest user string input",
+ },
+ )
+
+ assert response.status_code == 200
+ proxy.memory_handler.search_and_format_context.assert_not_awaited()
+ assert "body" in captured
+ sent = captured["body"]
+ assert isinstance(sent, dict)
+ assert sent.get("input") == "latest user string input"
+ assert sent.get("instructions") == "You are a Codex assistant."