Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(memory): gracefully degrade on context lookup and initialization errors (#3251) (#4049)
Abhinav Maurya00d4f441d3f1
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex f10d86c7b..b7e1b3c7d 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -3226,6 +3226,12 @@ class AnthropicHandlerMixin:f"{self.config.anthropic_pre_upstream_memory_context_timeout_seconds:.1f}s; ""continuing without it")+ except Exception as e:+ memory_context = None+ logger.warning(+ f"[{request_id}] Memory: Context lookup failed: {e}; "+ "continuing without it"+ )try:if memory_context:from headroom.proxy.helpers import (diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex bc77e620b..6d3d8c816 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -6647,6 +6647,11 @@ class OpenAIHandlerMixin:f"[{request_id}] Memory context lookup exceeded "f"{RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS:.1f}s; continuing without it")+ except Exception as e:+ memory_context = None+ logger.warning(+ f"[{request_id}] Memory context lookup failed: {e}; continuing without it"+ )if memory_context:from headroom.proxy.helpers import (append_text_to_latest_user_input_item,diff --git a/headroom/proxy/memory_handler.py b/headroom/proxy/memory_handler.pyindex a3ea65473..66ec7afe6 100644--- a/headroom/proxy/memory_handler.py+++ b/headroom/proxy/memory_handler.py@@ -782,11 +782,19 @@ class MemoryHandler:if self.is_project_unresolved(request_context):return None- await self._ensure_initialized()- if not self._backend:- return None+ try:+ await self._ensure_initialized()+ if not self._backend:+ return None- backend, scope, effective_user_id = self._resolve_for_request(user_id, request_context)+ backend, scope, effective_user_id = self._resolve_for_request(user_id, request_context)+ except Exception as e:+ logger.warning(+ "Memory: Failed to initialize or resolve backend for user %s: %s",+ user_id,+ e,+ )+ return None# Fail-closed when the router was unable to resolve a project in# PROJECT mode and `unresolved_project_fallback="empty"` (thediff --git a/tests/test_anthropic_pre_upstream_backpressure.py b/tests/test_anthropic_pre_upstream_backpressure.pyindex 1d76e0a5a..7faeef0a6 100644--- a/tests/test_anthropic_pre_upstream_backpressure.py+++ b/tests/test_anthropic_pre_upstream_backpressure.py@@ -736,6 +736,52 @@ def test_memory_context_timeout_fails_open_and_releases_semaphore():anyio.run(_run)+def test_memory_context_exception_fails_open_and_releases_semaphore():+ class _MemoryHandler:+ def __init__(self) -> None:+ self.config = SimpleNamespace(inject_context=True, inject_tools=False)+ self.initialized = False+ self.backend = None++ async def search_and_format_context(self, _user_id, _messages, **_kwargs):+ import sqlite3++ raise sqlite3.OperationalError("unable to open database file")++ def inject_tools(self, tools, _provider):+ return tools, False++ def get_beta_headers(self) -> dict[str, str]:+ return {}++ def has_memory_tool_calls(self, _response, _provider) -> bool:+ return False++ async def handle_memory_tool_calls(self, _response, _user_id, _provider, **_kwargs):+ return []++ async def _run() -> None:+ sem = asyncio.Semaphore(1)+ handler = _DummyAnthropicHandler(anthropic_pre_upstream_sem=sem)+ handler.memory_handler = _MemoryHandler()+ req = _build_request(+ {+ "model": "claude-3-5-sonnet-latest",+ "messages": [{"role": "user", "content": "hello"}],+ },+ {+ "authorization": "Bearer sk-ant-api-test",+ "x-headroom-user-id": "user-1",+ },+ )+ response = await handler.handle_anthropic_messages(req)+ assert response.status_code == 200+ assert sem._value == 1++ with _tokenizer_patch():+ anyio.run(_run)++# --------------------------------------------------------------------------- ## /livez stays fast under Anthropic pre-upstream contention. ## --------------------------------------------------------------------------- #diff --git a/tests/test_memory_handler_native_ops.py b/tests/test_memory_handler_native_ops.pyindex 0cf770ecd..6df2845df 100644--- a/tests/test_memory_handler_native_ops.py+++ b/tests/test_memory_handler_native_ops.py@@ -1807,3 +1807,17 @@ async def test_memory_list_dispatched_via_execute_memory_tool(handler: MemoryHanpayload = json.loads(out)assert payload["status"] == "ok"assert payload["memories"][0]["id"] == "m1"+++async def test_search_and_format_context_degrades_gracefully_on_backend_error(+ handler: MemoryHandler, monkeypatch: pytest.MonkeyPatch+) -> None:+ import sqlite3++ async def fail_init() -> None:+ raise sqlite3.OperationalError("unable to open database file")++ monkeypatch.setattr(handler, "_ensure_initialized", fail_init)+ result = await handler.search_and_format_context("u1", [{"role": "user", "content": "hello"}])+ assert result is Nonediff --git a/tests/test_openai_codex_routing.py b/tests/test_openai_codex_routing.pyindex 60aaf723e..bd919d252 100644--- a/tests/test_openai_codex_routing.py+++ b/tests/test_openai_codex_routing.py@@ -1094,3 +1094,69 @@ def test_responses_savings_accept_a_session_header_as_identity(monkeypatch):_post_responses(handler, body, {"session_id": "sess-1"})_post_responses(handler, body, {"session_id": "sess-2"})assert _booked(handler) == [100, 0, 100]+++def test_handle_openai_responses_memory_lookup_exception_fails_open_and_forwards_original_request(+ monkeypatch,+):+ import sqlite3++ class _FailingContextMemoryHandler:+ def __init__(self) -> None:+ self.config = SimpleNamespace(+ inject_context=True,+ inject_tools=False,+ project_root_override=str(Path(".").resolve()),+ )+ self.call_count = 0++ async def search_and_format_context(self, user_id, messages, **kwargs):+ self.call_count += 1+ raise sqlite3.OperationalError("unable to open database file")++ def compute_memory_tool_definitions(self, provider: str) -> list[dict]:+ return []++ def has_memory_tool_calls(self, response, provider: str) -> bool:+ return False++ handler = _DummyOpenAIHandler()+ memory_handler = _FailingContextMemoryHandler()+ handler.memory_handler = memory_handler+ handler.session_tracker_store = SimpleNamespace(+ compute_session_id=lambda *a, **k: "sess-api-memory-fallback",+ )++ monkeypatch.setattr("headroom.tokenizers.get_tokenizer", lambda model: _DummyTokenizer())++ req1 = _build_request(+ {"model": "gpt-4o-mini", "input": "turn 1 original", "store": False},+ {+ "Authorization": "Bearer sk-test",+ "User-Agent": "codex-cli/0.5",+ "x-headroom-user-id": "user-1",+ },+ )+ req2 = _build_request(+ {"model": "gpt-4o-mini", "input": "turn 2 original", "store": False},+ {+ "Authorization": "Bearer sk-test",+ "User-Agent": "codex-cli/0.5",+ "x-headroom-user-id": "user-1",+ },+ )++ # First request: lookup fails, original body forwarded without memory, response succeeds+ res1 = anyio.run(handler.handle_openai_responses, req1)+ assert res1.status_code == 200+ assert handler.captured_request is not None+ _, _, _, body1 = handler.captured_request+ assert body1["input"] == "turn 1 original"++ # Subsequent request: handler continues serving traffic normally+ res2 = anyio.run(handler.handle_openai_responses, req2)+ assert res2.status_code == 200+ assert handler.captured_request is not None+ _, _, _, body2 = handler.captured_request+ assert body2["input"] == "turn 2 original"+ assert memory_handler.call_count == 2diff --git a/tests/test_openai_codex_ws_lifecycle.py b/tests/test_openai_codex_ws_lifecycle.pyindex 39ee352e6..3f8b56a24 100644--- a/tests/test_openai_codex_ws_lifecycle.py+++ b/tests/test_openai_codex_ws_lifecycle.py@@ -2203,6 +2203,113 @@ async def test_ws_memory_enabled_non_memory_response_streams_completion():assert len(upstream.sent) == 1+async def test_ws_memory_context_lookup_exception_fails_open_and_survives_subsequent_traffic():+ import sqlite3++ class _FailingWsContextMemoryHandler(_MemoryWsHandler):+ def __init__(self) -> None:+ super().__init__()+ self.config.inject_context = True+ self.config.inject_tools = False+ self.call_count = 0++ async def search_and_format_context(self, user_id, messages, **kwargs):+ self.call_count += 1+ raise sqlite3.OperationalError("unable to open database file")++ first_frame = json.dumps(+ {+ "type": "response.create",+ "response": {+ "model": "gpt-5.4",+ "input": "turn 1 original",+ },+ }+ )+ second_frame = json.dumps(+ {+ "type": "response.create",+ "response": {+ "model": "gpt-5.4",+ "input": "turn 2 original",+ },+ }+ )+ turn1_events = [+ json.dumps({"type": "response.created", "response": {"id": "r-1"}}),+ json.dumps({"type": "response.completed", "response": {"id": "r-1"}}),+ ]+ turn2_events = [+ json.dumps({"type": "response.created", "response": {"id": "r-2"}}),+ json.dumps({"type": "response.completed", "response": {"id": "r-2"}}),+ ]+ expected_events = turn1_events + turn2_events++ first_sent = asyncio.Event()+ second_sent = asyncio.Event()+ turn1_done = asyncio.Event()+ turn2_done = asyncio.Event()+ both_completed = asyncio.Event()++ class _CoordinatedUpstream(_FakeUpstream):+ async def send(self, payload: str) -> None:+ self.sent.append(payload)+ if len(self.sent) == 1:+ first_sent.set()+ elif len(self.sent) == 2:+ second_sent.set()++ async def _iter(self):+ await first_sent.wait()+ for ev in turn1_events:+ yield ev+ await second_sent.wait()+ for ev in turn2_events:+ yield ev+ await both_completed.wait()++ class _CoordinatedClientWebSocket(_FakeWebSocket):+ async def send_text(self, text: str) -> None:+ await super().send_text(text)+ if '"id": "r-1"' in text and "response.completed" in text:+ turn1_done.set()+ elif '"id": "r-2"' in text and "response.completed" in text:+ turn2_done.set()+ both_completed.set()++ async def receive_text(self) -> str:+ if not first_sent.is_set():+ return first_frame+ await turn1_done.wait()+ if not second_sent.is_set():+ return second_frame+ await turn2_done.wait()+ raise _FakeWebSocketDisconnect("client closed")++ upstream = _CoordinatedUpstream([])+ fake_ws_mod = _make_fake_websockets_module(upstream)+ client_ws = _CoordinatedClientWebSocket([])+ client_ws.headers["x-headroom-user-id"] = "user-1"+ handler = _DummyOpenAIHandler()+ memory_handler = _FailingWsContextMemoryHandler()+ handler.memory_handler = memory_handler++ with patch.dict(sys.modules, {"websockets": fake_ws_mod}):+ await asyncio.wait_for(handler.handle_openai_responses_ws(client_ws), timeout=5.0)++ # Both turns must reach upstream without injected memory+ assert len(upstream.sent) == 2+ sent0 = json.loads(upstream.sent[0])["response"]+ sent1 = json.loads(upstream.sent[1])["response"]+ assert sent0["input"] == "turn 1 original"+ assert sent1["input"] == "turn 2 original"++ # Streaming socket connection survived and client received all upstream events+ assert client_ws.sent_text == expected_events+ assert memory_handler.call_count == 2++@pytest.mark.asyncioasync def test_ws_late_memory_call_after_streamed_message_passes_through():message_item = {