Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): keep Gemini memory context on entries with non-text parts (#4506)
Gaurav Dubey35a2b763f09b
diff --git a/headroom/proxy/handlers/gemini.py b/headroom/proxy/handlers/gemini.pyindex 30fb58ec5..7521b6fe7 100644--- a/headroom/proxy/handlers/gemini.py+++ b/headroom/proxy/handlers/gemini.py@@ -225,6 +225,76 @@ class GeminiHandlerMixin:# else: dropped by compression — omitreturn result+ @staticmethod+ def _carry_tail_append_into_preserved(+ original_contents: list[dict],+ preserved_indices: set[int],+ preserved_contents: dict[int, dict],+ before_messages: list[dict],+ after_messages: list[dict],+ ) -> bool:+ """Carry a memory tail append onto a preserved contents[] entry.++ ``_rebuild_gemini_contents`` forwards preserved (non-text) entries+ verbatim and discards their converted message, so text appended to+ that message never reaches the upstream body (#4419). Find the one+ message the tail append changed, map it to its contents[] index the+ same way the rebuild does (the k-th non-system message pairs with the+ k-th entry that has a text part), and, when that entry is preserved+ and is the latest user entry, append the same suffix to a copy of its+ last text part.++ Returns False when the append would be dropped: the target is a+ preserved entry older than the latest user entry. Writing memory into+ that older turn would change an already-cached prefix, so the caller+ skips the injection instead of logging one that never reaches the wire.+ """+ if len(before_messages) != len(after_messages):+ return True+ changed = [i for i, (b, a) in enumerate(zip(before_messages, after_messages)) if b is not a]+ if len(changed) != 1:+ return True+ msg_idx = changed[0]+ before = before_messages[msg_idx].get("content")+ after = after_messages[msg_idx].get("content")+ if not (isinstance(before, str) and isinstance(after, str)):+ return True+ if not after.startswith(before) or len(after) == len(before):+ return True+ suffix = after[len(before) :]++ ordinal = sum(1 for m in before_messages[:msg_idx] if m.get("role") != "system")+ text_entry_indices = [+ idx+ for idx, content in enumerate(original_contents)+ if any(isinstance(p.get("text"), str) for p in GeminiHandlerMixin._dict_parts(content))+ ]+ if ordinal >= len(text_entry_indices):+ return True+ content_idx = text_entry_indices[ordinal]+ if content_idx not in preserved_indices:+ return True+ latest_user_idx = max(+ (+ idx+ for idx, content in enumerate(original_contents)+ if (content.get("role", "user") if isinstance(content, dict) else "user") == "user"+ ),+ default=None,+ )+ if content_idx != latest_user_idx:+ return False++ entry = preserved_contents[content_idx]+ # Copy the raw list so every part (including non-dict ones) is kept as-is.+ parts = list(entry["parts"])+ last_text = max(+ i for i, p in enumerate(parts) if isinstance(p, dict) and isinstance(p.get("text"), str)+ )+ parts[last_text] = {**parts[last_text], "text": parts[last_text]["text"] + suffix}+ preserved_contents[content_idx] = {**entry, "parts": parts}+ return True+def _gemini_contents_to_messages(self,contents: list[dict],@@ -778,6 +848,17 @@ class GeminiHandlerMixin:provider="openai",))+ # Entries with non-text parts are restored verbatim by+ # _rebuild_gemini_contents; carry the append onto them so+ # it is not dropped after being logged, or skip it.+ if bytes_appended > 0 and not self._carry_tail_append_into_preserved(+ contents,+ preserved_indices,+ preserved_contents,+ optimized_messages,+ new_messages,+ ):+ bytes_appended = 0if bytes_appended > 0:optimized_messages = new_messageslogger.info(diff --git a/tests/test_gemini_memory_tail_preserved.py b/tests/test_gemini_memory_tail_preserved.pynew file mode 100644index 000000000..8ef0b891c--- /dev/null+++ b/tests/test_gemini_memory_tail_preserved.py@@ -0,0 +1,269 @@+"""Gemini memory context must reach the wire on entries with non-text parts (#4419).++The Gemini handler converts ``contents[]`` to OpenAI-shaped messages, appends+the memory block to the latest user message, then rebuilds ``contents[]``.+Entries carrying a non-text part (``functionResponse``, ``inlineData``, ...)+are restored verbatim by ``_rebuild_gemini_contents`` so compression never+rewrites them. When the latest user entry was one of those, the rebuild also+discarded the memory text, while ``injected_live_zone_tail_gemini`` had already+been logged. The memory block must instead land on the preserved entry's text.+"""++from __future__ import annotations++import copy+import json+from types import SimpleNamespace+from typing import Any++import pytest++from headroom.proxy.handlers.gemini import GeminiHandlerMixin+from headroom.proxy.memory_handler import MemoryHandler++MEMORY_BLOCK = "<memory>user prefers metric units</memory>"+++class _FakeRequest:+ def __init__(self) -> None:+ self.headers: dict[str, str] = {"x-headroom-user-id": "alice"}+ self.query_params: dict[str, str] = {}+ self.url = SimpleNamespace(path="/v1beta/models/gemini-pro:generateContent", query="")+ self.scope: dict = {"type": "http", "method": "POST"}+ self.client = SimpleNamespace(host="127.0.0.1", port=50000)+++class _UpstreamResponse:+ status_code = 200+ content = json.dumps(+ {+ "candidates": [{"content": {"parts": [{"text": "ok"}]}}],+ "usageMetadata": {"promptTokenCount": 10, "candidatesTokenCount": 1},+ }+ ).encode()+ headers = {"content-type": "application/json"}++ def json(self) -> object:+ return json.loads(self.content)+++class _MemoryHandler:+ """Memory handler double: real tail-append, fixed search result."""++ def __init__(self) -> None:+ self.config = SimpleNamespace(inject_context=True, project_root_override="")++ async def search_and_format_context(self, *args: Any, **kwargs: Any) -> str:+ return MEMORY_BLOCK++ _append_to_latest_user_tail = staticmethod(MemoryHandler._append_to_latest_user_tail)+++class _NoCcr:+ config = SimpleNamespace(enabled=False)++ def has_ccr_tool_calls(self, resp_json: Any, provider: str) -> bool:+ return False+++class _Handler(GeminiHandlerMixin):+ GEMINI_API_URL = "https://gemini.example"++ def __init__(self) -> None:+ from headroom.cache.prefix_tracker import SessionTrackerStore++ self.memory_handler = _MemoryHandler()+ self.rate_limiter = None+ self.usage_reporter = None+ self.session_tracker_store = SessionTrackerStore()+ self.config = SimpleNamespace(+ optimize=False,+ anthropic_pre_upstream_memory_context_timeout_seconds=1.0,+ ccr_inject_tool=False,+ )+ self.metrics = SimpleNamespace()+ self.ccr_response_handler = _NoCcr()+ self.outcomes: list = []+ self.forwarded: list[dict] = []++ async def _next_request_id(self) -> str:+ return "req-mem-1"++ async def _retry_request(self, method, url, headers, body): # noqa: ANN001, ANN201+ self.forwarded.append(copy.deepcopy(body))+ return _UpstreamResponse()++ async def _record_request_outcome(self, outcome) -> None: # noqa: ANN001+ self.outcomes.append(outcome)++ async def _count_tokens_offloaded(self, model, messages): # noqa: ANN001, ANN201+ return SimpleNamespace(), 10+++async def _send(monkeypatch: pytest.MonkeyPatch, body: dict) -> dict:+ async def payload(request): # noqa: ANN001, ANN202+ return body++ monkeypatch.setattr("headroom.proxy.helpers._read_request_json", payload)+ monkeypatch.setenv("HEADROOM_MEMORY_INJECTION_MODE", "live_zone_tail")++ handler = _Handler()+ response = await handler.handle_gemini_generate_content(_FakeRequest(), "gemini-pro")+ assert response.status_code == 200+ (forwarded,) = handler.forwarded+ return forwarded+++def _texts(entry: dict) -> list[str]:+ return [p["text"] for p in entry["parts"] if isinstance(p.get("text"), str)]+++async def test_memory_reaches_latest_user_entry_with_function_response(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ function_response = {"functionResponse": {"name": "get_weather", "response": {"temp_c": 21}}}+ body = {+ "contents": [+ {"role": "user", "parts": [{"text": "What is the weather?"}]},+ {"role": "model", "parts": [{"functionCall": {"name": "get_weather", "args": {}}}]},+ {"role": "user", "parts": [function_response, {"text": "Answer in one line."}]},+ ]+ }+ original = copy.deepcopy(body)+ latest_entry = body["contents"][-1]++ forwarded = await _send(monkeypatch, body)++ latest = forwarded["contents"][-1]+ # The non-text part is still forwarded untouched...+ assert latest["parts"][0] == function_response+ # ...and the memory block is on that entry's text, not dropped.+ assert _texts(latest) == ["Answer in one line.\n\n" + MEMORY_BLOCK]+ # Older turns are not touched.+ assert forwarded["contents"][:2] == original["contents"][:2]+ # The parsed request entry is copied, not mutated in place.+ assert latest_entry == original["contents"][-1]+++async def test_memory_reaches_latest_user_entry_with_inline_image(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ image = {"inlineData": {"mimeType": "image/png", "data": "iVBORw0KGgo="}}+ body = {+ "contents": [+ {"role": "user", "parts": [{"text": "Hi"}]},+ {"role": "model", "parts": [{"text": "Hello"}]},+ {"role": "user", "parts": [{"text": "Describe this"}, image, {"text": "briefly"}]},+ ]+ }++ forwarded = await _send(monkeypatch, body)++ entry = forwarded["contents"][-1]+ assert entry["parts"][1] == image+ # The block is appended after the last text part, matching the bytes the+ # text-only path produces for the joined message text.+ assert _texts(entry) == ["Describe this", "briefly\n\n" + MEMORY_BLOCK]+++async def test_text_only_latest_entry_is_unchanged(monkeypatch: pytest.MonkeyPatch) -> None:+ body = {+ "contents": [+ {"role": "user", "parts": [{"text": "hello"}]},+ {"role": "model", "parts": [{"text": "hi"}]},+ {"role": "user", "parts": [{"text": "what did I say?"}]},+ ]+ }++ forwarded = await _send(monkeypatch, body)++ assert forwarded["contents"][-1] == {+ "role": "user",+ "parts": [{"text": "what did I say?\n\n" + MEMORY_BLOCK}],+ }+ assert forwarded["contents"][0] == {"role": "user", "parts": [{"text": "hello"}]}+++async def test_older_preserved_entry_is_not_rewritten_or_logged(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The latest user entry has no text, so the tail append targets an older+ user entry. That entry is preserved and already in the provider's prompt+ cache: skip the injection instead of rewriting it or logging a drop."""+ logged: list[dict] = []+ monkeypatch.setattr(+ "headroom.proxy.helpers.log_memory_injection", lambda **kw: logged.append(kw)+ )+ image = {"inlineData": {"mimeType": "image/png", "data": "iVBORw0KGgo="}}+ body = {+ "contents": [+ {"role": "user", "parts": [{"text": "Hi"}]},+ {"role": "model", "parts": [{"text": "Hello"}]},+ {"role": "user", "parts": [image, {"text": "What city is this?"}]},+ {"role": "model", "parts": [{"functionCall": {"name": "lookup", "args": {}}}]},+ {"role": "user", "parts": [{"functionResponse": {"name": "lookup", "response": {}}}]},+ ]+ }+ original = copy.deepcopy(body)++ forwarded = await _send(monkeypatch, body)++ assert forwarded["contents"] == original["contents"]+ assert logged == []+++def test_carry_skips_when_target_entry_is_not_preserved() -> None:+ contents = [{"role": "user", "parts": [{"text": "a"}]}]+ preserved: dict[int, dict] = {}+ before = [{"role": "user", "content": "a"}]+ after = [{"role": "user", "content": "a\n\nmem"}]++ assert GeminiHandlerMixin._carry_tail_append_into_preserved(+ contents, set(), preserved, before, after+ )+ assert preserved == {}+++_IMAGE_ENTRY = {+ "role": "user",+ "parts": [{"inlineData": {"mimeType": "image/png", "data": "AA=="}}, {"text": "a"}],+}+_BEFORE = [{"role": "user", "content": "a"}]+_SYSTEM = {"role": "system", "content": "s"}+++ ("before", "after"),+ [+ pytest.param(_BEFORE, [*_BEFORE, {"role": "user", "content": "b"}], id="length-changed"),+ pytest.param(_BEFORE, list(_BEFORE), id="nothing-changed"),+ pytest.param(+ _BEFORE,+ [{"role": "user", "content": [{"type": "text", "text": "a\n\nmem"}]}],+ id="non-string-content",+ ),+ pytest.param(_BEFORE, [{"role": "user", "content": "rewritten"}], id="not-an-append"),+ pytest.param(+ [_SYSTEM, {"role": "user", "content": "x"}],+ [_SYSTEM, {"role": "user", "content": "x\n\nmem"}],+ id="no-matching-entry",+ ),+ ],+)+def test_carry_leaves_preserved_entry_alone_for_unrecognized_shapes(+ before: list[dict], after: list[dict]+) -> None:+ """Anything other than one string message gaining a suffix is not a tail+ append this helper understands: keep today's behavior untouched."""+ # "no-matching-entry" maps to a second text entry that does not exist.+ contents = [_IMAGE_ENTRY] if len(before) == 1 else [{"role": "user", "parts": []}]+ preserved = {0: _IMAGE_ENTRY}++ assert GeminiHandlerMixin._carry_tail_append_into_preserved(+ contents, {0}, preserved, before, after+ )+ assert preserved == {0: _IMAGE_ENTRY}