Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
"""Gemini memory context must reach the wire on entries with non-text parts (#4419).
The Gemini handler converts ``contents[]`` to OpenAI-shaped messages, appends
the memory block to the latest user message, then rebuilds ``contents[]``.
Entries carrying a non-text part (``functionResponse``, ``inlineData``, ...)
are restored verbatim by ``_rebuild_gemini_contents`` so compression never
rewrites them. When the latest user entry was one of those, the rebuild also
discarded the memory text, while ``injected_live_zone_tail_gemini`` had already
been logged. The memory block must instead land on the preserved entry's text.
"""
from __future__ import annotations
import copy
import json
from types import SimpleNamespace
from typing import Any
import pytest
from fastapi import Request
from headroom.proxy.handlers.gemini import GeminiHandlerMixin
from headroom.proxy.memory_handler import MemoryHandler
MEMORY_BLOCK = "<memory>user prefers metric units</memory>"
def _request(body: dict) -> Request:
payload = json.dumps(body).encode("utf-8")
split = len(payload) // 2
messages = iter(
[
{"type": "http.request", "body": payload[:split], "more_body": True},
{"type": "http.request", "body": payload[split:], "more_body": False},
]
)
async def receive() -> dict:
return next(messages, {"type": "http.disconnect"})
return Request(
{
"type": "http",
"method": "POST",
"scheme": "http",
"path": "/v1beta/models/gemini-pro:generateContent",
"query_string": b"",
"headers": [(b"x-headroom-user-id", b"alice"), (b"content-type", b"application/json")],
"client": ("127.0.0.1", 50000),
"server": ("127.0.0.1", 8787),
},
receive,
)
class _UpstreamResponse:
status_code = 200
content = json.dumps(
{
"candidates": [{"content": {"parts": [{"text": "ok"}]}}],
"usageMetadata": {"promptTokenCount": 10, "candidatesTokenCount": 1},
}
).encode()
headers = {"content-type": "application/json"}
def json(self) -> object:
return json.loads(self.content)
class _MemoryHandler:
"""Memory handler double: real tail-append, fixed search result."""
def __init__(self) -> None:
self.config = SimpleNamespace(inject_context=True, project_root_override="")
async def search_and_format_context(self, *args: Any, **kwargs: Any) -> str:
return MEMORY_BLOCK
_append_to_latest_user_tail = staticmethod(MemoryHandler._append_to_latest_user_tail)
class _NoCcr:
config = SimpleNamespace(enabled=False)
def has_ccr_tool_calls(self, resp_json: Any, provider: str) -> bool:
return False
class _Handler(GeminiHandlerMixin):
GEMINI_API_URL = "https://gemini.example"
def __init__(self) -> None:
from headroom.cache.prefix_tracker import SessionTrackerStore
self.memory_handler = _MemoryHandler()
self.rate_limiter = None
self.usage_reporter = None
self.session_tracker_store = SessionTrackerStore()
self.config = SimpleNamespace(
optimize=False,
anthropic_pre_upstream_memory_context_timeout_seconds=1.0,
ccr_inject_tool=False,
)
self.metrics = SimpleNamespace()
self.ccr_response_handler = _NoCcr()
self.outcomes: list = []
self.forwarded: list[dict] = []
async def _next_request_id(self) -> str:
return "req-mem-1"
async def _retry_request(self, method, url, headers, body): # noqa: ANN001, ANN201
self.forwarded.append(copy.deepcopy(body))
return _UpstreamResponse()
async def _record_request_outcome(self, outcome) -> None: # noqa: ANN001
self.outcomes.append(outcome)
async def _count_tokens_offloaded(self, model, messages): # noqa: ANN001, ANN201
return SimpleNamespace(), 10
async def _send(monkeypatch: pytest.MonkeyPatch, body: dict) -> dict:
monkeypatch.setenv("HEADROOM_MEMORY_INJECTION_MODE", "live_zone_tail")
handler = _Handler()
response = await handler.handle_gemini_generate_content(_request(body), "gemini-pro")
assert response.status_code == 200
(forwarded,) = handler.forwarded
return forwarded
def _texts(entry: dict) -> list[str]:
return [p["text"] for p in entry["parts"] if isinstance(p.get("text"), str)]
@pytest.mark.asyncio
async def test_memory_reaches_latest_user_entry_with_function_response(
monkeypatch: pytest.MonkeyPatch,
) -> None:
function_response = {"functionResponse": {"name": "get_weather", "response": {"temp_c": 21}}}
body = {
"contents": [
{"role": "user", "parts": [{"text": "What is the weather?"}]},
{"role": "model", "parts": [{"functionCall": {"name": "get_weather", "args": {}}}]},
{"role": "user", "parts": [function_response, {"text": "Answer in one line."}]},
]
}
original = copy.deepcopy(body)
latest_entry = body["contents"][-1]
forwarded = await _send(monkeypatch, body)
latest = forwarded["contents"][-1]
# The non-text part is still forwarded untouched...
assert latest["parts"][0] == function_response
# ...and the memory block is on that entry's text, not dropped.
assert _texts(latest) == ["Answer in one line.\n\n" + MEMORY_BLOCK]
# Older turns are not touched.
assert forwarded["contents"][:2] == original["contents"][:2]
# The parsed request entry is copied, not mutated in place.
assert latest_entry == original["contents"][-1]
@pytest.mark.asyncio
async def test_memory_reaches_latest_user_entry_with_inline_image(
monkeypatch: pytest.MonkeyPatch,
) -> None:
image = {"inlineData": {"mimeType": "image/png", "data": "iVBORw0KGgo="}}
body = {
"contents": [
{"role": "user", "parts": [{"text": "Hi"}]},
{"role": "model", "parts": [{"text": "Hello"}]},
{"role": "user", "parts": [{"text": "Describe this"}, image, {"text": "briefly"}]},
]
}
forwarded = await _send(monkeypatch, body)
entry = forwarded["contents"][-1]
assert entry["parts"][1] == image
# The block is appended after the last text part, matching the bytes the
# text-only path produces for the joined message text.
assert _texts(entry) == ["Describe this", "briefly\n\n" + MEMORY_BLOCK]
@pytest.mark.asyncio
async def test_text_only_latest_entry_is_unchanged(monkeypatch: pytest.MonkeyPatch) -> None:
body = {
"contents": [
{"role": "user", "parts": [{"text": "hello"}]},
{"role": "model", "parts": [{"text": "hi"}]},
{"role": "user", "parts": [{"text": "what did I say?"}]},
]
}
forwarded = await _send(monkeypatch, body)
assert forwarded["contents"][-1] == {
"role": "user",
"parts": [{"text": "what did I say?\n\n" + MEMORY_BLOCK}],
}
assert forwarded["contents"][0] == {"role": "user", "parts": [{"text": "hello"}]}
@pytest.mark.asyncio
async def test_older_preserved_entry_is_not_rewritten_or_logged(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The latest user entry has no text, so the tail append targets an older
user entry. That entry is preserved and already in the provider's prompt
cache: skip the injection instead of rewriting it or logging a drop."""
logged: list[dict] = []
monkeypatch.setattr(
"headroom.proxy.helpers.log_memory_injection", lambda **kw: logged.append(kw)
)
image = {"inlineData": {"mimeType": "image/png", "data": "iVBORw0KGgo="}}
body = {
"contents": [
{"role": "user", "parts": [{"text": "Hi"}]},
{"role": "model", "parts": [{"text": "Hello"}]},
{"role": "user", "parts": [image, {"text": "What city is this?"}]},
{"role": "model", "parts": [{"functionCall": {"name": "lookup", "args": {}}}]},
{"role": "user", "parts": [{"functionResponse": {"name": "lookup", "response": {}}}]},
]
}
original = copy.deepcopy(body)
forwarded = await _send(monkeypatch, body)
assert forwarded["contents"] == original["contents"]
assert logged == []
def test_carry_skips_when_target_entry_is_not_preserved() -> None:
contents = [{"role": "user", "parts": [{"text": "a"}]}]
preserved: dict[int, dict] = {}
before = [{"role": "user", "content": "a"}]
after = [{"role": "user", "content": "a\n\nmem"}]
assert GeminiHandlerMixin._carry_tail_append_into_preserved(
contents, set(), preserved, before, after
)
assert preserved == {}
_IMAGE_ENTRY = {
"role": "user",
"parts": [{"inlineData": {"mimeType": "image/png", "data": "AA=="}}, {"text": "a"}],
}
_BEFORE = [{"role": "user", "content": "a"}]
_SYSTEM = {"role": "system", "content": "s"}
@pytest.mark.parametrize(
("before", "after"),
[
pytest.param(_BEFORE, [*_BEFORE, {"role": "user", "content": "b"}], id="length-changed"),
pytest.param(_BEFORE, list(_BEFORE), id="nothing-changed"),
pytest.param(
_BEFORE,
[{"role": "user", "content": [{"type": "text", "text": "a\n\nmem"}]}],
id="non-string-content",
),
pytest.param(_BEFORE, [{"role": "user", "content": "rewritten"}], id="not-an-append"),
pytest.param(
[_SYSTEM, {"role": "user", "content": "x"}],
[_SYSTEM, {"role": "user", "content": "x\n\nmem"}],
id="no-matching-entry",
),
],
)
def test_carry_leaves_preserved_entry_alone_for_unrecognized_shapes(
before: list[dict], after: list[dict]
) -> None:
"""Anything other than one string message gaining a suffix is not a tail
append this helper understands: keep today's behavior untouched."""
# "no-matching-entry" maps to a second text entry that does not exist.
contents = [_IMAGE_ENTRY] if len(before) == 1 else [{"role": "user", "parts": []}]
preserved = {0: _IMAGE_ENTRY}
assert GeminiHandlerMixin._carry_tail_append_into_preserved(
contents, {0}, preserved, before, after
)
assert preserved == {0: _IMAGE_ENTRY}