Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(cache): preserve the forwarded OpenAI chat prefix across turns (#3757)

donjordeca575c63ec
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index fac68afd2..9b44decce 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -4174,6 +4174,17 @@ class OpenAIHandlerMixin:
f"[{request_id}] Restored {restored_count} frozen prefix message(s) "
"to preserve cache stability (openai)"
)
+ # The restore writes raw client originals, but the provider cached
+ # what we forwarded last turn. Replay that over the restored result;
+ # the overlay's own checks keep the originals wherever replay is not
+ # provably safe. Token counts are recomputed from the final body below.
+ optimized_messages = finalize_turn(
+ optimized_messages,
+ original_client_messages,
+ openai_prefix_tracker.get_last_original_messages(),
+ openai_prefix_tracker.get_last_forwarded_messages(),
+ confirmed_frozen_count=_openai_confirmed_frozen,
+ ).messages
# Memory: inject context and tools for OpenAI requests.
#
@@ -4646,6 +4657,7 @@ class OpenAIHandlerMixin:
waste_signals=waste_signals_dict,
prefix_tracker=openai_prefix_tracker,
optimized_messages=optimized_messages,
+ original_messages=original_client_messages,
backend=request_backend,
)
else:
@@ -4885,6 +4897,7 @@ class OpenAIHandlerMixin:
cache_read_tokens=cache_read_tokens,
cache_write_tokens=cache_write_tokens,
messages=optimized_messages,
+ original_messages=original_client_messages,
)
await self._record_request_outcome(
@@ -4983,6 +4996,7 @@ class OpenAIHandlerMixin:
optimization_latency,
pipeline_timing=pipeline_timing,
prefix_tracker=openai_prefix_tracker,
+ original_messages=original_client_messages,
outcome_provider=openai_chat_outcome_provider,
)
else:
@@ -5263,6 +5277,7 @@ class OpenAIHandlerMixin:
cache_read_tokens=cache_read_tokens,
cache_write_tokens=cache_write_tokens,
messages=optimized_messages,
+ original_messages=original_client_messages,
)
# OpenAI has no write penalty — uncached = total - cached
diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.py
index 834d94054..a6fc34ff5 100644
--- a/headroom/proxy/handlers/streaming.py
+++ b/headroom/proxy/handlers/streaming.py
@@ -2126,6 +2126,7 @@ class StreamingMixin:
waste_signals: dict[str, int] | None = None,
prefix_tracker: Any | None = None,
optimized_messages: list[dict] | None = None,
+ original_messages: list[dict] | None = None,
backend: Any | None = None,
) -> StreamingResponse:
"""Stream OpenAI chat completion response from backend.
@@ -2148,6 +2149,11 @@ class StreamingMixin:
the FINAL usage frame can update the tracker for the next turn
— mirroring the direct streaming path
(``_stream_response``/``_finalize_stream_response``).
+ ``original_messages`` is the immutable pre-transform client
+ snapshot for the same update: the tracker must record what the
+ client SENT as its ``last_original_messages``, not what we
+ forwarded, or next turn's overlay never matches the client
+ prefix and cannot replay the cached bytes.
NOTE: CCR request-level intercept on the streaming path is
intentionally OUT OF SCOPE. Mirrors the Anthropic streaming
@@ -2271,6 +2277,7 @@ class StreamingMixin:
cache_read_tokens=cache_read_tokens,
cache_write_tokens=cache_write_tokens,
messages=tracker_messages,
+ original_messages=original_messages,
)
# CCR Feedback: record headroom_retrieve tool calls so
diff --git a/tests/test_proxy_openai_cache_stability.py b/tests/test_proxy_openai_cache_stability.py
index 86987e1bf..fcf276c19 100644
--- a/tests/test_proxy_openai_cache_stability.py
+++ b/tests/test_proxy_openai_cache_stability.py
@@ -3,7 +3,9 @@
from __future__ import annotations
import copy
+import json
from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
@@ -12,6 +14,7 @@ pytest.importorskip("fastapi")
from fastapi.testclient import TestClient
+from headroom.backends.base import BackendResponse
from headroom.proxy.server import ProxyConfig, create_app
@@ -25,6 +28,7 @@ class _FakePrefixTracker:
self._frozen_count = frozen_count
self._previous_original = previous_original or []
self._previous_forwarded = previous_forwarded or []
+ self.update_calls: list[dict] = []
def get_frozen_message_count(self) -> int:
return self._frozen_count
@@ -40,10 +44,11 @@ class _FakePrefixTracker:
return copy.deepcopy(self._previous_forwarded)
def update_from_response(self, **kwargs): # noqa: ANN003
+ self.update_calls.append(kwargs)
return None
-def _make_proxy_client() -> TestClient:
+def _make_proxy_client(**config_overrides) -> TestClient: # noqa: ANN003
config = ProxyConfig(
optimize=False,
cache_enabled=False,
@@ -54,6 +59,7 @@ def _make_proxy_client() -> TestClient:
ccr_handle_responses=False,
ccr_context_tracking=False,
image_optimize=False,
+ **config_overrides,
)
app = create_app(config)
return TestClient(app)
@@ -564,3 +570,610 @@ def test_openai_handler_replays_the_provider_confirmed_prefix_even_when_it_infla
assert response.status_code == 200
assert captured["body"]["messages"][0] == previous_forwarded[0]
assert captured["body"]["messages"][1] == {"role": "user", "content": "new suffix"}
+
+
+def _tool_history() -> list[dict]:
+ """A history whose big tool result the pipeline can compress."""
+ return [
+ {"role": "user", "content": "run the build and show me the log"},
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "call_1",
+ "type": "function",
+ "function": {"name": "bash", "arguments": "{}"},
+ }
+ ],
+ },
+ {
+ "role": "tool",
+ "tool_call_id": "call_1",
+ "content": "build log line 0001 module m0 status OK\n" * 200,
+ },
+ ]
+
+
+def _compress_first_tool_result(**kwargs):
+ """Fake pipeline: compress the big tool result so forwarded ≠ client bytes."""
+ mutated = [dict(message) for message in kwargs["messages"]]
+ for i, message in enumerate(mutated):
+ if message.get("role") == "tool":
+ mutated[i] = {**message, "content": "[compressed build log]"}
+ break
+ return SimpleNamespace(
+ messages=mutated,
+ transforms_applied=["fake:mutated"],
+ timing={},
+ tokens_before=2000,
+ tokens_after=30,
+ waste_signals=None,
+ )
+
+
+def _install_tracker(proxy, tracker) -> None:
+ proxy.session_tracker_store.compute_session_id = lambda request, model, messages: (
+ "stable-session"
+ )
+ proxy.session_tracker_store.get_or_create = lambda session_id, provider: tracker
+ proxy.session_tracker_store.resolve_tracker = lambda *args, **kwargs: tracker
+
+
+def _chat_usage(prompt_tokens: int, cached_tokens: int) -> dict:
+ return {
+ "prompt_tokens": prompt_tokens,
+ "completion_tokens": 3,
+ "total_tokens": prompt_tokens + 3,
+ "prompt_tokens_details": {"cached_tokens": cached_tokens},
+ }
+
+
+def test_openai_streaming_chat_hands_client_originals_to_prefix_tracker() -> None:
+ """Direct streaming chat must give the stream finalizer the client's originals.
+
+ Without them the prefix tracker stores the forwarded (compressed) messages
+ as "original", so next turn's overlay never matches the client prefix and
+ cannot replay the cached bytes.
+ """
+ captured = {}
+ client_messages = _tool_history() + [{"role": "user", "content": "current turn"}]
+ with _make_proxy_client() as client:
+ proxy = client.app.state.proxy
+ proxy.config.optimize = True
+ proxy.config.mode = "cache"
+ _install_tracker(proxy, _FakePrefixTracker(frozen_count=0))
+ proxy.openai_pipeline.apply = _compress_first_tool_result
+
+ async def _fake_stream_response(*args, **kwargs): # noqa: ANN002, ANN003
+ captured.update(kwargs)
+ return httpx.Response(200, text="data: [DONE]\n\n")
+
+ proxy._stream_response = _fake_stream_response
+ client.post(
+ "/v1/chat/completions",
+ headers={"authorization": "Bearer test-key"},
+ json={"model": "gpt-4o-mini", "stream": True, "messages": client_messages},
+ )
+
+ assert captured.get("original_messages") == client_messages
+
+
+def test_openai_buffered_chat_hands_client_originals_to_prefix_tracker() -> None:
+ """Direct buffered chat must record the client's originals, not its own output.
+
+ Token mode isolates the recording behavior: in cache mode the frozen-prefix
+ restore reverts prefix bytes before they are forwarded, which would make the
+ forwarded list coincidentally equal the originals. The originals plumbing
+ under test here is mode-independent.
+ """
+ client_messages = _tool_history() + [{"role": "user", "content": "current turn"}]
+ with _make_proxy_client() as client:
+ proxy = client.app.state.proxy
+ proxy.config.optimize = True
+ proxy.config.mode = "token"
+ tracker = _FakePrefixTracker(frozen_count=0)
+ _install_tracker(proxy, tracker)
+ proxy.openai_pipeline.apply = _compress_first_tool_result
+
+ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
+ return httpx.Response(
+ 200,
+ json={
+ "id": "chatcmpl_originals_buffered",
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": "ok"},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": _chat_usage(30, 0),
+ },
+ )
+
+ proxy._retry_request = _fake_retry
+ response = client.post(
+ "/v1/chat/completions",
+ headers={"authorization": "Bearer test-key"},
+ json={"model": "gpt-4o-mini", "messages": client_messages},
+ )
+
+ assert response.status_code == 200
+ assert len(tracker.update_calls) == 1
+ call = tracker.update_calls[0]
+ # The tracker must store what the client SENT as "original", and the
+ # compressed form separately as what was forwarded.
+ assert call["original_messages"] == client_messages
+ assert call["messages"][2]["content"] == "[compressed build log]"
+
+
+def _mock_openai_backend_response() -> dict:
+ return {
+ "id": "chatcmpl-backend-originals",
+ "object": "chat.completion",
+ "model": "gpt-4o-mini",
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": "ok"},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": _chat_usage(30, 0),
+ }
+
+
+def _sse_chunks(prompt_tokens: int, cached_tokens: int) -> list[str]:
+ return [
+ 'data: {"id":"c1","object":"chat.completion.chunk",'
+ '"choices":[{"index":0,"delta":{"role":"assistant","content":"ok"}}]}\n\n',
+ 'data: {"id":"c1","object":"chat.completion.chunk",'
+ '"choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n',
+ 'data: {"id":"c1","object":"chat.completion.chunk","choices":[],'
+ f'"usage":{{"prompt_tokens":{prompt_tokens},"completion_tokens":3,'
+ f'"total_tokens":{prompt_tokens + 3},'
+ f'"prompt_tokens_details":{{"cached_tokens":{cached_tokens}}}}}}}\n\n',
+ "data: [DONE]\n\n",
+ ]
+
+
+def _mock_streaming_backend(chunks: list[str]) -> MagicMock:
+ async def fake_stream(body, headers): # noqa: ANN001
+ for chunk in chunks:
+ yield chunk
+
+ backend = MagicMock()
+ backend.name = "anyllm-openai"
+ backend.stream_openai_message = fake_stream
+ return backend
+
+
+def test_openai_backend_buffered_chat_hands_client_originals_to_prefix_tracker() -> None:
+ """Backend-routed buffered chat must record the client's originals too."""
+ client_messages = _tool_history() + [{"role": "user", "content": "current turn"}]
+ backend = MagicMock()
+ backend.name = "anyllm-openai"
+ backend.send_openai_message = AsyncMock(
+ return_value=BackendResponse(
+ body=_mock_openai_backend_response(),
+ status_code=200,
+ headers={"content-type": "application/json"},
+ )
+ )
+ with patch("headroom.proxy.server.AnyLLMBackend", return_value=backend):
+ with _make_proxy_client(backend="anyllm", anyllm_provider="openai") as client:
+ proxy = client.app.state.proxy
+ proxy.config.optimize = True
+ proxy.config.mode = "token"
+ tracker = _FakePrefixTracker(frozen_count=0)
+ _install_tracker(proxy, tracker)
+ proxy.openai_pipeline.apply = _compress_first_tool_result
+ response = client.post(
+ "/v1/chat/completions",
+ headers={"authorization": "Bearer test-key"},
+ json={"model": "gpt-4o-mini", "stream": False, "messages": client_messages},
+ )
+
+ assert response.status_code == 200
+ assert backend.send_openai_message.await_count == 1
+ assert len(tracker.update_calls) == 1
+ call = tracker.update_calls[0]
+ assert call["original_messages"] == client_messages
+ assert call["messages"][2]["content"] == "[compressed build log]"
+
+
+def test_openai_backend_streaming_chat_hands_client_originals_to_prefix_tracker() -> None:
+ """Backend-routed streaming chat must record the client's originals too."""
+ client_messages = _tool_history() + [{"role": "user", "content": "current turn"}]
+ backend = _mock_streaming_backend(_sse_chunks(prompt_tokens=30, cached_tokens=0))
+ with patch("headroom.proxy.server.AnyLLMBackend", return_value=backend):
+ with _make_proxy_client(backend="anyllm", anyllm_provider="openai") as client:
+ proxy = client.app.state.proxy
+ proxy.config.optimize = True
+ proxy.config.mode = "token"
+ tracker = _FakePrefixTracker(frozen_count=0)
+ _install_tracker(proxy, tracker)
+ proxy.openai_pipeline.apply = _compress_first_tool_result
+ response = client.post(
+ "/v1/chat/completions",
+ headers={"authorization": "Bearer test-key"},
+ json={"model": "gpt-4o-mini", "stream": True, "messages": client_messages},
+ )
+
+ assert response.status_code == 200
+ assert "[DONE]" in response.text
+ assert len(tracker.update_calls) == 1
+ call = tracker.update_calls[0]
+ assert call["original_messages"] == client_messages
+ assert call["messages"][2]["content"] == "[compressed build log]"
+
+
+def test_openai_chat_real_tracker_stores_client_originals_across_turns() -> None:
+ """Real PrefixCacheTracker, two buffered turns (token mode).
+
+ Turn 2's stored "original" history must be the client's own messages
+ (including the new turn), not the compressed bytes that were forwarded —
+ otherwise next turn's overlay can never match the client prefix and
+ cannot replay the previously forwarded bytes.
+ """
+ from headroom.cache.prefix_tracker import PrefixCacheTracker
+
+ turn1_messages = _tool_history()
+ turn2_messages = turn1_messages + [
+ {"role": "assistant", "content": "the build passed"},
+ {"role": "user", "content": "and now?"},
+ ]
+ with _make_proxy_client() as client:
+ proxy = client.app.state.proxy
+ proxy.config.optimize = True
+ proxy.config.mode = "token"
+ tracker = PrefixCacheTracker(provider="openai")
+ _install_tracker(proxy, tracker)
+ proxy.openai_pipeline.apply = _compress_first_tool_result
+
+ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
+ return httpx.Response(
+ 200,
+ json={
+ "id": "chatcmpl_real_tracker",
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": "ok"},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": _chat_usage(prompt_tokens=5000, cached_tokens=0),
+ },
+ )
+
+ proxy._retry_request = _fake_retry
+ for messages in (turn1_messages, turn2_messages):
+ response = client.post(
+ "/v1/chat/completions",
+ headers={"authorization": "Bearer test-key"},
+ json={"model": "gpt-4o-mini", "messages": messages},
+ )
+ assert response.status_code == 200
+
+ # The forwarded history is the compressed tool result...
+ forwarded = tracker.get_last_forwarded_messages()
+ assert forwarded[2]["content"] == "[compressed build log]"
+ # ...but the recorded "original" history must be exactly what the client
+ # sent on turn 2, byte-for-byte, new turns included.
+ assert tracker.get_last_original_messages() == turn2_messages
+
+
+def test_openai_cache_mode_keeps_replayed_prefix_over_frozen_restore() -> None:
+ """Cache mode: the frozen-prefix restore must not undo a replayed prefix.
+
+ The overlay replays last turn's forwarded bytes, which the provider cached.
+ Restoring the frozen prefix to the raw client original afterwards
+ re-forwards different bytes and busts the cache from message 0.
+ """
+ captured = {}
+ previous_original = [{"role": "user", "content": "original prefix"}]
+ previous_forwarded = [{"role": "user", "content": "comp"}]
+ fake_tracker = _FakePrefixTracker(1, previous_original, previous_forwarded)
+ with _make_proxy_client() as client:
+ proxy = client.app.state.proxy
+ proxy.config.optimize = True
+ proxy.config.mode = "cache"
+ proxy.session_tracker_store.compute_session_id = lambda request, model, messages: (
+ "stable-session"
+ )
+ proxy.session_tracker_store.get_or_create = lambda session_id, provider: fake_tracker
+ proxy.session_tracker_store.resolve_tracker = lambda *args, **kwargs: fake_tracker
+
+ def _fake_apply(**kwargs):
+ return SimpleNamespace(
+ messages=kwargs["messages"],
+ transforms_applied=[],
+ timing={},
+ tokens_before=20,
+ tokens_after=20,
+ waste_signals=None,
+ )
+
+ proxy.openai_pipeline.apply = _fake_apply
+
+ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
+ captured["body"] = body
+ return httpx.Response(
+ 200,
+ json={
+ "id": "chatcmpl_replay_restore",
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": "ok"},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": {"prompt_tokens": 20, "completion_tokens": 3, "total_tokens": 23},
+ },
+ )
+
+ proxy._retry_request = _fake_retry
+ response = client.post(
+ "/v1/chat/completions",
+ headers={"authorization": "Bearer test-key"},
+ json={
+ "model": "gpt-4o-mini",
+ "messages": [
+ {"role": "user", "content": "original prefix"},
+ {"role": "user", "content": "new suffix"},
+ ],
+ },
+ )
+
+ assert response.status_code == 200
+ assert captured["body"]["messages"] == [
+ previous_forwarded[0],
+ {"role": "user", "content": "new suffix"},
+ ]
+
+
+def _forward_cache_mode_chat(
+ tracker: _FakePrefixTracker,
+ messages: list[dict],
+ *,
+ mutate_index: int | None = None,
+) -> list[dict]:
+ """Send one cache-mode chat turn and return the messages forwarded upstream.
+
+ ``mutate_index`` makes the fake pipeline rewrite that message, standing in
+ for a transform that touched a frozen position."""
+ captured: dict = {}
+ with _make_proxy_client() as client:
+ proxy = client.app.state.proxy
+ proxy.config.optimize = True
+ proxy.config.mode = "cache"
+ proxy.session_tracker_store.compute_session_id = lambda request, model, messages: (
+ "stable-session"
+ )
+ proxy.session_tracker_store.get_or_create = lambda session_id, provider: tracker
+ proxy.session_tracker_store.resolve_tracker = lambda *args, **kwargs: tracker
+
+ def _fake_apply(**kwargs):
+ out = list(kwargs["messages"])
+ if mutate_index is not None:
+ out[mutate_index] = {**out[mutate_index], "content": "MUTATED_BY_PIPELINE"}
+ return SimpleNamespace(
+ messages=out,
+ transforms_applied=[],
+ timing={},
+ tokens_before=20,
+ tokens_after=20,
+ waste_signals=None,
+ )
+
+ proxy.openai_pipeline.apply = _fake_apply
+
+ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
+ captured["body"] = body
+ return httpx.Response(
+ 200,
+ json={
+ "id": "chatcmpl_restore_replay",
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": "ok"},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": {"prompt_tokens": 20, "completion_tokens": 3, "total_tokens": 23},
+ },
+ )
+
+ proxy._retry_request = _fake_retry
+ response = client.post(
+ "/v1/chat/completions",
+ headers={"authorization": "Bearer test-key"},
+ json={"model": "gpt-4o-mini", "messages": messages},
+ )
+ assert response.status_code == 200
+ return captured["body"]["messages"]
+
+
+def test_openai_cache_mode_replays_only_up_to_the_first_changed_message() -> None:
+ """Replay stops where the client's history diverges. Past that point the
+ frozen messages are restored to the client's bytes, so a pipeline rewrite
+ of a frozen message never reaches the provider, even on a replayed turn."""
+ previous_original = [
+ {"role": "user", "content": "first original " * 20},
+ {"role": "user", "content": "second original " * 20},
+ ]
+ previous_forwarded = [
+ {"role": "user", "content": "first comp"},
+ {"role": "user", "content": "second comp"},
+ ]
+ current = [
+ previous_original[0],
+ {"role": "user", "content": "second edited by the client"},
+ {"role": "user", "content": "new suffix"},
+ ]
+
+ forwarded = _forward_cache_mode_chat(
+ _FakePrefixTracker(2, previous_original, previous_forwarded),
+ current,
+ mutate_index=1,
+ )
+
+ assert forwarded == [previous_forwarded[0], current[1], current[2]]
+
+
+def test_openai_cache_mode_keeps_restored_originals_when_history_diverges() -> None:
+ """A rewritten history head has no safe replay: the restored originals go out."""
+ previous_original = [{"role": "user", "content": "original prefix"}]
+ previous_forwarded = [{"role": "user", "content": "comp"}]
+ current = [
+ {"role": "user", "content": "client rewrote the head"},
+ {"role": "user", "content": "new suffix"},
+ ]
+
+ forwarded = _forward_cache_mode_chat(
+ _FakePrefixTracker(1, previous_original, previous_forwarded),
+ current,
+ mutate_index=0,
+ )
+
+ assert forwarded == current
+
+
+def test_openai_cache_mode_block_append_replay_survives_the_restore() -> None:
+ """Blocks appended inside the last message: forwarded blocks, then new ones."""
+ previous_original = [{"role": "user", "content": [{"type": "text", "text": "block one " * 30}]}]
+ previous_forwarded = [{"role": "user", "content": [{"type": "text", "text": "[compacted]"}]}]
+ current = [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "block one " * 30},
+ {"type": "text", "text": "block two"},
+ ],
+ }
+ ]
+
+ forwarded = _forward_cache_mode_chat(
+ _FakePrefixTracker(1, previous_original, previous_forwarded), current
+ )
+
+ assert forwarded == [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "[compacted]"},
+ {"type": "text", "text": "block two"},
+ ],
+ }
+ ]
+
+
+def _compress_unfrozen_tool_results(**kwargs):
+ """Fake pipeline that, like the real one, only rewrites past the frozen count."""
+ frozen = kwargs.get("frozen_message_count") or 0
+ out = [
+ {**message, "content": f"[compressed {message['tool_call_id']}]"}
+ if i >= frozen and message.get("role") == "tool"
+ else message
+ for i, message in enumerate(kwargs["messages"])
+ ]
+ return SimpleNamespace(
+ messages=out,
+ transforms_applied=["fake:compressed"],
+ timing={},
+ tokens_before=2000,
+ tokens_after=30,
+ waste_signals=None,
+ )
+
+
+def _provider_reply(request: httpx.Request) -> httpx.Response:
+ usage = _chat_usage(prompt_tokens=4000, cached_tokens=2048)
+ if json.loads(request.content).get("stream"):
+ return httpx.Response(
+ 200,
+ content="".join(_sse_chunks(4000, 2048)).encode(),
+ headers={"content-type": "text/event-stream"},
+ )
+ return httpx.Response(
+ 200,
+ json={
+ "id": "chatcmpl_lifecycle",
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": "ok"},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": usage,
+ },
+ )
+
+
[email protected]("stream", [False, True], ids=["buffered", "stream"])
+def test_openai_cache_mode_forwards_last_turns_bytes_unchanged(stream: bool) -> None:
+ """Cache mode, real tracker, three turns: every turn re-forwards the previous
+ turn's forwarded messages byte-for-byte, so the provider's prefix cache holds.
+
+ Needs both halves: the tracker must record the client's originals (else the
+ overlay cannot match the client prefix), and the frozen-prefix restore must
+ not undo the replay (else the compressed tool results revert to raw bytes).
+ """
+ from headroom.cache.prefix_tracker import PrefixCacheTracker
+
+ def tool_turn(n: int) -> list[dict]:
+ call_id = f"call_{n}"
+ return [
+ {"role": "user", "content": f"turn {n}: run the build"},
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": call_id,
+ "type": "function",
+ "function": {"name": "bash", "arguments": "{}"},
+ }
+ ],
+ },
+ {
+ "role": "tool",
+ "tool_call_id": call_id,
+ "content": f"build {n} log line module m0 status OK\n" * 200,
+ },
+ ]
+
+ sent: list[list[dict]] = []
+
+ def provider(request: httpx.Request) -> httpx.Response:
+ sent.append(json.loads(request.content)["messages"])
+ return _provider_reply(request)
+
+ history: list[dict] = []
+ with _make_proxy_client() as client:
+ proxy = client.app.state.proxy
+ proxy.config.optimize = True
+ proxy.config.mode = "cache"
+ _install_tracker(proxy, PrefixCacheTracker(provider="openai"))
+ proxy.openai_pipeline.apply = _compress_unfrozen_tool_results
+ proxy.http_client = httpx.AsyncClient(transport=httpx.MockTransport(provider))
+ for n in (1, 2, 3):
+ history += tool_turn(n)
+ response = client.post(
+ "/v1/chat/completions",
+ headers={"authorization": "Bearer test-key"},
+ json={"model": "gpt-4o-mini", "stream": stream, "messages": history},
+ )
+ assert response.status_code == 200
+ response.read()
+ history.append({"role": "assistant", "content": "ok"})
+
+ assert len(sent) == 3
+ assert sent[0][2]["content"] == "[compressed call_1]"
+ assert sent[1][: len(sent[0])] == sent[0]
+ assert sent[2][: len(sent[1])] == sent[1]