Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(cache): preserve the forwarded OpenAI chat prefix across turns (#3757)
donjordeca575c63ec
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex fac68afd2..9b44decce 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -4174,6 +4174,17 @@ class OpenAIHandlerMixin:f"[{request_id}] Restored {restored_count} frozen prefix message(s) ""to preserve cache stability (openai)")+ # The restore writes raw client originals, but the provider cached+ # what we forwarded last turn. Replay that over the restored result;+ # the overlay's own checks keep the originals wherever replay is not+ # provably safe. Token counts are recomputed from the final body below.+ optimized_messages = finalize_turn(+ optimized_messages,+ original_client_messages,+ openai_prefix_tracker.get_last_original_messages(),+ openai_prefix_tracker.get_last_forwarded_messages(),+ confirmed_frozen_count=_openai_confirmed_frozen,+ ).messages# Memory: inject context and tools for OpenAI requests.#@@ -4646,6 +4657,7 @@ class OpenAIHandlerMixin:waste_signals=waste_signals_dict,prefix_tracker=openai_prefix_tracker,optimized_messages=optimized_messages,+ original_messages=original_client_messages,backend=request_backend,)else:@@ -4885,6 +4897,7 @@ class OpenAIHandlerMixin:cache_read_tokens=cache_read_tokens,cache_write_tokens=cache_write_tokens,messages=optimized_messages,+ original_messages=original_client_messages,)await self._record_request_outcome(@@ -4983,6 +4996,7 @@ class OpenAIHandlerMixin:optimization_latency,pipeline_timing=pipeline_timing,prefix_tracker=openai_prefix_tracker,+ original_messages=original_client_messages,outcome_provider=openai_chat_outcome_provider,)else:@@ -5263,6 +5277,7 @@ class OpenAIHandlerMixin:cache_read_tokens=cache_read_tokens,cache_write_tokens=cache_write_tokens,messages=optimized_messages,+ original_messages=original_client_messages,)# OpenAI has no write penalty — uncached = total - cacheddiff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.pyindex 834d94054..a6fc34ff5 100644--- a/headroom/proxy/handlers/streaming.py+++ b/headroom/proxy/handlers/streaming.py@@ -2126,6 +2126,7 @@ class StreamingMixin:waste_signals: dict[str, int] | None = None,prefix_tracker: Any | None = None,optimized_messages: list[dict] | None = None,+ original_messages: list[dict] | None = None,backend: Any | None = None,) -> StreamingResponse:"""Stream OpenAI chat completion response from backend.@@ -2148,6 +2149,11 @@ class StreamingMixin:the FINAL usage frame can update the tracker for the next turn— mirroring the direct streaming path(``_stream_response``/``_finalize_stream_response``).+ ``original_messages`` is the immutable pre-transform client+ snapshot for the same update: the tracker must record what the+ client SENT as its ``last_original_messages``, not what we+ forwarded, or next turn's overlay never matches the client+ prefix and cannot replay the cached bytes.NOTE: CCR request-level intercept on the streaming path isintentionally OUT OF SCOPE. Mirrors the Anthropic streaming@@ -2271,6 +2277,7 @@ class StreamingMixin:cache_read_tokens=cache_read_tokens,cache_write_tokens=cache_write_tokens,messages=tracker_messages,+ original_messages=original_messages,)# CCR Feedback: record headroom_retrieve tool calls sodiff --git a/tests/test_proxy_openai_cache_stability.py b/tests/test_proxy_openai_cache_stability.pyindex 86987e1bf..fcf276c19 100644--- a/tests/test_proxy_openai_cache_stability.py+++ b/tests/test_proxy_openai_cache_stability.py@@ -3,7 +3,9 @@from __future__ import annotationsimport copy+import jsonfrom types import SimpleNamespace+from unittest.mock import AsyncMock, MagicMock, patchimport httpximport pytest@@ -12,6 +14,7 @@ pytest.importorskip("fastapi")from fastapi.testclient import TestClient+from headroom.backends.base import BackendResponsefrom headroom.proxy.server import ProxyConfig, create_app@@ -25,6 +28,7 @@ class _FakePrefixTracker:self._frozen_count = frozen_countself._previous_original = previous_original or []self._previous_forwarded = previous_forwarded or []+ self.update_calls: list[dict] = []def get_frozen_message_count(self) -> int:return self._frozen_count@@ -40,10 +44,11 @@ class _FakePrefixTracker:return copy.deepcopy(self._previous_forwarded)def update_from_response(self, **kwargs): # noqa: ANN003+ self.update_calls.append(kwargs)return None-def _make_proxy_client() -> TestClient:+def _make_proxy_client(**config_overrides) -> TestClient: # noqa: ANN003config = ProxyConfig(optimize=False,cache_enabled=False,@@ -54,6 +59,7 @@ def _make_proxy_client() -> TestClient:ccr_handle_responses=False,ccr_context_tracking=False,image_optimize=False,+ **config_overrides,)app = create_app(config)return TestClient(app)@@ -564,3 +570,610 @@ def test_openai_handler_replays_the_provider_confirmed_prefix_even_when_it_inflaassert response.status_code == 200assert captured["body"]["messages"][0] == previous_forwarded[0]assert captured["body"]["messages"][1] == {"role": "user", "content": "new suffix"}+++def _tool_history() -> list[dict]:+ """A history whose big tool result the pipeline can compress."""+ return [+ {"role": "user", "content": "run the build and show me the log"},+ {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ {+ "id": "call_1",+ "type": "function",+ "function": {"name": "bash", "arguments": "{}"},+ }+ ],+ },+ {+ "role": "tool",+ "tool_call_id": "call_1",+ "content": "build log line 0001 module m0 status OK\n" * 200,+ },+ ]+++def _compress_first_tool_result(**kwargs):+ """Fake pipeline: compress the big tool result so forwarded ≠ client bytes."""+ mutated = [dict(message) for message in kwargs["messages"]]+ for i, message in enumerate(mutated):+ if message.get("role") == "tool":+ mutated[i] = {**message, "content": "[compressed build log]"}+ break+ return SimpleNamespace(+ messages=mutated,+ transforms_applied=["fake:mutated"],+ timing={},+ tokens_before=2000,+ tokens_after=30,+ waste_signals=None,+ )+++def _install_tracker(proxy, tracker) -> None:+ proxy.session_tracker_store.compute_session_id = lambda request, model, messages: (+ "stable-session"+ )+ proxy.session_tracker_store.get_or_create = lambda session_id, provider: tracker+ proxy.session_tracker_store.resolve_tracker = lambda *args, **kwargs: tracker+++def _chat_usage(prompt_tokens: int, cached_tokens: int) -> dict:+ return {+ "prompt_tokens": prompt_tokens,+ "completion_tokens": 3,+ "total_tokens": prompt_tokens + 3,+ "prompt_tokens_details": {"cached_tokens": cached_tokens},+ }+++def test_openai_streaming_chat_hands_client_originals_to_prefix_tracker() -> None:+ """Direct streaming chat must give the stream finalizer the client's originals.++ Without them the prefix tracker stores the forwarded (compressed) messages+ as "original", so next turn's overlay never matches the client prefix and+ cannot replay the cached bytes.+ """+ captured = {}+ client_messages = _tool_history() + [{"role": "user", "content": "current turn"}]+ with _make_proxy_client() as client:+ proxy = client.app.state.proxy+ proxy.config.optimize = True+ proxy.config.mode = "cache"+ _install_tracker(proxy, _FakePrefixTracker(frozen_count=0))+ proxy.openai_pipeline.apply = _compress_first_tool_result++ async def _fake_stream_response(*args, **kwargs): # noqa: ANN002, ANN003+ captured.update(kwargs)+ return httpx.Response(200, text="data: [DONE]\n\n")++ proxy._stream_response = _fake_stream_response+ client.post(+ "/v1/chat/completions",+ headers={"authorization": "Bearer test-key"},+ json={"model": "gpt-4o-mini", "stream": True, "messages": client_messages},+ )++ assert captured.get("original_messages") == client_messages+++def test_openai_buffered_chat_hands_client_originals_to_prefix_tracker() -> None:+ """Direct buffered chat must record the client's originals, not its own output.++ Token mode isolates the recording behavior: in cache mode the frozen-prefix+ restore reverts prefix bytes before they are forwarded, which would make the+ forwarded list coincidentally equal the originals. The originals plumbing+ under test here is mode-independent.+ """+ client_messages = _tool_history() + [{"role": "user", "content": "current turn"}]+ with _make_proxy_client() as client:+ proxy = client.app.state.proxy+ proxy.config.optimize = True+ proxy.config.mode = "token"+ tracker = _FakePrefixTracker(frozen_count=0)+ _install_tracker(proxy, tracker)+ proxy.openai_pipeline.apply = _compress_first_tool_result++ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001+ return httpx.Response(+ 200,+ json={+ "id": "chatcmpl_originals_buffered",+ "choices": [+ {+ "index": 0,+ "message": {"role": "assistant", "content": "ok"},+ "finish_reason": "stop",+ }+ ],+ "usage": _chat_usage(30, 0),+ },+ )++ proxy._retry_request = _fake_retry+ response = client.post(+ "/v1/chat/completions",+ headers={"authorization": "Bearer test-key"},+ json={"model": "gpt-4o-mini", "messages": client_messages},+ )++ assert response.status_code == 200+ assert len(tracker.update_calls) == 1+ call = tracker.update_calls[0]+ # The tracker must store what the client SENT as "original", and the+ # compressed form separately as what was forwarded.+ assert call["original_messages"] == client_messages+ assert call["messages"][2]["content"] == "[compressed build log]"+++def _mock_openai_backend_response() -> dict:+ return {+ "id": "chatcmpl-backend-originals",+ "object": "chat.completion",+ "model": "gpt-4o-mini",+ "choices": [+ {+ "index": 0,+ "message": {"role": "assistant", "content": "ok"},+ "finish_reason": "stop",+ }+ ],+ "usage": _chat_usage(30, 0),+ }+++def _sse_chunks(prompt_tokens: int, cached_tokens: int) -> list[str]:+ return [+ 'data: {"id":"c1","object":"chat.completion.chunk",'+ '"choices":[{"index":0,"delta":{"role":"assistant","content":"ok"}}]}\n\n',+ 'data: {"id":"c1","object":"chat.completion.chunk",'+ '"choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n',+ 'data: {"id":"c1","object":"chat.completion.chunk","choices":[],'+ f'"usage":{{"prompt_tokens":{prompt_tokens},"completion_tokens":3,'+ f'"total_tokens":{prompt_tokens + 3},'+ f'"prompt_tokens_details":{{"cached_tokens":{cached_tokens}}}}}}}\n\n',+ "data: [DONE]\n\n",+ ]+++def _mock_streaming_backend(chunks: list[str]) -> MagicMock:+ async def fake_stream(body, headers): # noqa: ANN001+ for chunk in chunks:+ yield chunk++ backend = MagicMock()+ backend.name = "anyllm-openai"+ backend.stream_openai_message = fake_stream+ return backend+++def test_openai_backend_buffered_chat_hands_client_originals_to_prefix_tracker() -> None:+ """Backend-routed buffered chat must record the client's originals too."""+ client_messages = _tool_history() + [{"role": "user", "content": "current turn"}]+ backend = MagicMock()+ backend.name = "anyllm-openai"+ backend.send_openai_message = AsyncMock(+ return_value=BackendResponse(+ body=_mock_openai_backend_response(),+ status_code=200,+ headers={"content-type": "application/json"},+ )+ )+ with patch("headroom.proxy.server.AnyLLMBackend", return_value=backend):+ with _make_proxy_client(backend="anyllm", anyllm_provider="openai") as client:+ proxy = client.app.state.proxy+ proxy.config.optimize = True+ proxy.config.mode = "token"+ tracker = _FakePrefixTracker(frozen_count=0)+ _install_tracker(proxy, tracker)+ proxy.openai_pipeline.apply = _compress_first_tool_result+ response = client.post(+ "/v1/chat/completions",+ headers={"authorization": "Bearer test-key"},+ json={"model": "gpt-4o-mini", "stream": False, "messages": client_messages},+ )++ assert response.status_code == 200+ assert backend.send_openai_message.await_count == 1+ assert len(tracker.update_calls) == 1+ call = tracker.update_calls[0]+ assert call["original_messages"] == client_messages+ assert call["messages"][2]["content"] == "[compressed build log]"+++def test_openai_backend_streaming_chat_hands_client_originals_to_prefix_tracker() -> None:+ """Backend-routed streaming chat must record the client's originals too."""+ client_messages = _tool_history() + [{"role": "user", "content": "current turn"}]+ backend = _mock_streaming_backend(_sse_chunks(prompt_tokens=30, cached_tokens=0))+ with patch("headroom.proxy.server.AnyLLMBackend", return_value=backend):+ with _make_proxy_client(backend="anyllm", anyllm_provider="openai") as client:+ proxy = client.app.state.proxy+ proxy.config.optimize = True+ proxy.config.mode = "token"+ tracker = _FakePrefixTracker(frozen_count=0)+ _install_tracker(proxy, tracker)+ proxy.openai_pipeline.apply = _compress_first_tool_result+ response = client.post(+ "/v1/chat/completions",+ headers={"authorization": "Bearer test-key"},+ json={"model": "gpt-4o-mini", "stream": True, "messages": client_messages},+ )++ assert response.status_code == 200+ assert "[DONE]" in response.text+ assert len(tracker.update_calls) == 1+ call = tracker.update_calls[0]+ assert call["original_messages"] == client_messages+ assert call["messages"][2]["content"] == "[compressed build log]"+++def test_openai_chat_real_tracker_stores_client_originals_across_turns() -> None:+ """Real PrefixCacheTracker, two buffered turns (token mode).++ Turn 2's stored "original" history must be the client's own messages+ (including the new turn), not the compressed bytes that were forwarded —+ otherwise next turn's overlay can never match the client prefix and+ cannot replay the previously forwarded bytes.+ """+ from headroom.cache.prefix_tracker import PrefixCacheTracker++ turn1_messages = _tool_history()+ turn2_messages = turn1_messages + [+ {"role": "assistant", "content": "the build passed"},+ {"role": "user", "content": "and now?"},+ ]+ with _make_proxy_client() as client:+ proxy = client.app.state.proxy+ proxy.config.optimize = True+ proxy.config.mode = "token"+ tracker = PrefixCacheTracker(provider="openai")+ _install_tracker(proxy, tracker)+ proxy.openai_pipeline.apply = _compress_first_tool_result++ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001+ return httpx.Response(+ 200,+ json={+ "id": "chatcmpl_real_tracker",+ "choices": [+ {+ "index": 0,+ "message": {"role": "assistant", "content": "ok"},+ "finish_reason": "stop",+ }+ ],+ "usage": _chat_usage(prompt_tokens=5000, cached_tokens=0),+ },+ )++ proxy._retry_request = _fake_retry+ for messages in (turn1_messages, turn2_messages):+ response = client.post(+ "/v1/chat/completions",+ headers={"authorization": "Bearer test-key"},+ json={"model": "gpt-4o-mini", "messages": messages},+ )+ assert response.status_code == 200++ # The forwarded history is the compressed tool result...+ forwarded = tracker.get_last_forwarded_messages()+ assert forwarded[2]["content"] == "[compressed build log]"+ # ...but the recorded "original" history must be exactly what the client+ # sent on turn 2, byte-for-byte, new turns included.+ assert tracker.get_last_original_messages() == turn2_messages+++def test_openai_cache_mode_keeps_replayed_prefix_over_frozen_restore() -> None:+ """Cache mode: the frozen-prefix restore must not undo a replayed prefix.++ The overlay replays last turn's forwarded bytes, which the provider cached.+ Restoring the frozen prefix to the raw client original afterwards+ re-forwards different bytes and busts the cache from message 0.+ """+ captured = {}+ previous_original = [{"role": "user", "content": "original prefix"}]+ previous_forwarded = [{"role": "user", "content": "comp"}]+ fake_tracker = _FakePrefixTracker(1, previous_original, previous_forwarded)+ with _make_proxy_client() as client:+ proxy = client.app.state.proxy+ proxy.config.optimize = True+ proxy.config.mode = "cache"+ proxy.session_tracker_store.compute_session_id = lambda request, model, messages: (+ "stable-session"+ )+ proxy.session_tracker_store.get_or_create = lambda session_id, provider: fake_tracker+ proxy.session_tracker_store.resolve_tracker = lambda *args, **kwargs: fake_tracker++ def _fake_apply(**kwargs):+ return SimpleNamespace(+ messages=kwargs["messages"],+ transforms_applied=[],+ timing={},+ tokens_before=20,+ tokens_after=20,+ waste_signals=None,+ )++ proxy.openai_pipeline.apply = _fake_apply++ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001+ captured["body"] = body+ return httpx.Response(+ 200,+ json={+ "id": "chatcmpl_replay_restore",+ "choices": [+ {+ "index": 0,+ "message": {"role": "assistant", "content": "ok"},+ "finish_reason": "stop",+ }+ ],+ "usage": {"prompt_tokens": 20, "completion_tokens": 3, "total_tokens": 23},+ },+ )++ proxy._retry_request = _fake_retry+ response = client.post(+ "/v1/chat/completions",+ headers={"authorization": "Bearer test-key"},+ json={+ "model": "gpt-4o-mini",+ "messages": [+ {"role": "user", "content": "original prefix"},+ {"role": "user", "content": "new suffix"},+ ],+ },+ )++ assert response.status_code == 200+ assert captured["body"]["messages"] == [+ previous_forwarded[0],+ {"role": "user", "content": "new suffix"},+ ]+++def _forward_cache_mode_chat(+ tracker: _FakePrefixTracker,+ messages: list[dict],+ *,+ mutate_index: int | None = None,+) -> list[dict]:+ """Send one cache-mode chat turn and return the messages forwarded upstream.++ ``mutate_index`` makes the fake pipeline rewrite that message, standing in+ for a transform that touched a frozen position."""+ captured: dict = {}+ with _make_proxy_client() as client:+ proxy = client.app.state.proxy+ proxy.config.optimize = True+ proxy.config.mode = "cache"+ proxy.session_tracker_store.compute_session_id = lambda request, model, messages: (+ "stable-session"+ )+ proxy.session_tracker_store.get_or_create = lambda session_id, provider: tracker+ proxy.session_tracker_store.resolve_tracker = lambda *args, **kwargs: tracker++ def _fake_apply(**kwargs):+ out = list(kwargs["messages"])+ if mutate_index is not None:+ out[mutate_index] = {**out[mutate_index], "content": "MUTATED_BY_PIPELINE"}+ return SimpleNamespace(+ messages=out,+ transforms_applied=[],+ timing={},+ tokens_before=20,+ tokens_after=20,+ waste_signals=None,+ )++ proxy.openai_pipeline.apply = _fake_apply++ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001+ captured["body"] = body+ return httpx.Response(+ 200,+ json={+ "id": "chatcmpl_restore_replay",+ "choices": [+ {+ "index": 0,+ "message": {"role": "assistant", "content": "ok"},+ "finish_reason": "stop",+ }+ ],+ "usage": {"prompt_tokens": 20, "completion_tokens": 3, "total_tokens": 23},+ },+ )++ proxy._retry_request = _fake_retry+ response = client.post(+ "/v1/chat/completions",+ headers={"authorization": "Bearer test-key"},+ json={"model": "gpt-4o-mini", "messages": messages},+ )+ assert response.status_code == 200+ return captured["body"]["messages"]+++def test_openai_cache_mode_replays_only_up_to_the_first_changed_message() -> None:+ """Replay stops where the client's history diverges. Past that point the+ frozen messages are restored to the client's bytes, so a pipeline rewrite+ of a frozen message never reaches the provider, even on a replayed turn."""+ previous_original = [+ {"role": "user", "content": "first original " * 20},+ {"role": "user", "content": "second original " * 20},+ ]+ previous_forwarded = [+ {"role": "user", "content": "first comp"},+ {"role": "user", "content": "second comp"},+ ]+ current = [+ previous_original[0],+ {"role": "user", "content": "second edited by the client"},+ {"role": "user", "content": "new suffix"},+ ]++ forwarded = _forward_cache_mode_chat(+ _FakePrefixTracker(2, previous_original, previous_forwarded),+ current,+ mutate_index=1,+ )++ assert forwarded == [previous_forwarded[0], current[1], current[2]]+++def test_openai_cache_mode_keeps_restored_originals_when_history_diverges() -> None:+ """A rewritten history head has no safe replay: the restored originals go out."""+ previous_original = [{"role": "user", "content": "original prefix"}]+ previous_forwarded = [{"role": "user", "content": "comp"}]+ current = [+ {"role": "user", "content": "client rewrote the head"},+ {"role": "user", "content": "new suffix"},+ ]++ forwarded = _forward_cache_mode_chat(+ _FakePrefixTracker(1, previous_original, previous_forwarded),+ current,+ mutate_index=0,+ )++ assert forwarded == current+++def test_openai_cache_mode_block_append_replay_survives_the_restore() -> None:+ """Blocks appended inside the last message: forwarded blocks, then new ones."""+ previous_original = [{"role": "user", "content": [{"type": "text", "text": "block one " * 30}]}]+ previous_forwarded = [{"role": "user", "content": [{"type": "text", "text": "[compacted]"}]}]+ current = [+ {+ "role": "user",+ "content": [+ {"type": "text", "text": "block one " * 30},+ {"type": "text", "text": "block two"},+ ],+ }+ ]++ forwarded = _forward_cache_mode_chat(+ _FakePrefixTracker(1, previous_original, previous_forwarded), current+ )++ assert forwarded == [+ {+ "role": "user",+ "content": [+ {"type": "text", "text": "[compacted]"},+ {"type": "text", "text": "block two"},+ ],+ }+ ]+++def _compress_unfrozen_tool_results(**kwargs):+ """Fake pipeline that, like the real one, only rewrites past the frozen count."""+ frozen = kwargs.get("frozen_message_count") or 0+ out = [+ {**message, "content": f"[compressed {message['tool_call_id']}]"}+ if i >= frozen and message.get("role") == "tool"+ else message+ for i, message in enumerate(kwargs["messages"])+ ]+ return SimpleNamespace(+ messages=out,+ transforms_applied=["fake:compressed"],+ timing={},+ tokens_before=2000,+ tokens_after=30,+ waste_signals=None,+ )+++def _provider_reply(request: httpx.Request) -> httpx.Response:+ usage = _chat_usage(prompt_tokens=4000, cached_tokens=2048)+ if json.loads(request.content).get("stream"):+ return httpx.Response(+ 200,+ content="".join(_sse_chunks(4000, 2048)).encode(),+ headers={"content-type": "text/event-stream"},+ )+ return httpx.Response(+ 200,+ json={+ "id": "chatcmpl_lifecycle",+ "choices": [+ {+ "index": 0,+ "message": {"role": "assistant", "content": "ok"},+ "finish_reason": "stop",+ }+ ],+ "usage": usage,+ },+ )++[email protected]("stream", [False, True], ids=["buffered", "stream"])+def test_openai_cache_mode_forwards_last_turns_bytes_unchanged(stream: bool) -> None:+ """Cache mode, real tracker, three turns: every turn re-forwards the previous+ turn's forwarded messages byte-for-byte, so the provider's prefix cache holds.++ Needs both halves: the tracker must record the client's originals (else the+ overlay cannot match the client prefix), and the frozen-prefix restore must+ not undo the replay (else the compressed tool results revert to raw bytes).+ """+ from headroom.cache.prefix_tracker import PrefixCacheTracker++ def tool_turn(n: int) -> list[dict]:+ call_id = f"call_{n}"+ return [+ {"role": "user", "content": f"turn {n}: run the build"},+ {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ {+ "id": call_id,+ "type": "function",+ "function": {"name": "bash", "arguments": "{}"},+ }+ ],+ },+ {+ "role": "tool",+ "tool_call_id": call_id,+ "content": f"build {n} log line module m0 status OK\n" * 200,+ },+ ]++ sent: list[list[dict]] = []++ def provider(request: httpx.Request) -> httpx.Response:+ sent.append(json.loads(request.content)["messages"])+ return _provider_reply(request)++ history: list[dict] = []+ with _make_proxy_client() as client:+ proxy = client.app.state.proxy+ proxy.config.optimize = True+ proxy.config.mode = "cache"+ _install_tracker(proxy, PrefixCacheTracker(provider="openai"))+ proxy.openai_pipeline.apply = _compress_unfrozen_tool_results+ proxy.http_client = httpx.AsyncClient(transport=httpx.MockTransport(provider))+ for n in (1, 2, 3):+ history += tool_turn(n)+ response = client.post(+ "/v1/chat/completions",+ headers={"authorization": "Bearer test-key"},+ json={"model": "gpt-4o-mini", "stream": stream, "messages": history},+ )+ assert response.status_code == 200+ response.read()+ history.append({"role": "assistant", "content": "ok"})++ assert len(sent) == 3+ assert sent[0][2]["content"] == "[compressed call_1]"+ assert sent[1][: len(sent[0])] == sent[0]+ assert sent[2][: len(sent[1])] == sent[1]