Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
"""Regression tests for OpenAI cache-mode stability in proxy mode."""
from __future__ import annotations
import copy
import json
import pathlib
import re
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
pytest.importorskip("fastapi")
from fastapi.testclient import TestClient
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import InMemoryMetricReader
from headroom.backends.base import BackendResponse
from headroom.observability import HeadroomOtelMetrics, reset_otel_metrics, set_otel_metrics
from headroom.proxy.server import ProxyConfig, create_app
from tests.test_observability_metrics import _collect_metrics
class _FakePrefixTracker:
def __init__(
self,
frozen_count: int,
previous_original: list[dict] | None = None,
previous_forwarded: list[dict] | None = None,
):
self._frozen_count = frozen_count
self._previous_original = previous_original or []
self._previous_forwarded = previous_forwarded or []
self.update_calls: list[dict] = []
def get_frozen_message_count(self) -> int:
return self._frozen_count
# Empty history → overlay_cached_prefix() is a no-op here, so these tests
# keep asserting the cache-freeze behavior they always have. The cross-turn
# overlay itself is exercised in test_cross_turn_cache_safety.py against the
# real tracker; these stubs just satisfy the handler's overlay call.
def get_last_original_messages(self): # noqa: ANN201
return copy.deepcopy(self._previous_original)
def get_last_forwarded_messages(self): # noqa: ANN201
return copy.deepcopy(self._previous_forwarded)
def update_from_response(self, **kwargs): # noqa: ANN003
self.update_calls.append(kwargs)
return None
def _make_proxy_client(**config_overrides) -> TestClient: # noqa: ANN003
config = ProxyConfig(
optimize=False,
cache_enabled=False,
rate_limit_enabled=False,
cost_tracking_enabled=False,
log_requests=False,
ccr_inject_tool=False,
ccr_handle_responses=False,
ccr_context_tracking=False,
image_optimize=False,
**config_overrides,
)
app = create_app(config)
return TestClient(app)
def test_openai_cache_mode_freezes_previous_turns() -> None:
captured = {}
with _make_proxy_client() as client:
proxy = client.app.state.proxy
proxy.config.optimize = True
proxy.config.mode = "cache"
fake_tracker = _FakePrefixTracker(frozen_count=0)
proxy.session_tracker_store.compute_session_id = lambda request, model, messages: (
"stable-session"
)
proxy.session_tracker_store.get_or_create = (
lambda session_id, provider, cache_ttl_seconds=None: fake_tracker
)
def _fake_apply(**kwargs):
captured["frozen_message_count"] = kwargs.get("frozen_message_count")
return SimpleNamespace(
messages=kwargs["messages"],
transforms_applied=[],
timing={},
tokens_before=60,
tokens_after=60,
waste_signals=None,
)
proxy.openai_pipeline.apply = _fake_apply
async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
return httpx.Response(
200,
json={
"id": "chatcmpl_1",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 60, "completion_tokens": 3, "total_tokens": 63},
},
)
proxy._retry_request = _fake_retry
response = client.post(
"/v1/chat/completions",
headers={"authorization": "Bearer test-key"},
json={
"model": "gpt-4o-mini",
"messages": [
{"role": "user", "content": "turn1"},
{"role": "assistant", "content": "turn1-assistant"},
{"role": "user", "content": "current turn"},
],
},
)
assert response.status_code == 200
assert captured["frozen_message_count"] == 2
def test_openai_handler_replays_nonempty_cached_prefix() -> None:
captured = {}
previous_original = [{"role": "user", "content": "original prefix"}]
previous_forwarded = [{"role": "user", "content": "comp"}]
fake_tracker = _FakePrefixTracker(0, previous_original, previous_forwarded)
with _make_proxy_client() as client:
proxy = client.app.state.proxy
proxy.session_tracker_store.compute_session_id = lambda request, model, messages: (
"stable-session"
)
proxy.session_tracker_store.resolve_tracker = lambda *args, **kwargs: fake_tracker
async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
captured["body"] = body
return httpx.Response(
200,
json={
"id": "chatcmpl_overlay",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 20, "completion_tokens": 3, "total_tokens": 23},
},
)
proxy._retry_request = _fake_retry
response = client.post(
"/v1/chat/completions",
headers={"authorization": "Bearer test-key"},
json={
"model": "gpt-4o-mini",
"messages": [
{"role": "user", "content": "original prefix"},
{"role": "user", "content": "new suffix"},
],
},
)
assert response.status_code == 200
assert captured["body"]["messages"] == [
previous_forwarded[0],
{"role": "user", "content": "new suffix"},
]
@pytest.mark.parametrize("tail_role", ["tool", "function"])
def test_openai_cache_mode_keeps_final_tool_observation_mutable(tail_role: str) -> None:
captured = {}
with _make_proxy_client() as client:
proxy = client.app.state.proxy
proxy.config.optimize = True
proxy.config.mode = "cache"
fake_tracker = _FakePrefixTracker(frozen_count=0)
proxy.session_tracker_store.compute_session_id = lambda request, model, messages: (
"stable-session"
)
proxy.session_tracker_store.get_or_create = (
lambda session_id, provider, cache_ttl_seconds=None: fake_tracker
)
def _fake_apply(**kwargs):
captured.setdefault("calls", []).append(
{
"frozen_message_count": kwargs.get("frozen_message_count"),
"roles": [msg.get("role") for msg in kwargs["messages"]],
"mode": proxy.config.mode,
}
)
return SimpleNamespace(
messages=kwargs["messages"],
transforms_applied=["test:compress-tail"],
timing={},
tokens_before=120,
tokens_after=80,
waste_signals=None,
)
proxy.openai_pipeline.apply = _fake_apply
async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
return httpx.Response(
200,
json={
"id": "chatcmpl_tool_tail",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 80, "completion_tokens": 3, "total_tokens": 83},
},
)
proxy._retry_request = _fake_retry
tail = {
"role": tail_role,
"content": "large command observation " * 200,
}
if tail_role == "tool":
tail["tool_call_id"] = "call_1"
else:
tail["name"] = "bash"
response = client.post(
"/v1/chat/completions",
headers={"authorization": "Bearer test-key"},
json={
"model": "gpt-4o-mini",
"messages": [
{"role": "user", "content": "turn1"},
{"role": "assistant", "content": "run command"},
tail,
],
},
)
assert response.status_code == 200
assert any(call["frozen_message_count"] == 2 for call in captured["calls"]), captured[
"calls"
]
def test_openai_cache_mode_restores_mutated_frozen_prefix() -> None:
captured = {}
with _make_proxy_client() as client:
proxy = client.app.state.proxy
proxy.config.optimize = True
proxy.config.mode = "cache"
fake_tracker = _FakePrefixTracker(frozen_count=0)
proxy.session_tracker_store.compute_session_id = lambda request, model, messages: (
"stable-session"
)
proxy.session_tracker_store.get_or_create = (
lambda session_id, provider, cache_ttl_seconds=None: fake_tracker
)
original_messages = [
{"role": "user", "content": "turn1"},
{"role": "assistant", "content": "turn1-assistant"},
{"role": "user", "content": "current turn"},
]
def _fake_apply(**kwargs):
mutated = list(kwargs["messages"])
mutated[0] = {**mutated[0], "content": "MUTATED_PREFIX"}
return SimpleNamespace(
messages=mutated,
transforms_applied=["fake:mutated"],
timing={},
tokens_before=70,
tokens_after=65,
waste_signals=None,
)
proxy.openai_pipeline.apply = _fake_apply
async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
captured["body"] = body
return httpx.Response(
200,
json={
"id": "chatcmpl_2",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 65, "completion_tokens": 3, "total_tokens": 68},
},
)
proxy._retry_request = _fake_retry
response = client.post(
"/v1/chat/completions",
headers={"authorization": "Bearer test-key"},
json={
"model": "gpt-4o-mini",
"messages": original_messages,
},
)
assert response.status_code == 200
sent_messages = captured["body"]["messages"]
assert sent_messages[0] == original_messages[0]
assert sent_messages[1] == original_messages[1]
def test_openai_chat_completions_compacts_tools_when_profile_enabled() -> None:
captured = {}
with _make_proxy_client() as client:
proxy = client.app.state.proxy
proxy.config.optimize = True
proxy.config.mode = "token"
proxy.config.savings_profile = "agent-90"
def _fake_apply(**kwargs): # noqa: ANN003
return SimpleNamespace(
messages=kwargs["messages"],
transforms_applied=[],
timing={},
tokens_before=10,
tokens_after=10,
waste_signals=None,
)
proxy.openai_pipeline.apply = _fake_apply
async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
captured["body"] = body
return httpx.Response(
200,
json={
"id": "chatcmpl_tools",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 500, "completion_tokens": 3, "total_tokens": 503},
},
)
proxy._retry_request = _fake_retry
verbose_schema_note = "schema annotation repeated for opencode tool definitions " * 50
tools = [
{
"type": "function",
"function": {
"name": "read_file",
"description": "read file helper",
"parameters": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "ReadFileParameters",
"type": "object",
"properties": {
"path": {
"type": "string",
"title": "Path",
"description": verbose_schema_note,
"examples": [verbose_schema_note],
}
},
"required": ["path"],
},
},
}
]
response = client.post(
"/v1/chat/completions",
headers={"authorization": "Bearer test-key"},
json={
"model": "gpt-4o-mini",
"messages": [{"role": "user", "content": "inspect this file"}],
"tools": tools,
},
)
assert response.status_code == 200
assert "openai:chat:tool_schema_compaction" in response.headers["x-headroom-transforms"]
assert int(response.headers["x-headroom-tokens-saved"]) > 0
sent_params = captured["body"]["tools"][0]["function"]["parameters"]
assert "$schema" not in sent_params
assert "title" not in sent_params
assert "title" not in sent_params["properties"]["path"]
assert "examples" not in sent_params["properties"]["path"]
def test_openai_handler_replays_the_provider_confirmed_prefix_even_when_it_inflates() -> None:
"""#3379's twin on the OpenAI path: inside the provider-confirmed prefix the
replay source is exactly what OpenAI hashed, so a byte-larger replay must
still go out. Declining it forwards recompressed history and busts the
prompt cache from the first changed byte (gpt-5: $1.25/M vs $0.125/M
cached), which is strictly worse than replaying at the cache-read rate."""
captured = {}
# Same client original both turns, but last turn's forwarded form is LARGER
# than what the pipeline produces now -- i.e. the replay inflates and the
# size bound alone would decline it.
previous_original = [{"role": "user", "content": "original prefix"}]
previous_forwarded = [{"role": "user", "content": "previously forwarded " * 20}]
fake_tracker = _FakePrefixTracker(1, previous_original, previous_forwarded)
with _make_proxy_client() as client:
proxy = client.app.state.proxy
proxy.session_tracker_store.compute_session_id = lambda request, model, messages: (
"stable-session"
)
proxy.session_tracker_store.resolve_tracker = lambda *args, **kwargs: fake_tracker
async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
captured["body"] = body
return httpx.Response(
200,
json={
"id": "chatcmpl_confirmed_floor",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 20, "completion_tokens": 3, "total_tokens": 23},
},
)
proxy._retry_request = _fake_retry
response = client.post(
"/v1/chat/completions",
headers={"authorization": "Bearer test-key"},
json={
"model": "gpt-4o-mini",
"messages": [
{"role": "user", "content": "original prefix"},
{"role": "user", "content": "new suffix"},
],
},
)
assert response.status_code == 200
assert captured["body"]["messages"][0] == previous_forwarded[0]
assert captured["body"]["messages"][1] == {"role": "user", "content": "new suffix"}
def _tool_history() -> list[dict]:
"""A history whose big tool result the pipeline can compress."""
return [
{"role": "user", "content": "run the build and show me the log"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "bash", "arguments": "{}"},
}
],
},
{
"role": "tool",
"tool_call_id": "call_1",
"content": "build log line 0001 module m0 status OK\n" * 200,
},
]
def _compress_first_tool_result(**kwargs):
"""Fake pipeline: compress the big tool result so forwarded ≠ client bytes."""
mutated = [dict(message) for message in kwargs["messages"]]
for i, message in enumerate(mutated):
if message.get("role") == "tool":
mutated[i] = {**message, "content": "[compressed build log]"}
break
return SimpleNamespace(
messages=mutated,
transforms_applied=["fake:mutated"],
timing={},
tokens_before=2000,
tokens_after=30,
waste_signals=None,
)
def _install_tracker(proxy, tracker) -> None:
proxy.session_tracker_store.compute_session_id = lambda request, model, messages: (
"stable-session"
)
proxy.session_tracker_store.get_or_create = (
lambda session_id, provider, cache_ttl_seconds=None: tracker
)
proxy.session_tracker_store.resolve_tracker = lambda *args, **kwargs: tracker
def _chat_usage(prompt_tokens: int, cached_tokens: int) -> dict:
return {
"prompt_tokens": prompt_tokens,
"completion_tokens": 3,
"total_tokens": prompt_tokens + 3,
"prompt_tokens_details": {"cached_tokens": cached_tokens},
}
def test_openai_streaming_chat_hands_client_originals_to_prefix_tracker() -> None:
"""Direct streaming chat must give the stream finalizer the client's originals.
Without them the prefix tracker stores the forwarded (compressed) messages
as "original", so next turn's overlay never matches the client prefix and
cannot replay the cached bytes.
"""
captured = {}
client_messages = _tool_history() + [{"role": "user", "content": "current turn"}]
with _make_proxy_client() as client:
proxy = client.app.state.proxy
proxy.config.optimize = True
proxy.config.mode = "cache"
_install_tracker(proxy, _FakePrefixTracker(frozen_count=0))
proxy.openai_pipeline.apply = _compress_first_tool_result
async def _fake_stream_response(*args, **kwargs): # noqa: ANN002, ANN003
captured.update(kwargs)
return httpx.Response(200, text="data: [DONE]\n\n")
proxy._stream_response = _fake_stream_response
client.post(
"/v1/chat/completions",
headers={"authorization": "Bearer test-key"},
json={"model": "gpt-4o-mini", "stream": True, "messages": client_messages},
)
assert captured.get("original_messages") == client_messages
def test_openai_buffered_chat_hands_client_originals_to_prefix_tracker() -> None:
"""Direct buffered chat must record the client's originals, not its own output.
Token mode isolates the recording behavior: in cache mode the frozen-prefix
restore reverts prefix bytes before they are forwarded, which would make the
forwarded list coincidentally equal the originals. The originals plumbing
under test here is mode-independent.
"""
client_messages = _tool_history() + [{"role": "user", "content": "current turn"}]
with _make_proxy_client() as client:
proxy = client.app.state.proxy
proxy.config.optimize = True
proxy.config.mode = "token"
tracker = _FakePrefixTracker(frozen_count=0)
_install_tracker(proxy, tracker)
proxy.openai_pipeline.apply = _compress_first_tool_result
async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
return httpx.Response(
200,
json={
"id": "chatcmpl_originals_buffered",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop",
}
],
"usage": _chat_usage(30, 0),
},
)
proxy._retry_request = _fake_retry
response = client.post(
"/v1/chat/completions",
headers={"authorization": "Bearer test-key"},
json={"model": "gpt-4o-mini", "messages": client_messages},
)
assert response.status_code == 200
assert len(tracker.update_calls) == 1
call = tracker.update_calls[0]
# The tracker must store what the client SENT as "original", and the
# compressed form separately as what was forwarded.
assert call["original_messages"] == client_messages
assert call["messages"][2]["content"] == "[compressed build log]"
def _mock_openai_backend_response() -> dict:
return {
"id": "chatcmpl-backend-originals",
"object": "chat.completion",
"model": "gpt-4o-mini",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop",
}
],
"usage": _chat_usage(30, 0),
}
def _sse_chunks(prompt_tokens: int, cached_tokens: int) -> list[str]:
return [
'data: {"id":"c1","object":"chat.completion.chunk",'
'"choices":[{"index":0,"delta":{"role":"assistant","content":"ok"}}]}\n\n',
'data: {"id":"c1","object":"chat.completion.chunk",'
'"choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n',
'data: {"id":"c1","object":"chat.completion.chunk","choices":[],'
f'"usage":{{"prompt_tokens":{prompt_tokens},"completion_tokens":3,'
f'"total_tokens":{prompt_tokens + 3},'
f'"prompt_tokens_details":{{"cached_tokens":{cached_tokens}}}}}}}\n\n',
"data: [DONE]\n\n",
]
def _mock_streaming_backend(chunks: list[str]) -> MagicMock:
async def fake_stream(body, headers): # noqa: ANN001
for chunk in chunks:
yield chunk
backend = MagicMock()
backend.name = "anyllm-openai"
backend.stream_openai_message = fake_stream
return backend
def test_openai_backend_buffered_chat_hands_client_originals_to_prefix_tracker() -> None:
"""Backend-routed buffered chat must record the client's originals too."""
client_messages = _tool_history() + [{"role": "user", "content": "current turn"}]
backend = MagicMock()
backend.name = "anyllm-openai"
backend.send_openai_message = AsyncMock(
return_value=BackendResponse(
body=_mock_openai_backend_response(),
status_code=200,
headers={"content-type": "application/json"},
)
)
with patch("headroom.proxy.server.AnyLLMBackend", return_value=backend):
with _make_proxy_client(backend="anyllm", anyllm_provider="openai") as client:
proxy = client.app.state.proxy
proxy.config.optimize = True
proxy.config.mode = "token"
tracker = _FakePrefixTracker(frozen_count=0)
_install_tracker(proxy, tracker)
proxy.openai_pipeline.apply = _compress_first_tool_result
response = client.post(
"/v1/chat/completions",
headers={"authorization": "Bearer test-key"},
json={"model": "gpt-4o-mini", "stream": False, "messages": client_messages},
)
assert response.status_code == 200
assert backend.send_openai_message.await_count == 1
assert len(tracker.update_calls) == 1
call = tracker.update_calls[0]
assert call["original_messages"] == client_messages
assert call["messages"][2]["content"] == "[compressed build log]"
def test_openai_backend_streaming_chat_hands_client_originals_to_prefix_tracker() -> None:
"""Backend-routed streaming chat must record the client's originals too."""
client_messages = _tool_history() + [{"role": "user", "content": "current turn"}]
backend = _mock_streaming_backend(_sse_chunks(prompt_tokens=30, cached_tokens=0))
with patch("headroom.proxy.server.AnyLLMBackend", return_value=backend):
with _make_proxy_client(backend="anyllm", anyllm_provider="openai") as client:
proxy = client.app.state.proxy
proxy.config.optimize = True
proxy.config.mode = "token"
tracker = _FakePrefixTracker(frozen_count=0)
_install_tracker(proxy, tracker)
proxy.openai_pipeline.apply = _compress_first_tool_result
response = client.post(
"/v1/chat/completions",
headers={"authorization": "Bearer test-key"},
json={"model": "gpt-4o-mini", "stream": True, "messages": client_messages},
)
assert response.status_code == 200
assert "[DONE]" in response.text
assert len(tracker.update_calls) == 1
call = tracker.update_calls[0]
assert call["original_messages"] == client_messages
assert call["messages"][2]["content"] == "[compressed build log]"
def test_openai_chat_real_tracker_stores_client_originals_across_turns() -> None:
"""Real PrefixCacheTracker, two buffered turns (token mode).
Turn 2's stored "original" history must be the client's own messages
(including the new turn), not the compressed bytes that were forwarded —
otherwise next turn's overlay can never match the client prefix and
cannot replay the previously forwarded bytes.
"""
from headroom.cache.prefix_tracker import PrefixCacheTracker
turn1_messages = _tool_history()
turn2_messages = turn1_messages + [
{"role": "assistant", "content": "the build passed"},
{"role": "user", "content": "and now?"},
]
with _make_proxy_client() as client:
proxy = client.app.state.proxy
proxy.config.optimize = True
proxy.config.mode = "token"
tracker = PrefixCacheTracker(provider="openai")
_install_tracker(proxy, tracker)
proxy.openai_pipeline.apply = _compress_first_tool_result
async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
return httpx.Response(
200,
json={
"id": "chatcmpl_real_tracker",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop",
}
],
"usage": _chat_usage(prompt_tokens=5000, cached_tokens=0),
},
)
proxy._retry_request = _fake_retry
for messages in (turn1_messages, turn2_messages):
response = client.post(
"/v1/chat/completions",
headers={"authorization": "Bearer test-key"},
json={"model": "gpt-4o-mini", "messages": messages},
)
assert response.status_code == 200
# The forwarded history is the compressed tool result...
forwarded = tracker.get_last_forwarded_messages()
assert forwarded[2]["content"] == "[compressed build log]"
# ...but the recorded "original" history must be exactly what the client
# sent on turn 2, byte-for-byte, new turns included.
assert tracker.get_last_original_messages() == turn2_messages
def test_openai_cache_mode_keeps_replayed_prefix_over_frozen_restore() -> None:
"""Cache mode: the frozen-prefix restore must not undo a replayed prefix.
The overlay replays last turn's forwarded bytes, which the provider cached.
Restoring the frozen prefix to the raw client original afterwards
re-forwards different bytes and busts the cache from message 0.
"""
captured = {}
previous_original = [{"role": "user", "content": "original prefix"}]
previous_forwarded = [{"role": "user", "content": "comp"}]
fake_tracker = _FakePrefixTracker(1, previous_original, previous_forwarded)
with _make_proxy_client() as client:
proxy = client.app.state.proxy
proxy.config.optimize = True
proxy.config.mode = "cache"
proxy.session_tracker_store.compute_session_id = lambda request, model, messages: (
"stable-session"
)
proxy.session_tracker_store.get_or_create = (
lambda session_id, provider, cache_ttl_seconds=None: fake_tracker
)
proxy.session_tracker_store.resolve_tracker = lambda *args, **kwargs: fake_tracker
def _fake_apply(**kwargs):
return SimpleNamespace(
messages=kwargs["messages"],
transforms_applied=[],
timing={},
tokens_before=20,
tokens_after=20,
waste_signals=None,
)
proxy.openai_pipeline.apply = _fake_apply
async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
captured["body"] = body
return httpx.Response(
200,
json={
"id": "chatcmpl_replay_restore",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 20, "completion_tokens": 3, "total_tokens": 23},
},
)
proxy._retry_request = _fake_retry
response = client.post(
"/v1/chat/completions",
headers={"authorization": "Bearer test-key"},
json={
"model": "gpt-4o-mini",
"messages": [
{"role": "user", "content": "original prefix"},
{"role": "user", "content": "new suffix"},
],
},
)
assert response.status_code == 200
assert captured["body"]["messages"] == [
previous_forwarded[0],
{"role": "user", "content": "new suffix"},
]
def _forward_cache_mode_chat(
tracker: _FakePrefixTracker,
messages: list[dict],
*,
mutate_index: int | None = None,
) -> list[dict]:
"""Send one cache-mode chat turn and return the messages forwarded upstream.
``mutate_index`` makes the fake pipeline rewrite that message, standing in
for a transform that touched a frozen position."""
captured: dict = {}
with _make_proxy_client() as client:
proxy = client.app.state.proxy
proxy.config.optimize = True
proxy.config.mode = "cache"
proxy.session_tracker_store.compute_session_id = lambda request, model, messages: (
"stable-session"
)
proxy.session_tracker_store.get_or_create = (
lambda session_id, provider, cache_ttl_seconds=None: tracker
)
proxy.session_tracker_store.resolve_tracker = lambda *args, **kwargs: tracker
def _fake_apply(**kwargs):
out = list(kwargs["messages"])
if mutate_index is not None:
out[mutate_index] = {**out[mutate_index], "content": "MUTATED_BY_PIPELINE"}
return SimpleNamespace(
messages=out,
transforms_applied=[],
timing={},
tokens_before=20,
tokens_after=20,
waste_signals=None,
)
proxy.openai_pipeline.apply = _fake_apply
async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
captured["body"] = body
return httpx.Response(
200,
json={
"id": "chatcmpl_restore_replay",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 20, "completion_tokens": 3, "total_tokens": 23},
},
)
proxy._retry_request = _fake_retry
response = client.post(
"/v1/chat/completions",
headers={"authorization": "Bearer test-key"},
json={"model": "gpt-4o-mini", "messages": messages},
)
assert response.status_code == 200
return captured["body"]["messages"]
def test_openai_cache_mode_replays_only_up_to_the_first_changed_message() -> None:
"""Replay stops where the client's history diverges. Past that point the
frozen messages are restored to the client's bytes, so a pipeline rewrite
of a frozen message never reaches the provider, even on a replayed turn."""
previous_original = [
{"role": "user", "content": "first original " * 20},
{"role": "user", "content": "second original " * 20},
]
previous_forwarded = [
{"role": "user", "content": "first comp"},
{"role": "user", "content": "second comp"},
]
current = [
previous_original[0],
{"role": "user", "content": "second edited by the client"},
{"role": "user", "content": "new suffix"},
]
forwarded = _forward_cache_mode_chat(
_FakePrefixTracker(2, previous_original, previous_forwarded),
current,
mutate_index=1,
)
assert forwarded == [previous_forwarded[0], current[1], current[2]]
def test_openai_cache_mode_keeps_restored_originals_when_history_diverges() -> None:
"""A rewritten history head has no safe replay: the restored originals go out."""
previous_original = [{"role": "user", "content": "original prefix"}]
previous_forwarded = [{"role": "user", "content": "comp"}]
current = [
{"role": "user", "content": "client rewrote the head"},
{"role": "user", "content": "new suffix"},
]
forwarded = _forward_cache_mode_chat(
_FakePrefixTracker(1, previous_original, previous_forwarded),
current,
mutate_index=0,
)
assert forwarded == current
def test_openai_cache_mode_block_append_replay_survives_the_restore() -> None:
"""Blocks appended inside the last message: forwarded blocks, then new ones."""
previous_original = [{"role": "user", "content": [{"type": "text", "text": "block one " * 30}]}]
previous_forwarded = [{"role": "user", "content": [{"type": "text", "text": "[compacted]"}]}]
current = [
{
"role": "user",
"content": [
{"type": "text", "text": "block one " * 30},
{"type": "text", "text": "block two"},
],
}
]
forwarded = _forward_cache_mode_chat(
_FakePrefixTracker(1, previous_original, previous_forwarded), current
)
assert forwarded == [
{
"role": "user",
"content": [
{"type": "text", "text": "[compacted]"},
{"type": "text", "text": "block two"},
],
}
]
def _compress_unfrozen_tool_results(**kwargs):
"""Fake pipeline that, like the real one, only rewrites past the frozen count."""
frozen = kwargs.get("frozen_message_count") or 0
out = [
{**message, "content": f"[compressed {message['tool_call_id']}]"}
if i >= frozen and message.get("role") == "tool"
else message
for i, message in enumerate(kwargs["messages"])
]
return SimpleNamespace(
messages=out,
transforms_applied=["fake:compressed"],
timing={},
tokens_before=2000,
tokens_after=30,
waste_signals=None,
)
def _provider_reply(request: httpx.Request) -> httpx.Response:
usage = _chat_usage(prompt_tokens=4000, cached_tokens=2048)
if json.loads(request.content).get("stream"):
return httpx.Response(
200,
content="".join(_sse_chunks(4000, 2048)).encode(),
headers={"content-type": "text/event-stream"},
)
return httpx.Response(
200,
json={
"id": "chatcmpl_lifecycle",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop",
}
],
"usage": usage,
},
)
@pytest.mark.parametrize("stream", [False, True], ids=["buffered", "stream"])
def test_openai_cache_mode_forwards_last_turns_bytes_unchanged(stream: bool) -> None:
"""Cache mode, real tracker, three turns: every turn re-forwards the previous
turn's forwarded messages byte-for-byte, so the provider's prefix cache holds.
Needs both halves: the tracker must record the client's originals (else the
overlay cannot match the client prefix), and the frozen-prefix restore must
not undo the replay (else the compressed tool results revert to raw bytes).
"""
from headroom.cache.prefix_tracker import PrefixCacheTracker
def tool_turn(n: int) -> list[dict]:
call_id = f"call_{n}"
return [
{"role": "user", "content": f"turn {n}: run the build"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": call_id,
"type": "function",
"function": {"name": "bash", "arguments": "{}"},
}
],
},
{
"role": "tool",
"tool_call_id": call_id,
"content": f"build {n} log line module m0 status OK\n" * 200,
},
]
sent: list[list[dict]] = []
def provider(request: httpx.Request) -> httpx.Response:
sent.append(json.loads(request.content)["messages"])
return _provider_reply(request)
history: list[dict] = []
with _make_proxy_client() as client:
proxy = client.app.state.proxy
proxy.config.optimize = True
proxy.config.mode = "cache"
_install_tracker(proxy, PrefixCacheTracker(provider="openai"))
proxy.openai_pipeline.apply = _compress_unfrozen_tool_results
proxy.http_client = httpx.AsyncClient(transport=httpx.MockTransport(provider))
for n in (1, 2, 3):
history += tool_turn(n)
response = client.post(
"/v1/chat/completions",
headers={"authorization": "Bearer test-key"},
json={"model": "gpt-4o-mini", "stream": stream, "messages": history},
)
assert response.status_code == 200
response.read()
history.append({"role": "assistant", "content": "ok"})
assert len(sent) == 3
assert sent[0][2]["content"] == "[compressed call_1]"
assert sent[1][: len(sent[0])] == sent[0]
assert sent[2][: len(sent[1])] == sent[1]
def test_openai_chat_custom_base_reports_zai_as_outcome_provider(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""An OpenAI-compatible ``x-headroom-base-url`` upstream is not ``openai``."""
# Allowlist the host so the SSRF guard does not need live DNS.
monkeypatch.setenv("HEADROOM_ALLOWED_BASE_URLS", "api.z.ai")
captured = {}
with _make_proxy_client() as client:
proxy = client.app.state.proxy
async def _fake_stream_response(*args, **kwargs): # noqa: ANN002, ANN003
captured.update(kwargs)
return httpx.Response(200, text="data: [DONE]\n\n")
proxy._stream_response = _fake_stream_response
client.post(
"/v1/chat/completions",
headers={
"authorization": "Bearer test-key",
"x-headroom-base-url": "https://api.z.ai/api/coding/paas/v4",
},
json={
"model": "glm-5",
"stream": True,
"messages": [{"role": "user", "content": "hi"}],
},
)
assert captured.get("outcome_provider") == "zai"
def test_openai_chat_outcome_provider_uses_the_fixed_taxonomy(
monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path
) -> None:
"""Buffered chat labels each custom base from the fixed set, never the host."""
# Keep the SSRF guard out of the way: these hosts are deliberately
# unresolvable, and a DNS miss makes the handler ignore the override.
monkeypatch.setattr("headroom.proxy.handlers.openai.is_safe_upstream_url", lambda url: True)
# Keep the lifetime savings store out of the user's real on-disk state.
monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(tmp_path / "savings.json"))
with _make_proxy_client() as client:
proxy = client.app.state.proxy
async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001, ANN002, ANN003
return httpx.Response(
200,
json={
"id": "chatcmpl_1",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11},
},
)
proxy._retry_request = _fake_retry
def _chat(base: str | None) -> None:
headers = {"authorization": "Bearer test-key"}
if base is not None:
headers["x-headroom-base-url"] = base
response = client.post(
"/v1/chat/completions",
headers=headers,
json={"model": "glm-5", "messages": [{"role": "user", "content": "hi"}]},
)
assert response.status_code == 200
_chat("https://api.meta.ai/v1")
_chat("https://api.meta.ai/v1/sub")
_chat("https://llm.example.internal/v1")
# No header: the plain OpenAI path keeps its own label (control).
_chat(None)
_chat("https://api.x.ai")
_chat("https://api.x.ai.")
# Grok CLI on the default OpenAI target is routed to api.x.ai without
# a base-url header (#2693); it is still xAI traffic, not OpenAI's.
response = client.post(
"/v1/chat/completions",
headers={"authorization": "Bearer test-key", "user-agent": "grok-shell/0.2.117"},
json={"model": "grok-4", "messages": [{"role": "user", "content": "hi"}]},
)
assert response.status_code == 200
by_provider = proxy.metrics.requests_by_provider
assert by_provider["meta"] == 2
assert by_provider["custom"] == 1
assert by_provider["openai"] == 1
assert by_provider["xai"] == 3
def test_openai_chat_custom_base_flood_cannot_grow_the_provider_set(
monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path
) -> None:
"""Regression for the #3759 review: a host flood stays inside the fixed set.
The old code derived the outcome provider from the request-controlled
``x-headroom-base-url`` hostname, so one fresh hostname per request grew
the provider stores and exported series without bound. 980 distinct flood
hosts plus 20 z.ai requests cover success, upstream failure, local 429 and
upstream 429, and every sink must count exactly ``zai`` and ``custom``.
"""
fixed_providers = {"openai", "zen", "zai", "meta", "custom"}
zai_base = "https://api.z.ai/api/coding/paas/v4"
limited_auth = "Bearer rate-limited"
# The flood hosts are unresolvable on purpose; without this stub the SSRF
# guard would drop the override and relabel the flood as "openai".
monkeypatch.setattr("headroom.proxy.handlers.openai.is_safe_upstream_url", lambda url: True)
# Bucket by the raw credential so the stub limiter below can refuse just
# ``limited_auth``; the identity rule itself is covered by its own tests.
monkeypatch.setattr(
"headroom.proxy.handlers.openai.rate_limit_identity",
lambda request, headers: headers.get("authorization", ""),
)
# Isolated sinks: a fresh lifetime store and workspace, so the exact
# counts below are this test's traffic and nothing reaches ~/.headroom.
monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(tmp_path / "savings.json"))
monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(tmp_path / "workspace"))
reader = InMemoryMetricReader()
set_otel_metrics(HeadroomOtelMetrics(meter_provider=MeterProvider(metric_readers=[reader])))
class _LimitOneKey:
"""Headroom's own limiter, refusing only ``limited_auth``."""
async def check_request(self, key: str = "default") -> tuple[bool, float]:
return key != limited_auth, 1.0
async def check_tokens(self, key: str, tokens: int) -> tuple[bool, float]:
return True, 0.0
async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001, ANN002, ANN003
if "flood-down-" in url:
raise httpx.ConnectError("flood host down")
if "flood-upstream429-" in url:
return httpx.Response(429, json={"error": {"message": "slow down"}})
return httpx.Response(
200,
json={
"id": "chatcmpl_flood",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6},
},
)
expected_status = {"ok": 200, "down": 502, "limited": 429, "upstream429": 429}
sent: dict[tuple[str, str], int] = {}
flood_hosts: set[str] = set()
try:
with _make_proxy_client() as client:
proxy = client.app.state.proxy
proxy._retry_request = _fake_retry
proxy.rate_limiter = _LimitOneKey()
for i in range(1000):
if i % 50 == 0:
provider, kind = "zai", ("ok" if i % 100 == 0 else "limited")
base = zai_base
else:
provider, kind = "custom", ("limited", "upstream429", "down", "ok")[i % 4]
host = f"flood-{kind}-{i}.example.invalid"
flood_hosts.add(host)
base = f"https://{host}/v1"
sent[provider, kind] = sent.get((provider, kind), 0) + 1
auth = limited_auth if kind == "limited" else "Bearer test-key"
response = client.post(
"/v1/chat/completions",
headers={"authorization": auth, "x-headroom-base-url": base},
json={"model": "glm-5", "messages": [{"role": "user", "content": f"turn {i}"}]},
)
assert response.status_code == expected_status[kind], (i, base)
# The traffic this test sends, pinned so the claim stays exact.
assert len(flood_hosts) == 980
assert sent == {
("zai", "ok"): 10,
("zai", "limited"): 10,
("custom", "ok"): 250,
("custom", "down"): 240,
("custom", "limited"): 240,
("custom", "upstream429"): 250,
}
metrics = proxy.metrics
# In-memory stores: exact counts; "unknown" is the seeded zero bucket.
assert dict(metrics.requests_by_provider) == {"zai": 10, "custom": 250}
assert dict(metrics.requests_failed_by_provider) == {"unknown": 0, "custom": 240}
assert metrics.requests_rate_limited_by_source == {"headroom": 250, "upstream": 250}
# Lifetime store, isolated under tmp_path.
lifetime = metrics.savings_tracker.lifetime_response()["requests"]
assert lifetime["by_provider"] == {"zai": 10, "custom": 250}
assert lifetime["failed_by_provider"] == {"custom": 240}
assert lifetime["rate_limited_by_source"] == {"headroom": 250, "upstream": 250}
# Prometheus export: every provider series is from the fixed set.
# /metrics is an operator route: read it as the local operator does.
operator = TestClient(
client.app, base_url="http://127.0.0.1", client=("127.0.0.1", 12345)
)
export = operator.get("/metrics").text
assert set(re.findall(r'provider="([^"]+)"', export)) <= fixed_providers | {"unknown"}
assert 'headroom_requests_by_provider{provider="zai"} 10' in export
assert 'headroom_requests_by_provider{provider="custom"} 250' in export
# OTel: every proxy data point carries a fixed provider attribute, and
# z.ai stays its own series on the success and local-429 paths.
otel = _collect_metrics(reader)
otel_points: dict[tuple[str, str, str], float] = {}
for name, metric in otel.items():
if not name.startswith("headroom.proxy."):
continue
for point in metric.data.data_points:
label = point.attributes.get("provider")
assert label in fixed_providers, (name, label)
if name.startswith("headroom.proxy.requests"):
key = (name, label, point.attributes.get("source", ""))
otel_points[key] = otel_points.get(key, 0) + point.value
assert otel_points == {
("headroom.proxy.requests", "zai", ""): 10,
("headroom.proxy.requests", "custom", ""): 250,
("headroom.proxy.requests.failed", "custom", ""): 240,
("headroom.proxy.requests.rate_limited", "zai", "headroom"): 10,
("headroom.proxy.requests.rate_limited", "custom", "headroom"): 240,
("headroom.proxy.requests.rate_limited", "custom", "upstream"): 250,
}
finally:
reset_otel_metrics()