Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): read stream cache usage from message_delta; keep Fireworks sessions on one replica (#4174)
Tejas Chopra30b22df7fda6
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex 10d6b8ba7..742372ec7 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -19,6 +19,7 @@ from pathlib import Pathfrom typing import TYPE_CHECKING, Anyfrom urllib.parse import urlsplit+from headroom.proxy.session_affinity import session_affinity_headersfrom headroom.proxy.stage_timer import StageTimer, emit_stage_timings_logfrom headroom.proxy.tool_schema_savings_policy import without_deferral_flags@@ -3927,6 +3928,9 @@ class AnthropicHandlerMixin:request.url.path,request.url.query,)+ # Keep a conversation on one replica of providers that route on a+ # header (Fireworks), so each turn finds the previous turn's cache.+ headers.update(session_affinity_headers(url, session_id, headers))try:ccr_handler_config = getattr(self.ccr_response_handler, "config", None)diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.pyindex 8efcbb39a..d776bc7fb 100644--- a/headroom/proxy/handlers/streaming.py+++ b/headroom/proxy/handlers/streaming.py@@ -372,6 +372,27 @@ class StreamingMixin:delta_usage = _sse_dict(data.get("usage"))if delta_usage:usage_found["output_tokens"] = delta_usage.get("output_tokens", 0)+ # Usage here is cumulative. Some Anthropic-compatible+ # upstreams (Fireworks) send zeros in message_start and+ # the real input and cache counts only here; without+ # them the prefix tracker sees no cache and freezes+ # nothing.+ for key in (+ "input_tokens",+ "cache_read_input_tokens",+ "cache_creation_input_tokens",+ ):+ value = usage_int(delta_usage.get(key))+ if value > usage_int(usage_found.get(key)):+ usage_found[key] = value+ # Their TTL split too, or one-hour writes price as five-minute.+ delta_5m, delta_1h = self._extract_anthropic_cache_ttl_metrics(delta_usage)+ for key, value in (+ ("cache_creation_ephemeral_5m_input_tokens", delta_5m),+ ("cache_creation_ephemeral_1h_input_tokens", delta_1h),+ ):+ if value > usage_int(usage_found.get(key)):+ usage_found[key] = valueelif provider == "openai":chunk_usage = data.get("usage")diff --git a/headroom/proxy/session_affinity.py b/headroom/proxy/session_affinity.pynew file mode 100644index 000000000..870fcfa21--- /dev/null+++ b/headroom/proxy/session_affinity.py@@ -0,0 +1,30 @@+"""Session-affinity headers for hosted providers that route on them.++Some providers run many replicas, each with its own prompt cache, and pick one+per request. Without a hint, consecutive turns of one conversation can land on+different replicas and miss the cache the previous turn built. Providers that+route on a header get Headroom's session id in it, so a conversation stays on+one replica.++Only hosts listed here receive the header; every other upstream (Anthropic,+OpenAI, Bedrock, Vertex, ...) is left untouched. A header the client already+set wins.+"""++from __future__ import annotations++from collections.abc import Mapping+from urllib.parse import urlsplit++# Host -> header it routes on. Fireworks: docs.fireworks.ai prompt caching.+AFFINITY_HEADERS: dict[str, str] = {"api.fireworks.ai": "x-session-affinity"}+++def session_affinity_headers(+ url: str, session_id: str | None, headers: Mapping[str, str]+) -> dict[str, str]:+ """Headers to add for ``url``: the provider's affinity header, or nothing."""+ name = AFFINITY_HEADERS.get(urlsplit(url).hostname or "")+ if name is None or not session_id or any(k.lower() == name for k in headers):+ return {}+ return {name: session_id}diff --git a/tests/test_proxy_cache_ttl_metrics.py b/tests/test_proxy_cache_ttl_metrics.pyindex 006f6a989..f2dfc2207 100644--- a/tests/test_proxy_cache_ttl_metrics.py+++ b/tests/test_proxy_cache_ttl_metrics.py@@ -518,3 +518,66 @@ def test_prefix_cache_stats_prices_xai_cache_reads_with_grok_models(# No catalog cache rates: xAI's own fallback ratio, not Anthropic's 0.1.assert stats["by_provider"]["xai"]["savings_usd"] == round(1_000_000 * 3e-6 * (1 - 0.16), 4)+++def test_streaming_parser_reads_cache_usage_sent_only_in_message_delta() -> None:+ # Fireworks' Anthropic endpoint sends zeros in message_start and the real+ # input and cache counts in message_delta. The prefix tracker freezes history+ # by cache_read, so dropping them makes it treat a warm cache as cold.+ from headroom.proxy.server import HeadroomProxy, ProxyConfig++ proxy = HeadroomProxy(+ ProxyConfig(+ optimize=False,+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ log_requests=False,+ ccr_inject_tool=False,+ ccr_handle_responses=False,+ ccr_context_tracking=False,+ )+ )+ chunk = (+ b'data: {"type":"message_start","message":{"usage":{"input_tokens":0,'+ b'"output_tokens":0,"cache_read_input_tokens":0,"cache_creation_input_tokens":0}}}\n\n'+ b'data: {"type":"message_delta","delta":{"stop_reason":"tool_use"},"usage":'+ b'{"input_tokens":1,"output_tokens":5,"cache_read_input_tokens":2802,'+ b'"cache_creation_input_tokens":0}}\n\n'+ )+ usage = proxy._parse_sse_usage_from_buffer({"sse_buffer": bytearray(chunk)}, "anthropic")++ assert usage["cache_read_input_tokens"] == 2802+ assert usage["input_tokens"] == 1+ assert usage["output_tokens"] == 5+++def test_streaming_parser_reads_ttl_buckets_sent_only_in_message_delta() -> None:+ # One-hour cache writes reported only in message_delta must stay one-hour,+ # or the cost tracker prices them as five-minute writes.+ from headroom.proxy.server import HeadroomProxy, ProxyConfig++ proxy = HeadroomProxy(+ ProxyConfig(+ optimize=False,+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ log_requests=False,+ ccr_inject_tool=False,+ ccr_handle_responses=False,+ ccr_context_tracking=False,+ )+ )+ chunk = (+ b'data: {"type":"message_start","message":{"usage":{"input_tokens":0,'+ b'"cache_creation_input_tokens":0}}}\n\n'+ b'data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":'+ b'{"input_tokens":5,"output_tokens":2,"cache_creation_input_tokens":1000,'+ b'"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":1000}}}\n\n'+ )+ usage = proxy._parse_sse_usage_from_buffer({"sse_buffer": bytearray(chunk)}, "anthropic")++ assert usage["cache_creation_input_tokens"] == 1000+ assert usage["cache_creation_ephemeral_1h_input_tokens"] == 1000+ assert usage["cache_creation_ephemeral_5m_input_tokens"] == 0diff --git a/tests/test_session_affinity.py b/tests/test_session_affinity.pynew file mode 100644index 000000000..3d58e9bd1--- /dev/null+++ b/tests/test_session_affinity.py@@ -0,0 +1,72 @@+"""Session-affinity header: sent to providers that route on it, and nowhere else."""++from __future__ import annotations++import httpx+import pytest++pytest.importorskip("fastapi")++from fastapi.testclient import TestClient++from headroom.proxy.server import ProxyConfig, create_app++REPLY = {+ "id": "m1",+ "type": "message",+ "role": "assistant",+ "content": [{"type": "text", "text": "ok"}],+ "stop_reason": "end_turn",+ "usage": {"input_tokens": 3, "output_tokens": 1},+}+++def forwarded_headers(upstream: str, extra: dict[str, str] | None = None) -> list[dict]:+ config = ProxyConfig(+ optimize=False,+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ log_requests=False,+ ccr_inject_tool=False,+ ccr_handle_responses=False,+ ccr_context_tracking=False,+ anthropic_api_url=upstream,+ )+ sent: list[dict] = []++ async def fake_upstream(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001, ANN003+ sent.append({k.lower(): v for k, v in headers.items()})+ return httpx.Response(200, json=REPLY)++ with TestClient(create_app(config)) as client:+ client.app.state.proxy._retry_request = fake_upstream+ for _ in range(2):+ response = client.post(+ "/v1/messages",+ headers={"x-api-key": "k", "anthropic-version": "2023-06-01", **(extra or {})},+ json={+ "model": "kimi",+ "max_tokens": 5,+ "system": "You are a coding agent.",+ "messages": [{"role": "user", "content": "hi"}],+ },+ )+ assert response.status_code == 200+ return sent+++def test_fireworks_gets_one_stable_affinity_per_conversation() -> None:+ first, second = forwarded_headers("https://api.fireworks.ai/inference")+ assert first["x-session-affinity"]+ assert first["x-session-affinity"] == second["x-session-affinity"]++[email protected]("upstream", ["https://api.anthropic.com", "http://vllm.internal:8000"])+def test_other_upstreams_get_no_affinity_header(upstream: str) -> None:+ assert all("x-session-affinity" not in h for h in forwarded_headers(upstream))+++def test_client_affinity_header_is_kept() -> None:+ sent = forwarded_headers("https://api.fireworks.ai/inference", {"X-Session-Affinity": "mine"})+ assert all(h["x-session-affinity"] == "mine" for h in sent)