Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): read stream cache usage from message_delta; keep Fireworks sessions on one replica (#4174)

Tejas Chopra30b22df7fda6
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index 10d6b8ba7..742372ec7 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -19,6 +19,7 @@ from pathlib import Path
from typing import TYPE_CHECKING, Any
from urllib.parse import urlsplit
+from headroom.proxy.session_affinity import session_affinity_headers
from headroom.proxy.stage_timer import StageTimer, emit_stage_timings_log
from headroom.proxy.tool_schema_savings_policy import without_deferral_flags
@@ -3927,6 +3928,9 @@ class AnthropicHandlerMixin:
request.url.path,
request.url.query,
)
+ # Keep a conversation on one replica of providers that route on a
+ # header (Fireworks), so each turn finds the previous turn's cache.
+ headers.update(session_affinity_headers(url, session_id, headers))
try:
ccr_handler_config = getattr(self.ccr_response_handler, "config", None)
diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.py
index 8efcbb39a..d776bc7fb 100644
--- a/headroom/proxy/handlers/streaming.py
+++ b/headroom/proxy/handlers/streaming.py
@@ -372,6 +372,27 @@ class StreamingMixin:
delta_usage = _sse_dict(data.get("usage"))
if delta_usage:
usage_found["output_tokens"] = delta_usage.get("output_tokens", 0)
+ # Usage here is cumulative. Some Anthropic-compatible
+ # upstreams (Fireworks) send zeros in message_start and
+ # the real input and cache counts only here; without
+ # them the prefix tracker sees no cache and freezes
+ # nothing.
+ for key in (
+ "input_tokens",
+ "cache_read_input_tokens",
+ "cache_creation_input_tokens",
+ ):
+ value = usage_int(delta_usage.get(key))
+ if value > usage_int(usage_found.get(key)):
+ usage_found[key] = value
+ # Their TTL split too, or one-hour writes price as five-minute.
+ delta_5m, delta_1h = self._extract_anthropic_cache_ttl_metrics(delta_usage)
+ for key, value in (
+ ("cache_creation_ephemeral_5m_input_tokens", delta_5m),
+ ("cache_creation_ephemeral_1h_input_tokens", delta_1h),
+ ):
+ if value > usage_int(usage_found.get(key)):
+ usage_found[key] = value
elif provider == "openai":
chunk_usage = data.get("usage")
diff --git a/headroom/proxy/session_affinity.py b/headroom/proxy/session_affinity.py
new file mode 100644
index 000000000..870fcfa21
--- /dev/null
+++ b/headroom/proxy/session_affinity.py
@@ -0,0 +1,30 @@
+"""Session-affinity headers for hosted providers that route on them.
+
+Some providers run many replicas, each with its own prompt cache, and pick one
+per request. Without a hint, consecutive turns of one conversation can land on
+different replicas and miss the cache the previous turn built. Providers that
+route on a header get Headroom's session id in it, so a conversation stays on
+one replica.
+
+Only hosts listed here receive the header; every other upstream (Anthropic,
+OpenAI, Bedrock, Vertex, ...) is left untouched. A header the client already
+set wins.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from urllib.parse import urlsplit
+
+# Host -> header it routes on. Fireworks: docs.fireworks.ai prompt caching.
+AFFINITY_HEADERS: dict[str, str] = {"api.fireworks.ai": "x-session-affinity"}
+
+
+def session_affinity_headers(
+ url: str, session_id: str | None, headers: Mapping[str, str]
+) -> dict[str, str]:
+ """Headers to add for ``url``: the provider's affinity header, or nothing."""
+ name = AFFINITY_HEADERS.get(urlsplit(url).hostname or "")
+ if name is None or not session_id or any(k.lower() == name for k in headers):
+ return {}
+ return {name: session_id}
diff --git a/tests/test_proxy_cache_ttl_metrics.py b/tests/test_proxy_cache_ttl_metrics.py
index 006f6a989..f2dfc2207 100644
--- a/tests/test_proxy_cache_ttl_metrics.py
+++ b/tests/test_proxy_cache_ttl_metrics.py
@@ -518,3 +518,66 @@ def test_prefix_cache_stats_prices_xai_cache_reads_with_grok_models(
# No catalog cache rates: xAI's own fallback ratio, not Anthropic's 0.1.
assert stats["by_provider"]["xai"]["savings_usd"] == round(1_000_000 * 3e-6 * (1 - 0.16), 4)
+
+
+def test_streaming_parser_reads_cache_usage_sent_only_in_message_delta() -> None:
+ # Fireworks' Anthropic endpoint sends zeros in message_start and the real
+ # input and cache counts in message_delta. The prefix tracker freezes history
+ # by cache_read, so dropping them makes it treat a warm cache as cold.
+ from headroom.proxy.server import HeadroomProxy, ProxyConfig
+
+ proxy = HeadroomProxy(
+ ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ ccr_inject_tool=False,
+ ccr_handle_responses=False,
+ ccr_context_tracking=False,
+ )
+ )
+ chunk = (
+ b'data: {"type":"message_start","message":{"usage":{"input_tokens":0,'
+ b'"output_tokens":0,"cache_read_input_tokens":0,"cache_creation_input_tokens":0}}}\n\n'
+ b'data: {"type":"message_delta","delta":{"stop_reason":"tool_use"},"usage":'
+ b'{"input_tokens":1,"output_tokens":5,"cache_read_input_tokens":2802,'
+ b'"cache_creation_input_tokens":0}}\n\n'
+ )
+ usage = proxy._parse_sse_usage_from_buffer({"sse_buffer": bytearray(chunk)}, "anthropic")
+
+ assert usage["cache_read_input_tokens"] == 2802
+ assert usage["input_tokens"] == 1
+ assert usage["output_tokens"] == 5
+
+
+def test_streaming_parser_reads_ttl_buckets_sent_only_in_message_delta() -> None:
+ # One-hour cache writes reported only in message_delta must stay one-hour,
+ # or the cost tracker prices them as five-minute writes.
+ from headroom.proxy.server import HeadroomProxy, ProxyConfig
+
+ proxy = HeadroomProxy(
+ ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ ccr_inject_tool=False,
+ ccr_handle_responses=False,
+ ccr_context_tracking=False,
+ )
+ )
+ chunk = (
+ b'data: {"type":"message_start","message":{"usage":{"input_tokens":0,'
+ b'"cache_creation_input_tokens":0}}}\n\n'
+ b'data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":'
+ b'{"input_tokens":5,"output_tokens":2,"cache_creation_input_tokens":1000,'
+ b'"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":1000}}}\n\n'
+ )
+ usage = proxy._parse_sse_usage_from_buffer({"sse_buffer": bytearray(chunk)}, "anthropic")
+
+ assert usage["cache_creation_input_tokens"] == 1000
+ assert usage["cache_creation_ephemeral_1h_input_tokens"] == 1000
+ assert usage["cache_creation_ephemeral_5m_input_tokens"] == 0
diff --git a/tests/test_session_affinity.py b/tests/test_session_affinity.py
new file mode 100644
index 000000000..3d58e9bd1
--- /dev/null
+++ b/tests/test_session_affinity.py
@@ -0,0 +1,72 @@
+"""Session-affinity header: sent to providers that route on it, and nowhere else."""
+
+from __future__ import annotations
+
+import httpx
+import pytest
+
+pytest.importorskip("fastapi")
+
+from fastapi.testclient import TestClient
+
+from headroom.proxy.server import ProxyConfig, create_app
+
+REPLY = {
+ "id": "m1",
+ "type": "message",
+ "role": "assistant",
+ "content": [{"type": "text", "text": "ok"}],
+ "stop_reason": "end_turn",
+ "usage": {"input_tokens": 3, "output_tokens": 1},
+}
+
+
+def forwarded_headers(upstream: str, extra: dict[str, str] | None = None) -> list[dict]:
+ config = ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ ccr_inject_tool=False,
+ ccr_handle_responses=False,
+ ccr_context_tracking=False,
+ anthropic_api_url=upstream,
+ )
+ sent: list[dict] = []
+
+ async def fake_upstream(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001, ANN003
+ sent.append({k.lower(): v for k, v in headers.items()})
+ return httpx.Response(200, json=REPLY)
+
+ with TestClient(create_app(config)) as client:
+ client.app.state.proxy._retry_request = fake_upstream
+ for _ in range(2):
+ response = client.post(
+ "/v1/messages",
+ headers={"x-api-key": "k", "anthropic-version": "2023-06-01", **(extra or {})},
+ json={
+ "model": "kimi",
+ "max_tokens": 5,
+ "system": "You are a coding agent.",
+ "messages": [{"role": "user", "content": "hi"}],
+ },
+ )
+ assert response.status_code == 200
+ return sent
+
+
+def test_fireworks_gets_one_stable_affinity_per_conversation() -> None:
+ first, second = forwarded_headers("https://api.fireworks.ai/inference")
+ assert first["x-session-affinity"]
+ assert first["x-session-affinity"] == second["x-session-affinity"]
+
+
[email protected]("upstream", ["https://api.anthropic.com", "http://vllm.internal:8000"])
+def test_other_upstreams_get_no_affinity_header(upstream: str) -> None:
+ assert all("x-session-affinity" not in h for h in forwarded_headers(upstream))
+
+
+def test_client_affinity_header_is_kept() -> None:
+ sent = forwarded_headers("https://api.fireworks.ai/inference", {"X-Session-Affinity": "mine"})
+ assert all(h["x-session-affinity"] == "mine" for h in sent)