Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): record cache metrics for non-streaming backend paths (#1271)

Sujit Patil85804043ff1f
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index c3f38abec..0573dbf32 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -2397,12 +2397,16 @@ class AnthropicHandlerMixin:
)
except Exception:
attempted_input_tokens = original_tokens
- # Backend (Bedrock / Vertex) non-streaming.
- # Cache metrics aren't extracted from the backend
- # response here yet — that's a follow-up. The
- # funnel passes 0s for the cache fields, which
- # is the same observable behaviour as the
- # pre-refactor code (which also omitted them).
+
+ cr_tokens = usage.get("cache_read_input_tokens", 0)
+ cw_tokens = usage.get("cache_creation_input_tokens", 0)
+ cw_5m_tokens, cw_1h_tokens = self._extract_anthropic_cache_ttl_metrics(
+ usage
+ )
+ uncached_input_tokens = max(
+ 0, attempted_input_tokens - cr_tokens - cw_tokens
+ )
+
await self._record_request_outcome(
RequestOutcome(
request_id=request_id,
@@ -2413,6 +2417,11 @@ class AnthropicHandlerMixin:
output_tokens=output_tokens,
tokens_saved=tokens_saved,
attempted_input_tokens=attempted_input_tokens,
+ cache_read_tokens=cr_tokens,
+ cache_write_tokens=cw_tokens,
+ cache_write_5m_tokens=cw_5m_tokens,
+ cache_write_1h_tokens=cw_1h_tokens,
+ uncached_input_tokens=uncached_input_tokens,
total_latency_ms=total_latency,
overhead_ms=optimization_latency,
pipeline_timing=pipeline_timing,
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 8b8d3c328..e323aa6d1 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -3030,14 +3030,21 @@ class OpenAIHandlerMixin:
cache_read_tokens = prompt_details.get("cached_tokens", 0) or 0
# Bedrock reports cache creation directly. Only infer
- # when no explicit count is available.
+ # when no explicit count is available. Skip inference
+ # entirely when upstream omitted prompt_tokens.
if cache_creation_input_tokens > 0:
cache_write_tokens = cache_creation_input_tokens
- else:
+ elif "prompt_tokens" in usage:
cache_write_tokens = _infer_openai_cache_write_tokens(
total_input_tokens,
cache_read_tokens,
)
+ else:
+ cache_write_tokens = 0
+
+ uncached_input_tokens = max(
+ 0, total_input_tokens - cache_read_tokens - cache_write_tokens
+ )
openai_prefix_tracker.update_from_response(
cache_read_tokens=cache_read_tokens,
@@ -3055,6 +3062,9 @@ class OpenAIHandlerMixin:
output_tokens=output_tokens,
tokens_saved=tokens_saved,
attempted_input_tokens=total_input_tokens + tokens_saved,
+ cache_read_tokens=cache_read_tokens,
+ cache_write_tokens=cache_write_tokens,
+ uncached_input_tokens=uncached_input_tokens,
total_latency_ms=total_latency,
overhead_ms=optimization_latency,
pipeline_timing=pipeline_timing,
diff --git a/tests/test_backend_nonstreaming_cache_metrics.py b/tests/test_backend_nonstreaming_cache_metrics.py
new file mode 100644
index 000000000..9dbbb6c77
--- /dev/null
+++ b/tests/test_backend_nonstreaming_cache_metrics.py
@@ -0,0 +1,364 @@
+"""Cache-metric coverage for backend-routed **non-streaming** requests.
+
+Sibling of ``tests/test_backend_streaming_cache_metrics.py`` (issue #327).
+That file fixed the *streaming* backend paths so cache reads/writes reach the
+``PERF`` log line consumed by ``headroom perf``. The **non-streaming** backend
+paths were left behind — the same bug class on the parallel code path:
+
+* ``AnthropicHandlerMixin`` non-streaming backend branch
+ (``anthropic.py`` ``send_message`` path): reads ``usage`` from the backend
+ response body but extracts only ``output_tokens``. The accompanying comment
+ admits "Cache metrics aren't extracted from the backend response here yet —
+ that's a follow-up." So Bedrock / Vertex non-streaming traffic reported
+ ``cache_read=0 cache_write=0`` even though the response carried
+ ``cache_read_input_tokens`` / ``cache_creation_input_tokens``.
+
+* ``OpenAIHandlerMixin`` non-streaming backend branch
+ (``openai.py`` ``send_openai_message`` path): worse — cache fields ARE
+ extracted and fed to ``openai_prefix_tracker``, but never threaded into the
+ ``RequestOutcome``, so the funnel (Prometheus / cost tracker / RequestLog /
+ PERF) all see zeros.
+
+Both surface to the user as "Cache write: 0 tokens" in ``headroom perf``,
+identical to the streaming regression that motivated issue #327.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import Any
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+fastapi = pytest.importorskip("fastapi")
+httpx = pytest.importorskip("httpx")
+
+from fastapi.testclient import TestClient # noqa: E402
+
+from headroom.backends.base import BackendResponse # noqa: E402
+from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
+
+PERF_RE = re.compile(
+ r"\bcache_read=(?P<cr>\d+)\s+cache_write=(?P<cw>\d+)\s+cache_hit_pct=(?P<chp>\d+)"
+)
+
+
+def _find_perf_record(records: list[logging.LogRecord]) -> tuple[int, int, int]:
+ """Find the structured PERF log line and return (cache_read, cache_write, hit_pct)."""
+ for record in records:
+ msg = record.getMessage()
+ if " PERF " not in msg:
+ continue
+ m = PERF_RE.search(msg)
+ if m:
+ return int(m["cr"]), int(m["cw"]), int(m["chp"])
+ raise AssertionError(
+ "No PERF log line with cache_read/cache_write/cache_hit_pct found. "
+ f"Captured {len(records)} records.\n" + "\n".join(r.getMessage() for r in records[-15:])
+ )
+
+
+class _ListHandler(logging.Handler):
+ """Tiny direct handler that survives the proxy disabling propagation.
+
+ ``caplog`` attaches to root; ``headroom.proxy.helpers._setup_file_logging``
+ flips ``logging.getLogger("headroom").propagate = False`` once a proxy
+ instance is constructed in the test, after which root-attached handlers
+ stop receiving headroom-namespaced records. Attaching directly to
+ ``headroom.proxy`` sidesteps that.
+ """
+
+ def __init__(self) -> None:
+ super().__init__(level=logging.INFO)
+ self.records: list[logging.LogRecord] = []
+
+ def emit(self, record: logging.LogRecord) -> None: # noqa: D401
+ self.records.append(record)
+
+
+def _attach_proxy_log_capture():
+ handler = _ListHandler()
+ target = logging.getLogger("headroom.proxy")
+ target.addHandler(handler)
+ prior_level = target.level
+ target.setLevel(logging.INFO)
+ return handler, target, prior_level
+
+
+def _detach_proxy_log_capture(handler, target, prior_level) -> None:
+ target.removeHandler(handler)
+ target.setLevel(prior_level)
+
+
+def _make_anthropic_backend(body: dict[str, Any]) -> MagicMock:
+ """Build a mock backend whose ``send_message`` returns ``body`` (Anthropic shape).
+
+ The body is the Anthropic Messages non-streaming response, including a
+ ``usage`` block that carries cache counters — exactly what Bedrock /
+ Vertex / LiteLLM(anthropic) return for a cached turn.
+ """
+
+ async def fake_send(body_: dict, headers: dict) -> BackendResponse:
+ return BackendResponse(body=body, status_code=200)
+
+ # A streaming coroutine is never exercised on the non-streaming path, but
+ # the server's backend-factory calls ``map_model_id`` / ``supports_model``
+ # during wiring, so provide no-op mocks for those too.
+ mock = MagicMock()
+ mock.name = "anyllm-anthropic"
+ mock.send_message = fake_send
+ mock.map_model_id = MagicMock(return_value="claude-3-5-sonnet-20241022")
+ mock.supports_model = MagicMock(return_value=True)
+ return mock
+
+
+def _make_openai_backend(body: dict[str, Any]) -> MagicMock:
+ """Build a mock backend whose ``send_openai_message`` returns ``body`` (OpenAI shape).
+
+ The body carries a ``usage`` block with ``prompt_tokens_details.cached_tokens``
+ (the OpenAI / Azure-GPT-via-LiteLLM non-streaming shape). Bedrock-style
+ top-level ``cache_*_input_tokens`` keys are also honored by the handler.
+ """
+
+ async def fake_send(body_: dict, headers: dict) -> BackendResponse:
+ return BackendResponse(body=body, status_code=200)
+
+ mock = MagicMock()
+ mock.name = "anyllm-openai"
+ mock.send_openai_message = fake_send
+ mock.map_model_id = MagicMock(return_value="gpt-5.5")
+ mock.supports_model = MagicMock(return_value=True)
+ return mock
+
+
+# =============================================================================
+# Bug A — OpenAI backend non-streaming (Azure/LiteLLM/AnyLLM OpenAI, stream=False)
+# =============================================================================
+
+
+def test_openai_backend_nonstreaming_emits_perf_with_cache_read_and_inferred_write() -> None:
+ """OpenAI backend non-streaming must surface cache reads + inferred writes.
+
+ OpenAI Chat Completions non-streaming carries::
+
+ usage: {
+ prompt_tokens: 1000,
+ completion_tokens: 50,
+ prompt_tokens_details: { cached_tokens: 700 }
+ }
+
+ OpenAI never reports a separate write counter, so it is inferred as
+ ``max(prompt_tokens - cached_tokens, 0)``. The handler already computes
+ both and feeds them to ``openai_prefix_tracker`` — this test pins that they
+ also reach the PERF log line (previously computed-then-dropped).
+ """
+ config = ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ backend="anyllm",
+ anyllm_provider="openai",
+ )
+
+ body = {
+ "id": "chatcmpl-1",
+ "object": "chat.completion",
+ "choices": [
+ {"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}
+ ],
+ "usage": {
+ "prompt_tokens": 1000,
+ "completion_tokens": 50,
+ "total_tokens": 1050,
+ "prompt_tokens_details": {"cached_tokens": 700},
+ },
+ }
+ backend = _make_openai_backend(body)
+
+ log_handle = _attach_proxy_log_capture()
+ try:
+ with patch("headroom.proxy.server.AnyLLMBackend", return_value=backend):
+ app = create_app(config)
+ with TestClient(app) as client:
+ resp = client.post(
+ "/v1/chat/completions",
+ json={
+ "model": "gpt-5.5",
+ "messages": [{"role": "user", "content": "hi"}],
+ },
+ headers={"Authorization": "Bearer test-key"},
+ )
+ assert resp.status_code == 200, resp.text[:200]
+ finally:
+ _detach_proxy_log_capture(*log_handle)
+
+ handler = log_handle[0]
+ cr, cw, chp = _find_perf_record(handler.records)
+ assert cr == 700, f"expected cache_read=700, got {cr}"
+ assert cw == 300, f"expected inferred cache_write=300 (=1000-700), got {cw}"
+ assert chp == 70, f"expected cache_hit_pct=70, got {chp}"
+
+
+def test_openai_backend_nonstreaming_perf_zeros_when_upstream_omits_cache_usage() -> None:
+ """When the upstream omits usage entirely, cache values must be zero — not absent.
+
+ Mirrors the streaming twin: no ``usage`` block at all means no
+ ``prompt_tokens`` to infer a write from, so all cache counters stay 0.
+ (When ``usage`` IS present but lacks cache details, the inferred write is
+ ``prompt_tokens - 0``, which is non-zero — that is covered by the positive
+ test above.)
+ """
+ config = ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ backend="anyllm",
+ anyllm_provider="openai",
+ )
+
+ body = {
+ "id": "chatcmpl-1",
+ "object": "chat.completion",
+ "choices": [
+ {"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}
+ ],
+ }
+ backend = _make_openai_backend(body)
+
+ log_handle = _attach_proxy_log_capture()
+ try:
+ with patch("headroom.proxy.server.AnyLLMBackend", return_value=backend):
+ app = create_app(config)
+ with TestClient(app) as client:
+ resp = client.post(
+ "/v1/chat/completions",
+ json={
+ "model": "gpt-5.5",
+ "messages": [{"role": "user", "content": "hi"}],
+ },
+ headers={"Authorization": "Bearer test-key"},
+ )
+ assert resp.status_code == 200
+ finally:
+ _detach_proxy_log_capture(*log_handle)
+
+ handler = log_handle[0]
+ cr, cw, chp = _find_perf_record(handler.records)
+ assert (cr, cw, chp) == (0, 0, 0)
+
+
+# =============================================================================
+# Bug B — Anthropic backend non-streaming (Bedrock / Vertex / LiteLLM, stream=False)
+# =============================================================================
+
+
+def test_anthropic_backend_nonstreaming_emits_perf_with_cache_read_and_write() -> None:
+ """Anthropic backend non-streaming must surface cache_read + cache_write.
+
+ Bedrock / Vertex / LiteLLM(anthropic) non-streaming returns an Anthropic
+ Messages body whose ``usage`` carries ``cache_read_input_tokens`` and
+ ``cache_creation_input_tokens``. The handler previously extracted only
+ ``output_tokens`` from this same dict — the cache counters were right
+ there, unread. Mirror of the streaming test
+ ``test_bedrock_streaming_emits_perf_with_message_start_cache_usage``.
+ """
+ config = ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ backend="anyllm",
+ anyllm_provider="anthropic",
+ )
+
+ body = {
+ "id": "msg_1",
+ "type": "message",
+ "role": "assistant",
+ "model": "claude-3-5-sonnet-20241022",
+ "content": [{"type": "text", "text": "hi"}],
+ "stop_reason": "end_turn",
+ "usage": {
+ "input_tokens": 1000,
+ "output_tokens": 50,
+ "cache_read_input_tokens": 500,
+ "cache_creation_input_tokens": 200,
+ },
+ }
+ backend = _make_anthropic_backend(body)
+
+ log_handle = _attach_proxy_log_capture()
+ try:
+ with patch("headroom.proxy.server.AnyLLMBackend", return_value=backend):
+ app = create_app(config)
+ with TestClient(app) as client:
+ resp = client.post(
+ "/v1/messages",
+ json={
+ "model": "claude-3-5-sonnet-20241022",
+ "messages": [{"role": "user", "content": "hi"}],
+ "max_tokens": 64,
+ },
+ headers={
+ "x-api-key": "sk-ant-test",
+ "anthropic-version": "2023-06-01",
+ },
+ )
+ assert resp.status_code == 200, resp.text[:200]
+ finally:
+ _detach_proxy_log_capture(*log_handle)
+
+ handler = log_handle[0]
+ cr, cw, chp = _find_perf_record(handler.records)
+ assert cr == 500, f"expected cache_read=500, got {cr}"
+ assert cw == 200, f"expected cache_write=200, got {cw}"
+ # round(500 / (500 + 200) * 100) = round(71.43) = 71
+ assert chp == 71, f"expected cache_hit_pct=71, got {chp}"
+
+
+def test_anthropic_backend_nonstreaming_perf_zeros_when_upstream_omits_cache_usage() -> None:
+ """When the upstream omits cache counters, cache values must be zero."""
+ config = ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ backend="anyllm",
+ anyllm_provider="anthropic",
+ )
+
+ body = {
+ "id": "msg_1",
+ "type": "message",
+ "role": "assistant",
+ "model": "claude-3-5-sonnet-20241022",
+ "content": [{"type": "text", "text": "hi"}],
+ "stop_reason": "end_turn",
+ "usage": {"input_tokens": 1000, "output_tokens": 50},
+ }
+ backend = _make_anthropic_backend(body)
+
+ log_handle = _attach_proxy_log_capture()
+ try:
+ with patch("headroom.proxy.server.AnyLLMBackend", return_value=backend):
+ app = create_app(config)
+ with TestClient(app) as client:
+ resp = client.post(
+ "/v1/messages",
+ json={
+ "model": "claude-3-5-sonnet-20241022",
+ "messages": [{"role": "user", "content": "hi"}],
+ "max_tokens": 64,
+ },
+ headers={
+ "x-api-key": "sk-ant-test",
+ "anthropic-version": "2023-06-01",
+ },
+ )
+ assert resp.status_code == 200
+ finally:
+ _detach_proxy_log_capture(*log_handle)
+
+ handler = log_handle[0]
+ cr, cw, chp = _find_perf_record(handler.records)
+ assert (cr, cw, chp) == (0, 0, 0)