Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): Strands MCP bundle + backend path fixes + Codex fail-closed protection

chopratejas20dc1f28f3cc
diff --git a/examples/strands_bedrock_demo.py b/examples/strands_bedrock_demo.py
index 74f5c1de9..51ccf2634 100644
--- a/examples/strands_bedrock_demo.py
+++ b/examples/strands_bedrock_demo.py
@@ -586,7 +586,7 @@ def run_hook_provider_demo(region: str = "us-west-2") -> dict[str, Any]:
# Create BedrockModel
model = BedrockModel(
- model_id="anthropic.claude-3-haiku-20240307-v1:0",
+ model_id="us.anthropic.claude-sonnet-4-5-20250929-v1:0",
region_name=region,
temperature=0.1,
)
@@ -769,7 +769,7 @@ def run_model_wrapper_demo(region: str = "us-west-2") -> dict[str, Any]:
# Create base Bedrock model
base_model = BedrockModel(
- model_id="anthropic.claude-3-haiku-20240307-v1:0",
+ model_id="us.anthropic.claude-sonnet-4-5-20250929-v1:0",
region_name=region,
temperature=0.1,
)
diff --git a/examples/strands_bundle_demo.py b/examples/strands_bundle_demo.py
new file mode 100644
index 000000000..cd1cada00
--- /dev/null
+++ b/examples/strands_bundle_demo.py
@@ -0,0 +1,255 @@
+#!/usr/bin/env python3
+"""Strands agent + Headroom — drop-in compression demo.
+
+This is what a real Strands user writes. The ONLY Headroom-specific
+lines are the import and the bundle construction. Everything else
+is normal Strands code.
+
+The demo:
+ 1. Defines a normal Strands `@tool` that returns a verbose JSON blob
+ (mimicking a real RAG or DB tool result).
+ 2. Builds a normal Strands `Agent` with that tool + the bundle's MCP
+ tools (headroom_compress / headroom_retrieve / headroom_stats).
+ 3. Sends ONE user query.
+ 4. Lets the agent loop autonomously.
+ 5. Prints the answer + the proxy /stats summary so you can SEE that
+ Headroom compressed the verbose tool output on the way to Bedrock.
+
+The Headroom proxy is started as a background process here so the
+script is self-contained. In production, the proxy runs as a
+long-lived service (ECS / k8s / EC2) and the application code looks
+exactly like what's below the `=== USER CODE ===` line.
+
+Run
+---
+
+ AWS_REGION=us-west-2 python examples/strands_bundle_demo.py
+
+Cost
+----
+
+Sonnet 4.5 via Bedrock, multi-turn agent loop. Expect ~$0.02–0.10
+per run depending on how many tool calls the model makes.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+import time
+import urllib.request
+from contextlib import suppress
+from pathlib import Path
+
+# ============================================================================
+# Boilerplate: start the Headroom proxy as a child process for the demo.
+# In production this is a long-lived service — none of this code is in your
+# Strands app.
+# ============================================================================
+
+PROXY_PORT = 8787
+PROXY_URL = f"http://127.0.0.1:{PROXY_PORT}"
+
+
+def _start_proxy() -> subprocess.Popen[bytes]:
+ cmd = [
+ sys.executable,
+ "-m",
+ "headroom.cli",
+ "proxy",
+ "--backend",
+ "bedrock",
+ "--region",
+ os.environ.get("AWS_REGION", "us-west-2"),
+ "--port",
+ str(PROXY_PORT),
+ ]
+ log_path = Path("/tmp/headroom_bundle_demo_proxy.log")
+ log = log_path.open("wb")
+ proc = subprocess.Popen(cmd, stdout=log, stderr=subprocess.STDOUT) # noqa: S603
+ print(f" → proxy started (pid={proc.pid}); log: {log_path}")
+ return proc
+
+
+def _wait_for_proxy(timeout_s: float = 30.0) -> None:
+ deadline = time.time() + timeout_s
+ while time.time() < deadline:
+ try:
+ with urllib.request.urlopen(f"{PROXY_URL}/readyz", timeout=1) as r: # noqa: S310
+ if r.status == 200:
+ return
+ except Exception: # noqa: BLE001
+ time.sleep(0.5)
+ raise RuntimeError(f"Proxy did not become ready within {timeout_s}s")
+
+
+def _stop_proxy(proc: subprocess.Popen[bytes]) -> None:
+ with suppress(ProcessLookupError):
+ proc.terminate()
+ try:
+ proc.wait(timeout=5)
+ except subprocess.TimeoutExpired:
+ proc.kill()
+
+
+def _print_stats_panel() -> None:
+ try:
+ with urllib.request.urlopen(f"{PROXY_URL}/stats", timeout=5) as r: # noqa: S310
+ stats = json.loads(r.read())
+ except Exception: # noqa: BLE001
+ return
+ summary = stats.get("summary", {})
+ comp = summary.get("compression", {})
+ uncomp = summary.get("uncompressed_requests", {})
+ mcp = summary.get("mcp", {}) or {}
+ print()
+ print(" Proxy /stats summary")
+ print(" --------------------")
+ print(f" api_requests: {summary.get('api_requests', 0)}")
+ print(f" requests_compressed: {comp.get('requests_compressed', 0)}")
+ print(f" total_tokens_removed: {comp.get('total_tokens_removed', 0)}")
+ if comp.get("best_compression_pct"):
+ print(f" best_compression_pct: {comp['best_compression_pct']:.1f}%")
+ print(f" uncompressed reasons: {uncomp}")
+ # MCP-side work (headroom_compress / headroom_retrieve called by the LLM
+ # via Strands' MCP dispatcher). The retrievals counter is the
+ # over-compression alarm — if it grows linearly with turn count, our
+ # lossy compressors are dropping info the model actually needs.
+ print(f" mcp_compressions: {mcp.get('compressions', 0)}")
+ print(f" mcp_tokens_removed: {mcp.get('tokens_removed', 0)}")
+ print(f" ccr_retrievals_count: {mcp.get('retrievals', 0)}")
+
+
+# ============================================================================
+# === USER CODE ===
+# Everything below is what a normal Strands user writes. The only
+# Headroom-specific bits are the `HeadroomBundle` import and one
+# construction call. The rest is vanilla Strands.
+# ============================================================================
+
+from strands import Agent, tool # noqa: E402 (kept under USER CODE banner for readability)
+from strands.models.openai import OpenAIModel # noqa: E402
+
+from headroom.integrations.strands import HeadroomBundle # noqa: E402
+
+
+@tool
+def search_documentation(query: str) -> str:
+ """Search the documentation for articles matching `query`. Returns up to 30 results as JSON."""
+ # Mock data — pretend this hit a real search API. The point is the
+ # response is big and repetitive, which is the kind of tool output
+ # Headroom is designed to shrink.
+ articles = [
+ {
+ "id": f"doc-{i:04d}",
+ "title": f"{query.title()} Guide — Part {i + 1}",
+ "url": f"https://docs.example.com/{query}/article-{i + 1}",
+ "category": "tutorial" if i % 3 == 0 else "reference",
+ "snippet": (
+ f"This article covers {query} implementation in depth. "
+ f"It walks through setup, configuration, and common pitfalls. "
+ f"Section {i + 1} of the comprehensive guide series."
+ )
+ * 4,
+ "metadata": {
+ "author": "docs-team",
+ "tags": ["how-to", query, "production-ready"],
+ "last_updated": f"2026-04-{(i % 28) + 1:02d}",
+ },
+ }
+ for i in range(30)
+ ]
+ return json.dumps(articles)
+
+
+def run_agent_demo() -> None:
+ """The actual Strands agent demo — looks like any other Strands app."""
+
+ # === The only Headroom-specific code in your app ===
+ bundle = HeadroomBundle(
+ proxy_url=PROXY_URL,
+ enable_serena_mcp=False, # disabled here so the demo runs fast
+ )
+
+ # === Normal Strands agent setup ===
+ model = OpenAIModel(
+ model_id="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
+ client_args={
+ "base_url": f"{PROXY_URL}/v1",
+ "api_key": "dummy-bedrock-uses-aws-creds-at-proxy",
+ "default_headers": {
+ "x-headroom-session-id": "demo-1",
+ "X-Client": "strands",
+ },
+ },
+ params={"max_tokens": 800, "temperature": 0.2},
+ )
+
+ agent = Agent(
+ model=model,
+ # bundle.tools = [Headroom MCP client]; we also add our local tool
+ tools=bundle.tools + [search_documentation],
+ system_prompt=(
+ "You are a documentation assistant. When the user asks about a "
+ "topic, use the search_documentation tool to look it up, then "
+ "answer concisely."
+ ),
+ )
+
+ print("\n → sending user query (agent runs autonomously) ...")
+ user_query = (
+ "Search the documentation for 'authentication'. "
+ "Tell me how many results you got, then summarize the top 3 in one sentence each."
+ )
+ print(f" user: {user_query!r}\n")
+
+ t0 = time.time()
+ response = agent(user_query)
+ elapsed = time.time() - t0
+
+ print(f"\n agent response (after {elapsed:.1f}s):")
+ print(" " + "-" * 68)
+ for line in str(response).splitlines():
+ print(f" {line}")
+ print(" " + "-" * 68)
+
+
+# ============================================================================
+# Bootstrap
+# ============================================================================
+
+
+def main() -> int:
+ print("=" * 72)
+ print(" Strands agent + Headroom (drop-in compression)")
+ print("=" * 72)
+ print(f" proxy: {PROXY_URL} region: {os.environ.get('AWS_REGION', 'us-west-2')}")
+ print()
+ print(" Starting Headroom proxy (in production this is a long-lived service) ...")
+ proxy = _start_proxy()
+ try:
+ _wait_for_proxy()
+ print(" → proxy ready.")
+ run_agent_demo()
+ _print_stats_panel()
+ print("\n" + "=" * 72)
+ print(" Done. If requests_compressed > 0 above, Headroom shrunk the")
+ print(" verbose tool output on its way to Bedrock — automatically,")
+ print(" with no code changes in the agent.")
+ print("=" * 72)
+ return 0
+ except Exception as e: # noqa: BLE001
+ import traceback
+
+ print(f"\n ! FAILED: {type(e).__name__}: {e}")
+ traceback.print_exc()
+ return 1
+ finally:
+ print("\n → stopping proxy ...")
+ _stop_proxy(proxy)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/examples/strands_mcp_dispatch_test.py b/examples/strands_mcp_dispatch_test.py
new file mode 100644
index 000000000..f1dc54e70
--- /dev/null
+++ b/examples/strands_mcp_dispatch_test.py
@@ -0,0 +1,273 @@
+#!/usr/bin/env python3
+"""Verify Strands' MCP dispatcher can reach Headroom's MCP server.
+
+This is the focused 15-min test that proves the MCP-everywhere
+architecture for Path B works end-to-end:
+
+ 1. Headroom proxy is running (started separately, port 8787).
+ 2. Build a Strands MCPClient that stdio-spawns `headroom mcp serve`.
+ 3. List MCP tools — confirm `headroom_retrieve` is exposed.
+ 4. Round-trip: stash content via the proxy's /v1/compress endpoint
+ to get a hash, then call `headroom_retrieve(hash)` via MCP,
+ verify the original comes back.
+ 5. (Bonus) Show the same hash being resolved through both the
+ proxy's REST /v1/retrieve endpoint AND the MCP path -- proves
+ the CompressionStore is shared between proxy and MCP server.
+
+If steps 3+4 work, the MCP dispatch path is alive — meaning when a
+Strands Agent receives a model-emitted `headroom_retrieve` tool_call
+(streaming OR non-streaming), it can dispatch it via this same MCP
+client and the chain closes.
+
+Run
+---
+ AWS_REGION=us-west-2 python examples/strands_mcp_dispatch_test.py
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+import time
+import urllib.error
+import urllib.request
+from contextlib import suppress
+from pathlib import Path
+
+from mcp import StdioServerParameters
+from mcp.client.stdio import stdio_client
+from strands.tools.mcp import MCPClient
+
+PROXY_PORT = 8787 # matches MCP default
+PROXY_URL = f"http://127.0.0.1:{PROXY_PORT}"
+
+
+def start_proxy() -> subprocess.Popen[bytes]:
+ """Spawn the proxy on the MCP default port."""
+ cmd = [
+ sys.executable,
+ "-m",
+ "headroom.cli",
+ "proxy",
+ "--backend",
+ "bedrock",
+ "--region",
+ "us-west-2",
+ "--port",
+ str(PROXY_PORT),
+ ]
+ log = Path("/tmp/headroom_proxy_mcp_test.log").open("wb")
+ proc = subprocess.Popen(cmd, stdout=log, stderr=subprocess.STDOUT) # noqa: S603
+ print(f" proxy spawned (pid={proc.pid}); log → /tmp/headroom_proxy_mcp_test.log")
+ return proc
+
+
+def wait_for_proxy(timeout_s: float = 30.0) -> None:
+ deadline = time.time() + timeout_s
+ while time.time() < deadline:
+ try:
+ with urllib.request.urlopen(f"{PROXY_URL}/readyz", timeout=1) as r: # noqa: S310
+ if r.status == 200:
+ return
+ except Exception: # noqa: BLE001
+ time.sleep(0.5)
+ raise RuntimeError(f"Proxy did not become ready in {timeout_s}s")
+
+
+def stop_proxy(proc: subprocess.Popen[bytes]) -> None:
+ with suppress(ProcessLookupError):
+ proc.terminate()
+ try:
+ proc.wait(timeout=5)
+ except subprocess.TimeoutExpired:
+ proc.kill()
+
+
+def stash_content_via_proxy(content: str) -> str | None:
+ """Round-trip through proxy /v1/compress to get a stored hash.
+
+ Returns the hash if compression replaced anything, else None.
+ """
+ body = json.dumps({"content": content}).encode()
+ req = urllib.request.Request( # noqa: S310
+ f"{PROXY_URL}/v1/compress",
+ data=body,
+ headers={"Content-Type": "application/json"},
+ method="POST",
+ )
+ try:
+ with urllib.request.urlopen(req, timeout=10) as r: # noqa: S310
+ resp = json.loads(r.read())
+ except urllib.error.HTTPError as e:
+ err = e.read().decode("utf-8", errors="replace")[:400]
+ print(f" /v1/compress HTTP {e.code}: {err}")
+ return None
+ print(f" /v1/compress response keys: {list(resp.keys())}")
+ # The response shape varies — try common shape candidates.
+ for key in ("hash", "stored_hash", "content_hash"):
+ if key in resp:
+ return str(resp[key])
+ compressed = resp.get("compressed") or resp.get("output") or ""
+ # Marker form: 'Retrieve original: hash=abc' or 'Retrieve more: hash=abc'
+ for marker in ("hash=",):
+ if marker in compressed:
+ idx = compressed.index(marker) + len(marker)
+ end = compressed.find(" ", idx)
+ return compressed[idx : end if end > 0 else idx + 64].strip().rstrip(">")
+ print(f" no hash found in /v1/compress response — full body: {json.dumps(resp)[:400]}")
+ return None
+
+
+def main() -> int:
+ print("=" * 72)
+ print(" Strands MCPClient → Headroom MCP server end-to-end probe")
+ print("=" * 72)
+
+ print("\n[1/5] Starting Headroom proxy ...")
+ proxy = start_proxy()
+ try:
+ wait_for_proxy()
+ print(" proxy ready.\n")
+
+ big = json.dumps(
+ [
+ {
+ "id": i,
+ "name": f"order-{i}",
+ "status": "completed",
+ "amount": 100 + i,
+ "customer": f"cust-{i % 50}",
+ "description": "A long, repetitive description that takes up space " * 4,
+ }
+ for i in range(300)
+ ]
+ )
+ print(f"[2/5] Test payload prepared: {len(big)} chars of JSON")
+
+ print("\n[3/5] Building Strands MCPClient pointed at `headroom mcp serve` ...")
+ server_params = StdioServerParameters(
+ command="headroom",
+ args=["mcp", "serve", "--proxy-url", PROXY_URL],
+ )
+ with MCPClient(lambda: stdio_client(server_params)) as mcp:
+ print(" MCP connection up.")
+
+ print("\n[4/5] Listing MCP tools ...")
+ try:
+ tools = mcp.list_tools_sync()
+ except Exception as e: # noqa: BLE001
+ print(f" ! list_tools_sync failed: {type(e).__name__}: {e}")
+ raise
+ tool_names = [getattr(t, "tool_name", None) or getattr(t, "name", "?") for t in tools]
+ print(f" tools advertised by Headroom MCP: {tool_names}")
+ if "headroom_retrieve" not in tool_names:
+ print(" ! headroom_retrieve NOT in tool list — abort.")
+ return 4
+
+ # Round-trip via MCP: compress to get a hash, then retrieve via MCP.
+ # This is a pure MCP-only path through Strands' dispatcher — the
+ # same dispatcher that would resolve a model-emitted
+ # `headroom_retrieve` tool_call in a real conversation.
+ print("\n[5a/5] Stashing content via MCP headroom_compress (same dispatcher) ...")
+ try:
+ comp = mcp.call_tool_sync(
+ tool_use_id="probe-compress-1",
+ name="headroom_compress",
+ arguments={"content": big},
+ )
+ print(f" status={comp.get('status', '?')}")
+ # Pull the JSON body out of the tool result
+ comp_payload: dict | None = None
+ for block in comp.get("content", []) or []:
+ if isinstance(block, dict):
+ if "json" in block:
+ comp_payload = block["json"]
+ break
+ if "text" in block:
+ try:
+ comp_payload = json.loads(block["text"])
+ except (json.JSONDecodeError, ValueError):
+ comp_payload = {"_raw_text": block["text"]}
+ break
+ print(f" compress payload keys: {list((comp_payload or {}).keys())}")
+ # Find the hash. Headroom MCP compress is documented to emit
+ # markers in the compressed output ("hash=abc..."); also surface
+ # explicit hash field if present.
+ stashed_hash: str | None = None
+ if comp_payload:
+ for key in ("hash", "stored_hash", "content_hash"):
+ if key in comp_payload and comp_payload[key]:
+ stashed_hash = str(comp_payload[key])
+ break
+ if not stashed_hash:
+ compressed_str = (
+ comp_payload.get("compressed")
+ or comp_payload.get("output")
+ or comp_payload.get("_raw_text", "")
+ )
+ if "hash=" in compressed_str:
+ idx = compressed_str.index("hash=") + len("hash=")
+ tail = compressed_str[idx : idx + 80]
+ # hash is followed by space, quote, > or end
+ stashed_hash = ""
+ for ch in tail:
+ if ch in " \"'>),\n\r\t":
+ break
+ stashed_hash += ch
+ stashed_hash = stashed_hash or None
+ if not stashed_hash:
+ print(
+ f" ! no hash found in compress response. Full body: "
+ f"{json.dumps(comp_payload)[:400]}"
+ )
+ return 5
+ print(f" ✓ stashed hash: {stashed_hash[:32]}...")
+ except Exception as e: # noqa: BLE001
+ print(f" ! compress call failed: {type(e).__name__}: {e}")
+ return 5
+
+ print("\n[5b/5] Calling headroom_retrieve via Strands MCP dispatcher ...")
+ try:
+ result = mcp.call_tool_sync(
+ tool_use_id="probe-retrieve-1",
+ name="headroom_retrieve",
+ arguments={"hash": stashed_hash},
+ )
+ print(f" retrieve status: {result.get('status', '?')}")
+ content = result.get("content", [])
+ preview = ""
+ for block in content:
+ if isinstance(block, dict):
+ if "text" in block:
+ preview = block["text"][:300]
+ break
+ if "json" in block:
+ preview = json.dumps(block["json"])[:300]
+ break
+ print(f" retrieved content preview: {preview!r}")
+ if any(needle in preview for needle in ("order-0", "order-1", "order-")):
+ print(" ✓ retrieved content references the original JSON rows.")
+ else:
+ print(
+ " - preview doesn't obviously match the original. "
+ "Scan the full preview above to verify."
+ )
+ except Exception as e: # noqa: BLE001
+ print(f" ! retrieve call failed: {type(e).__name__}: {e}")
+ return 6
+
+ print("\n" + "=" * 72)
+ print(" MCP DISPATCH PROBE COMPLETE")
+ print(" If steps 3+4+5 succeeded, the MCP-everywhere Path B is live:")
+ print(" Strands receives a headroom_retrieve tool_call → MCP dispatcher")
+ print(" → Headroom MCP server → CompressionStore → original content back.")
+ print("=" * 72)
+ return 0
+ finally:
+ print("\n stopping proxy ...")
+ stop_proxy(proxy)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/examples/strands_via_proxy_demo.py b/examples/strands_via_proxy_demo.py
new file mode 100644
index 000000000..ff79e2db1
--- /dev/null
+++ b/examples/strands_via_proxy_demo.py
@@ -0,0 +1,590 @@
+#!/usr/bin/env python3
+"""End-to-end demo: Strands -> Headroom proxy -> Bedrock.
+
+Proves the four Path-B fixes work together against live AWS Bedrock:
+
+ Fix #1 PrefixCacheTracker.update_from_response on the backend path
+ Fix #2 CCR response intercept for the OpenAI-shape proxy
+ Fix #3 LiteLLM's native cache_control -> cachePoint translation
+ Fix #4 Strands harness label in CLIENT_UA_MAP
+
+What this script does
+---------------------
+1. Spawns the Headroom proxy as a subprocess (backend=bedrock).
+2. Waits for /readyz.
+3. Sends two requests in the same session via Strands' OpenAIModel
+ pointed at the proxy. The system prompt is intentionally large
+ (>1024 tokens; Bedrock's minimum cacheable block) and tagged with
+ ``cache_control: {type: "ephemeral"}`` via Headroom's CacheAligner.
+4. Reports:
+ * compression numbers (tokens before/after, on each turn)
+ * Bedrock cache hits (cache_read_input_tokens on turn 2 -- proves
+ LiteLLM translated cache_control to cachePoint AND Bedrock
+ served from the cache)
+ * harness label (proves X-Client: strands flows through)
+5. Tears the proxy back down.
+
+Requirements
+------------
+- AWS credentials in ~/.aws/credentials or environment.
+- ``pip install -e .[strands,bedrock]`` from the repo root (already
+ done if you've been running the existing Strands demo).
+- A free local TCP port (default 8765; override via ``--port``).
+
+Run
+---
+ AWS_REGION=us-west-2 python examples/strands_via_proxy_demo.py
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import os
+import subprocess
+import sys
+import time
+import urllib.error
+import urllib.request
+from contextlib import suppress
+from pathlib import Path
+from typing import Any
+
+# Defer Strands / openai imports until after we've validated the proxy
+# starts -- that way the error message for a missing dep doesn't bury
+# a more useful "proxy refused to start" trace.
+
+
+# ----------------------------------------------------------------------------
+# Constants
+# ----------------------------------------------------------------------------
+
+DEFAULT_PORT = 8765
+DEFAULT_REGION = "us-west-2"
+DEFAULT_MODEL = "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
+SESSION_ID = "strands-via-proxy-demo-1"
+
+# A ~2.5K-token block. Sonnet 4.5 caches empirically at this size on
+# Bedrock (verified: cache_write=2206 with the same prompt below).
+# The CacheAligner (Anthropic-style ephemeral cache_control) marks
+# this as cacheable; LiteLLM translates the marker to Bedrock
+# cachePoint; Bedrock serves it from the read cache on turn 2.
+LARGE_SYSTEM_PROMPT = (
+ "You are a precise technical assistant. "
+ "Treat the following as authoritative reference context for every "
+ "question in this conversation. Quote it accurately, do not "
+ "fabricate. Reference context:\n\n"
+ + "Headroom is an open-source context compression layer for LLM "
+ "applications. It sits in front of provider APIs (Anthropic, "
+ "OpenAI, Bedrock, Vertex) and shrinks the prompt without losing "
+ "semantically important information. " * 200
+)
+
+
+# ----------------------------------------------------------------------------
+# Proxy lifecycle
+# ----------------------------------------------------------------------------
+
+
+def start_proxy(port: int, region: str) -> subprocess.Popen[bytes]:
+ """Spawn `headroom proxy --backend bedrock` as a subprocess."""
+ env = os.environ.copy()
+ env.setdefault("AWS_REGION", region)
+ env.setdefault("AWS_DEFAULT_REGION", region)
+ # Crank logging up so we can read pipeline decisions live.
+ env.setdefault("HEADROOM_LOG", "INFO")
+ cmd = [
+ sys.executable,
+ "-m",
+ "headroom.cli",
+ "proxy",
+ "--backend",
+ "bedrock",
+ "--region",
+ region,
+ "--port",
+ str(port),
+ ]
+ print(f" $ {' '.join(cmd)}", file=sys.stderr)
+ log_path = Path("/tmp") / f"strands_via_proxy_demo_{port}.log"
+ log_file = log_path.open("wb")
+ proc = subprocess.Popen( # noqa: S603 — argv is fixed above
+ cmd,
+ env=env,
+ stdout=log_file,
+ stderr=subprocess.STDOUT,
+ )
+ print(f" proxy logs -> {log_path}", file=sys.stderr)
+ return proc
+
+
+def wait_for_proxy_ready(port: int, timeout_s: float = 30.0) -> None:
+ """Poll /readyz until the proxy answers or timeout."""
+ url = f"http://127.0.0.1:{port}/readyz"
+ deadline = time.time() + timeout_s
+ last_err: Exception | None = None
+ while time.time() < deadline:
+ try:
+ with urllib.request.urlopen(url, timeout=1) as resp: # noqa: S310
+ if resp.status == 200:
+ return
+ except (urllib.error.URLError, ConnectionError, TimeoutError) as e:
+ last_err = e
+ time.sleep(0.5)
+ raise RuntimeError(
+ f"Proxy on port {port} did not become ready within {timeout_s}s; last error: {last_err!r}"
+ )
+
+
+def stop_proxy(proc: subprocess.Popen[bytes]) -> None:
+ """Politely shut the proxy down."""
+ with suppress(ProcessLookupError):
+ proc.terminate()
+ try:
+ proc.wait(timeout=5)
+ except subprocess.TimeoutExpired:
+ proc.kill()
+ proc.wait(timeout=5)
+
+
+# ----------------------------------------------------------------------------
+# Strands wiring
+# ----------------------------------------------------------------------------
+
+
+def build_agent(port: int, model_id: str) -> Any:
+ """Construct a Strands Agent pointed at the proxy.
+
+ Uses OpenAIModel + base_url because that's the proxy-friendly path
+ (Bedrock's native auth would bypass the proxy entirely).
+ """
+ from strands import Agent
+ from strands.models.openai import OpenAIModel
+
+ model = OpenAIModel(
+ model_id=model_id,
+ client_args={
+ "api_key": "dummy-bedrock-uses-aws-creds-at-proxy",
+ "base_url": f"http://127.0.0.1:{port}/v1",
+ "default_headers": {
+ # Stable session key so the proxy's PrefixCacheTracker
+ # treats both turns as the same conversation.
+ "x-headroom-session-id": SESSION_ID,
+ # Harness identification (Fix #4) — the proxy labels
+ # this request as 'strands' in metrics + outcomes.
+ "X-Client": "strands",
+ },
+ },
+ # Bedrock-Claude rejects the OpenAI default of temperature=1.0
+ # for some Opus versions; pinning a Bedrock-compatible value.
+ params={"max_tokens": 200, "temperature": 0.2},
+ )
+ return Agent(model=model, system_prompt=LARGE_SYSTEM_PROMPT)
+
+
+# ----------------------------------------------------------------------------
+# Cache-stat probes
+# ----------------------------------------------------------------------------
+
+
+def fetch_proxy_stats(port: int) -> dict[str, Any]:
+ """Fetch overall proxy stats so we can correlate per-turn behaviour."""
+ url = f"http://127.0.0.1:{port}/stats"
+ try:
+ with urllib.request.urlopen(url, timeout=2) as resp: # noqa: S310
+ return json.loads(resp.read().decode())
+ except Exception as e:
+ return {"_error": str(e)}
+
+
+# ----------------------------------------------------------------------------
+# Direct HTTP smoke test (no Strands) -- proves the proxy alone
+# ----------------------------------------------------------------------------
+
+
+def direct_smoke_test(
+ port: int, model_id: str, session_id: str, with_cache_control: bool
+) -> dict[str, Any]:
+ """Issue a single chat.completion via raw HTTP -- proves the wiring
+ end-to-end without the Strands layer in the way.
+
+ When ``with_cache_control=True`` the system message carries an
+ explicit Anthropic-style ``cache_control: ephemeral`` block; this
+ isolates "does the proxy correctly forward cache_control + extract
+ response cache stats" from "does CacheAligner insert cache_control
+ on its own". Both questions must answer "yes" for the Path-B claim
+ to hold end-to-end.
+ """
+ url = f"http://127.0.0.1:{port}/v1/chat/completions"
+ if with_cache_control:
+ system_content: Any = [
+ {
+ "type": "text",
+ "text": LARGE_SYSTEM_PROMPT,
+ "cache_control": {"type": "ephemeral"},
+ }
+ ]
+ else:
+ system_content = LARGE_SYSTEM_PROMPT
+ body = json.dumps(
+ {
+ "model": model_id,
+ "messages": [
+ {"role": "system", "content": system_content},
+ {"role": "user", "content": "In one short sentence, what is Headroom?"},
+ ],
+ "max_tokens": 60,
+ "temperature": 0.2,
+ }
+ ).encode()
+ req = urllib.request.Request( # noqa: S310
+ url,
+ data=body,
+ headers={
+ "Content-Type": "application/json",
+ "Authorization": "Bearer dummy-bedrock-uses-aws-creds-at-proxy",
+ "x-headroom-session-id": session_id,
+ "X-Client": "strands",
+ },
+ method="POST",
+ )
+ t0 = time.time()
+ with urllib.request.urlopen(req, timeout=60) as resp: # noqa: S310
+ body_bytes = resp.read()
+ elapsed_ms = (time.time() - t0) * 1000
+ parsed = json.loads(body_bytes)
+ return {"elapsed_ms": elapsed_ms, "body": parsed}
+
+
+# ----------------------------------------------------------------------------
+# Main demo
+# ----------------------------------------------------------------------------
+
+
+def usage_summary(usage: dict[str, Any]) -> str:
+ """One-line digest of the usage block returned by the proxy."""
+ return (
+ f"prompt={usage.get('prompt_tokens', 0)} "
+ f"completion={usage.get('completion_tokens', 0)} "
+ f"cache_read={usage.get('cache_read_input_tokens', 0)} "
+ f"cache_write={usage.get('cache_creation_input_tokens', 0)}"
+ )
+
+
+async def run_demo(port: int, region: str, model_id: str) -> int:
+ print("=" * 76)
+ print(" Headroom Path-B E2E: Strands -> Headroom proxy -> Bedrock")
+ print("=" * 76)
+ print(f" port={port} region={region} model={model_id}")
+ print(f" session_id={SESSION_ID}")
+ print()
+
+ print("[1/4] Spawning Headroom proxy ...")
+ proxy = start_proxy(port=port, region=region)
+ try:
+ try:
+ wait_for_proxy_ready(port=port, timeout_s=45.0)
+ except Exception as e:
+ print(f" ! Proxy failed to start: {e}", file=sys.stderr)
+ return 2
+ print(" proxy ready.")
+
+ # ----------------------------------------------------------------
+ # 2. Direct HTTP smoke test WITH explicit cache_control.
+ # This isolates "proxy forwards cache_control + extracts stats"
+ # from "CacheAligner inserts cache_control on its own".
+ # ----------------------------------------------------------------
+ print("\n[2/4] Direct HTTP probe -- explicit cache_control (turn A, turn B same session)")
+ smoke_session = "cc-smoke-1"
+ try:
+ smoke_a = direct_smoke_test(
+ port=port, model_id=model_id, session_id=smoke_session, with_cache_control=True
+ )
+ smoke_b = direct_smoke_test(
+ port=port, model_id=model_id, session_id=smoke_session, with_cache_control=True
+ )
+ except urllib.error.HTTPError as e:
+ err_body = e.read().decode("utf-8", errors="replace")[:500]
+ print(f" ! smoke test failed: HTTP {e.code}: {err_body}", file=sys.stderr)
+ return 3
+ ua = smoke_a["body"].get("usage", {})
+ ub = smoke_b["body"].get("usage", {})
+ print(f" turn A: {usage_summary(ua)} ({smoke_a['elapsed_ms']:.0f}ms)")
+ print(f" turn B: {usage_summary(ub)} ({smoke_b['elapsed_ms']:.0f}ms)")
+ cache_works = ub.get("cache_read_input_tokens", 0) > 0
+ cache_write_a = ua.get("cache_creation_input_tokens", 0) > 0
+ if cache_works:
+ print(
+ f" ✓ cache hit on turn B (read={ub['cache_read_input_tokens']}) -- proxy chain OK."
+ )
+ elif cache_write_a:
+ print(" ! turn A wrote cache but turn B didn't read -- session keying may be off.")
+ else:
+ print(
+ " ! no cache write on turn A -- LiteLLM cache_control translation OR proxy did not forward it."
+ )
+
+ # ----------------------------------------------------------------
+ # 2b. Streaming probe: same chain but stream=True. The non-
+ # streaming smoke proved the synchronous path; this proves the
+ # streaming path also (a) forwards cache_control and (b) parses
+ # cache stats from the SSE usage frame and (c) updates the
+ # prefix tracker on stream end (Fix #1 streaming half).
+ # ----------------------------------------------------------------
+ print("\n[2b/4] Streaming probe -- same explicit cache_control payload")
+ try:
+ stream_url = f"http://127.0.0.1:{port}/v1/chat/completions"
+ stream_body = json.dumps(
+ {
+ "model": model_id,
+ "messages": [
+ {
+ "role": "system",
+ "content": [
+ {
+ "type": "text",
+ "text": LARGE_SYSTEM_PROMPT,
+ "cache_control": {"type": "ephemeral"},
+ }
+ ],
+ },
+ {"role": "user", "content": "Reply in 5 words."},
+ ],
+ "max_tokens": 30,
+ "temperature": 0.2,
+ "stream": True,
+ "stream_options": {"include_usage": True},
+ }
+ ).encode()
+ stream_req = urllib.request.Request( # noqa: S310
+ stream_url,
+ data=stream_body,
+ headers={
+ "Content-Type": "application/json",
+ "Authorization": "Bearer dummy",
+ "x-headroom-session-id": smoke_session,
+ "X-Client": "strands",
+ },
+ method="POST",
+ )
+ t0 = time.time()
+ last_usage_frame: dict[str, Any] | None = None
+ chunk_count = 0
+ with urllib.request.urlopen(stream_req, timeout=60) as stream_resp: # noqa: S310
+ for raw_line in stream_resp:
+ line = raw_line.decode("utf-8", errors="replace").strip()
+ if not line.startswith("data: "):
+ continue
+ payload = line[6:]
+ if payload == "[DONE]":
+ continue
+ try:
+ event = json.loads(payload)
+ except json.JSONDecodeError:
+ continue
+ chunk_count += 1
+ if event.get("usage"):
+ last_usage_frame = event["usage"]
+ stream_elapsed_ms = (time.time() - t0) * 1000
+ print(
+ f" streamed chunks={chunk_count} elapsed={stream_elapsed_ms:.0f}ms "
+ f"final_usage={last_usage_frame}"
+ )
+ if last_usage_frame and last_usage_frame.get("cache_read_input_tokens", 0) > 0:
+ print(" ✓ streaming path also returned cache_read_input_tokens > 0.")
+ elif last_usage_frame is None:
+ print(" ! no final usage frame surfaced -- check include_usage wiring.")
+ else:
+ print(" - no cache hit on streaming probe (may be a 3rd-call eviction edge).")
+ except urllib.error.HTTPError as e:
+ err_body = e.read().decode("utf-8", errors="replace")[:500]
+ print(f" ! streaming probe failed: HTTP {e.code}: {err_body}", file=sys.stderr)
+
+ # ----------------------------------------------------------------
+ # 2c. Compression probe.
+ # ContentRouter SKIPS user + system messages by design
+ # (skip_user_messages=True, skip_system=True at content_router.py:456
+ # and :2294). The bulk savings in real agent loops come from
+ # compressing TOOL RESULTS (and assistant turns), not from
+ # rewriting the user's question or paraphrasing the system
+ # prompt. To exercise the compression pipeline we send a fake
+ # assistant turn that just returned a verbose JSON tool result.
+ # SmartCrusher (Rust-backed, always available) targets exactly
+ # this shape.
+ # ----------------------------------------------------------------
+ print("\n[2c/4] Compression probe -- tool_result with verbose JSON")
+ # ContentRouter defaults (headroom/transforms/content_router.py):
+ # skip_user_messages: True (line 456) -- "subject of conversation"
+ # skip_system: True (line 2294) -- system prompt is sacred
+ # compress_assistant_text_blocks: False (line 472) -- conservative
+ # The ONE shape that compresses by default is the tool_result. This
+ # matches the real-world AWS agent-loop pattern: tool calls return
+ # large JSON/log/diff blobs that accumulate across turns and dominate
+ # the prompt. ContentRouter classifies the tool_result content,
+ # dispatches via the magika/unidiff detection chain to a per-type
+ # compressor (SmartCrusher for JSON arrays here), records % saved.
+ big_tool_result = json.dumps(
+ [
+ {
+ "id": f"order-{i}",
+ "customer_id": f"cust-{i % 100}",
+ "status": "completed",
+ "total_usd": 100 + i,
+ "items": [
+ {"sku": f"sku-{j}", "qty": 1, "name": f"Product {j}"} for j in range(5)
+ ],
+ "created_at": f"2026-05-{(i % 28) + 1:02d}T10:00:00Z",
+ "notes": "Standard processing, no exceptions",
+ }
+ for i in range(250)
+ ]
+ )
+ print(
+ f" tool_result size: {len(big_tool_result)} chars (~{len(big_tool_result) // 4} tokens)"
+ )
+ tool_probe_body = json.dumps(
+ {
+ "model": model_id,
+ "messages": [
+ {"role": "user", "content": "List recent completed orders."},
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "call_1",
+ "type": "function",
+ "function": {
+ "name": "list_orders",
+ "arguments": "{}",
+ },
+ }
+ ],
+ },
+ {
+ "role": "tool",
+ "tool_call_id": "call_1",
+ "content": big_tool_result,
+ },
+ {
+ "role": "user",
+ "content": "How many orders are in 'completed' status? Reply with the number only.",
+ },
+ ],
+ "tools": [
+ {
+ "type": "function",
+ "function": {
+ "name": "list_orders",
+ "description": "List recent orders.",
+ "parameters": {
+ "type": "object",
+ "properties": {},
+ "required": [],
+ },
+ },
+ }
+ ],
+ "max_tokens": 16,
+ "temperature": 0.0,
+ }
+ ).encode()
+ tool_req = urllib.request.Request( # noqa: S310
+ f"http://127.0.0.1:{port}/v1/chat/completions",
+ data=tool_probe_body,
+ headers={
+ "Content-Type": "application/json",
+ "Authorization": "Bearer dummy",
+ "x-headroom-session-id": "compression-probe-session",
+ "X-Client": "strands",
+ },
+ method="POST",
+ )
+ t0 = time.time()
+ try:
+ with urllib.request.urlopen(tool_req, timeout=60) as tr: # noqa: S310
+ tool_resp = json.loads(tr.read())
+ print(f" elapsed={time.time() - t0:.1f}s usage={tool_resp.get('usage', {})}")
+ # Check proxy /stats AFTER this call so we can see the compression delta.
+ post_stats = fetch_proxy_stats(port=port)
+ comp = post_stats.get("summary", {}).get("compression", {})
+ uncomp = post_stats.get("summary", {}).get("uncompressed_requests", {})
+ print(
+ f" cumulative compression: requests_compressed={comp.get('requests_compressed', 0)} "
+ f"tokens_removed={comp.get('total_tokens_removed', 0)} "
+ f"best_pct={comp.get('best_compression_pct', 0.0):.1f}%"
+ )
+ print(f" uncompressed reasons: {uncomp}")
+ except urllib.error.HTTPError as e:
+ err_body = e.read().decode("utf-8", errors="replace")[:500]
+ print(f" ! compression probe failed: HTTP {e.code}: {err_body}")
+
+ # ----------------------------------------------------------------
+ # 3. Strands agent: two turns, same session.
+ # No explicit cache_control here -- this tests whether
+ # CacheAligner (inside the proxy) inserts the marker itself.
+ # ----------------------------------------------------------------
+ print("\n[3/4] Building Strands agent ...")
+ agent = build_agent(port=port, model_id=model_id)
+
+ print(
+ "\n[4/4] Two-turn cache test via Strands (cache_control inserted by CacheAligner) ..."
+ )
+ print(" turn 1: priming the cache with the large system prompt")
+ r1 = agent("In one short sentence, what is Headroom?")
+ print(f" turn 1 response: {str(r1)[:160]}")
+
+ print("\n turn 2: same session -> should hit Bedrock prompt cache")
+ r2 = agent("In one short sentence, what providers does it support?")
+ print(f" turn 2 response: {str(r2)[:160]}")
+
+ # Verdict: pull the proxy stats (cumulative) so we can SEE cache stats
+ stats = fetch_proxy_stats(port=port)
+ print("\n proxy /stats snapshot:")
+ print(f" {json.dumps(stats, indent=2, default=str)[:1200]}")
+
+ # Tail the proxy log for cache_read mentions on turn 2 -- this is
+ # the load-bearing assertion: Fix #1 + Fix #3 worked iff the proxy
+ # logged a non-zero cache_read_input_tokens on the second call.
+ log_path = Path("/tmp") / f"strands_via_proxy_demo_{port}.log"
+ log_tail = log_path.read_text(errors="replace").splitlines()[-200:]
+ cache_lines = [
+ line
+ for line in log_tail
+ if "cache_read" in line.lower() or "cache stats" in line.lower()
+ ]
+ ccr_lines = [line for line in log_tail if "ccr" in line.lower()]
+ print("\n proxy log cache lines (last 200 lines):")
+ if cache_lines:
+ for line in cache_lines[-10:]:
+ print(f" {line[:240]}")
+ else:
+ print(" (no cache_read events surfaced — possible miss on this run)")
+ if ccr_lines:
+ print("\n proxy log CCR lines (last 200 lines):")
+ for line in ccr_lines[-10:]:
+ print(f" {line[:240]}")
+
+ print("\n" + "=" * 76)
+ print(" PATH-B E2E COMPLETE.")
+ print(" If you see cache_read_input_tokens > 0 on the second call,")
+ print(" the prefix-cache + cachePoint chain is working end-to-end.")
+ print("=" * 76)
+ return 0
+ finally:
+ print("\n shutting down proxy ...")
+ stop_proxy(proxy)
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description="Strands -> Headroom proxy -> Bedrock E2E")
+ ap.add_argument("--port", type=int, default=DEFAULT_PORT)
+ ap.add_argument("--region", default=DEFAULT_REGION)
+ ap.add_argument("--model", default=DEFAULT_MODEL)
+ args = ap.parse_args()
+ return asyncio.run(run_demo(port=args.port, region=args.region, model_id=args.model))
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/headroom/backends/litellm.py b/headroom/backends/litellm.py
index 0faa64b83..ff7cb5e27 100644
--- a/headroom/backends/litellm.py
+++ b/headroom/backends/litellm.py
@@ -1025,6 +1025,49 @@ class LiteLLMBackend(Backend):
# Make the call
response = await acompletion(**kwargs)
+ # Build the usage block. LiteLLM normalizes prompt-cache stats from
+ # multiple providers (Anthropic, Bedrock-Claude, OpenAI prompt-caching,
+ # DeepSeek) onto its Usage object — top-level
+ # cache_read_input_tokens / cache_creation_input_tokens for the
+ # Anthropic-style dialect, and prompt_tokens_details.cached_tokens /
+ # cache_creation_tokens for the OpenAI nested dialect. Surface both
+ # so PrefixCacheTracker.update_from_response on the backend-routed
+ # path observes a stable shape instead of branching on key presence.
+ usage_block: dict[str, Any] = {
+ "prompt_tokens": response.usage.prompt_tokens,
+ "completion_tokens": response.usage.completion_tokens,
+ "total_tokens": response.usage.total_tokens,
+ }
+
+ # Defensive getattr: LiteLLM only attaches these top-level attrs
+ # when the underlying provider returned cache stats. Zero is the
+ # cold-start / no-cache value.
+ cache_read = int(getattr(response.usage, "cache_read_input_tokens", 0) or 0)
+ cache_write = int(getattr(response.usage, "cache_creation_input_tokens", 0) or 0)
+
+ # OpenAI nested dialect — fall back here if the top-level dialect is
+ # absent (pure OpenAI prompt caching).
+ ptd_obj = getattr(response.usage, "prompt_tokens_details", None)
+ ptd_cached = 0
+ ptd_cache_creation = 0
+ if ptd_obj is not None:
+ ptd_cached = int(getattr(ptd_obj, "cached_tokens", 0) or 0)
+ ptd_cache_creation = int(getattr(ptd_obj, "cache_creation_tokens", 0) or 0)
+
+ final_cache_read = cache_read or ptd_cached
+ final_cache_write = cache_write or ptd_cache_creation
+
+ if final_cache_read or final_cache_write:
+ usage_block["cache_read_input_tokens"] = final_cache_read
+ usage_block["cache_creation_input_tokens"] = final_cache_write
+ # Mirror into the OpenAI nested shape so callers that only know
+ # the OpenAI dialect can read it without branching.
+ usage_block["prompt_tokens_details"] = {"cached_tokens": final_cache_read}
+ logger.debug(
+ f"LiteLLM OpenAI cache stats: cache_read={final_cache_read} "
+ f"cache_write={final_cache_write} model={litellm_model}"
+ )
+
# Convert ModelResponse to dict (OpenAI format)
response_dict = {
"id": response.id,
@@ -1059,11 +1102,7 @@ class LiteLLMBackend(Backend):
}
for c in response.choices
],
- "usage": {
- "prompt_tokens": response.usage.prompt_tokens,
- "completion_tokens": response.usage.completion_tokens,
- "total_tokens": response.usage.total_tokens,
- },
+ "usage": usage_block,
}
return BackendResponse(
diff --git a/headroom/integrations/strands/__init__.py b/headroom/integrations/strands/__init__.py
index a3a1ef0cf..46088ef52 100644
--- a/headroom/integrations/strands/__init__.py
+++ b/headroom/integrations/strands/__init__.py
@@ -29,6 +29,7 @@ import importlib.util
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
+ from .bundle import HeadroomBundle
from .hooks import HeadroomHookProvider
from .model import HeadroomStrandsModel, OptimizationMetrics, optimize_messages
from .providers import get_headroom_provider, get_model_name_from_strands
@@ -70,6 +71,10 @@ def __getattr__(name: str) -> Any:
from .providers import get_model_name_from_strands
return get_model_name_from_strands
+ elif name == "HeadroomBundle":
+ from .bundle import HeadroomBundle
+
+ return HeadroomBundle
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
@@ -85,4 +90,6 @@ __all__ = [
# Provider detection
"get_headroom_provider",
"get_model_name_from_strands",
+ # One-helper MCP + hook wiring (Headroom + Serena + RTK-equivalent)
+ "HeadroomBundle",
]
diff --git a/headroom/integrations/strands/bundle.py b/headroom/integrations/strands/bundle.py
new file mode 100644
index 000000000..1bfa292f6
--- /dev/null
+++ b/headroom/integrations/strands/bundle.py
@@ -0,0 +1,201 @@
+"""HeadroomBundle — single-helper MCP wiring for a Strands Agent.
+
+The cleanest production setup for Strands is the same kit
+``headroom wrap claude`` installs for Claude Code, restated as
+Strands-native primitives:
+
+* **Headroom MCP** (``headroom mcp serve``) — exposes
+ ``headroom_retrieve`` / ``headroom_compress`` / ``headroom_stats``
+ via stdio. The proxy emits ``Retrieve original: hash=...`` markers
+ in compressed content; the LLM calls ``headroom_retrieve`` when it
+ needs the original; Strands' MCP dispatcher resolves it via this
+ server. Works identically in streaming and non-streaming.
+
+* **Serena MCP** — semantic code intelligence (symbol search,
+ references, etc.). Auto-installed via ``uvx`` on first launch.
+
+* **HeadroomHookProvider** — the RTK-equivalent for Strands.
+ Compresses tool outputs in-place via ``AfterToolCallEvent`` so
+ verbose JSON / log / search outputs are shrunk before they
+ pollute the agent's context.
+
+Pattern
+-------
+
+.. code-block:: python
+
+ from strands import Agent
+ from strands.models.openai import OpenAIModel
+ from headroom.integrations.strands import HeadroomBundle
+
+ model = OpenAIModel(
+ model_id="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
+ client_args={"base_url": "http://127.0.0.1:8787/v1", "api_key": "x"},
+ )
+
+ bundle = HeadroomBundle(proxy_url="http://127.0.0.1:8787")
+ agent = Agent(
+ model=model,
+ tools=bundle.tools, # Strands starts the MCP subprocesses on first use
+ hooks=bundle.hooks,
+ )
+ response = agent("Search the codebase for the auth middleware.")
+
+Lifecycle
+---------
+
+The bundle does **not** start the MCP subprocesses itself —
+:class:`strands.tools.mcp.MCPClient` is lazily started by Strands'
+``Agent`` when it loads tools, and stopped when the agent is torn
+down. Construct the bundle, hand its ``tools`` / ``hooks`` to the
+agent, and let Strands own the lifecycle. This matches Strands'
+contract: MCP clients passed via ``tools=[...]`` MUST be unstarted.
+
+The bundle does **not** start the proxy either — it connects to one.
+Production deploys run the proxy as a long-lived service
+(ECS / k8s / EC2); local-dev users start it manually with
+``headroom proxy``. This keeps the bundle stateless and lets the
+proxy scale independently of the agent fleet.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from functools import partial
+from typing import Any
+
+# Strands + MCP SDK imports are required dependencies of this bundle —
+# fail loud on import so a missing dep surfaces at construction time,
+# not three frames deep inside an Agent call.
+from mcp import StdioServerParameters # noqa: E402
+from mcp.client.stdio import stdio_client # noqa: E402
+from strands.tools.mcp import MCPClient # noqa: E402
+
+from headroom import HeadroomConfig
+from headroom.mcp_registry.install import (
+ DEFAULT_PROXY_URL,
+ build_headroom_spec,
+ build_serena_spec,
+)
+
+from .hooks import HeadroomHookProvider
+
+logger = logging.getLogger(__name__)
+
+#: Default Serena context — see https://github.com/oraios/serena for the
+#: full context catalog. ``ide-assistant`` is the closest match for a
+#: code-aware agent loop (the same context ``headroom wrap claude`` uses).
+DEFAULT_SERENA_CONTEXT = "ide-assistant"
+
+
+def _make_headroom_client(proxy_url: str) -> MCPClient:
+ spec = build_headroom_spec(proxy_url)
+ params = StdioServerParameters(
+ command=spec.command,
+ args=list(spec.args),
+ env=dict(spec.env) if spec.env else None,
+ )
+ # `partial` (not lambda) so mypy can infer the callable's signature.
+ return MCPClient(partial(stdio_client, params))
+
+
+def _make_serena_client(context: str) -> MCPClient:
+ spec = build_serena_spec(context)
+ params = StdioServerParameters(
+ command=spec.command,
+ args=list(spec.args),
+ env=dict(spec.env) if spec.env else None,
+ )
+ return MCPClient(partial(stdio_client, params))
+
+
+@dataclass
+class HeadroomBundle:
+ """Single helper that hands a Strands Agent every Headroom integration.
+
+ Attributes:
+ proxy_url: HTTP URL the Headroom MCP server should contact for
+ retrieval. Default :data:`DEFAULT_PROXY_URL`
+ (``http://127.0.0.1:8787``).
+ serena_context: Serena context label. Default ``"ide-assistant"``.
+ enable_headroom_mcp: Include the Headroom MCP server. Default True.
+ enable_serena_mcp: Include the Serena MCP server. Default True.
+ Disabling skips the ``uvx`` first-launch download entirely.
+ enable_hooks: Include :class:`HeadroomHookProvider` for in-place
+ tool-output compression (the RTK-equivalent for Strands).
+ Default True.
+ config: Optional :class:`HeadroomConfig` passed to
+ :class:`HeadroomHookProvider`. Default uses framework
+ defaults.
+
+ The bundle is **stateless** w.r.t. subprocess management — Strands'
+ ``Agent`` owns the MCP subprocess lifecycle once you pass
+ ``bundle.tools`` to it. Constructing a bundle is cheap; the
+ subprocesses don't start until ``Agent`` calls ``load_tools``.
+ """
+
+ proxy_url: str = DEFAULT_PROXY_URL
+ serena_context: str = DEFAULT_SERENA_CONTEXT
+ enable_headroom_mcp: bool = True
+ enable_serena_mcp: bool = True
+ # The proxy is the single source of truth for compression — it sees
+ # the full message list, owns CompressionPolicy, owns PrefixCacheTracker,
+ # and places `cache_control` breakpoints. The in-process hook
+ # (HeadroomHookProvider) is an optimisation for memory/network when
+ # Strands runs on a different host or holds very long conversations.
+ # Default is OFF so the bundle stays "one helper, just the proxy
+ # does the work" for the typical case. Flip on for long-running or
+ # cross-host deploys.
+ enable_hooks: bool = False
+ config: HeadroomConfig | None = None
+
+ _headroom_mcp: MCPClient | None = field(default=None, init=False, repr=False, compare=False)
+ _serena_mcp: MCPClient | None = field(default=None, init=False, repr=False, compare=False)
+ _hook: HeadroomHookProvider | None = field(default=None, init=False, repr=False, compare=False)
+
+ def __post_init__(self) -> None:
+ if self.enable_headroom_mcp:
+ self._headroom_mcp = _make_headroom_client(self.proxy_url)
+ logger.info(
+ "HeadroomBundle: Headroom MCP client constructed (proxy_url=%s)",
+ self.proxy_url,
+ )
+ if self.enable_serena_mcp:
+ self._serena_mcp = _make_serena_client(self.serena_context)
+ logger.info(
+ "HeadroomBundle: Serena MCP client constructed (context=%s)",
+ self.serena_context,
+ )
+ if self.enable_hooks:
+ self._hook = HeadroomHookProvider(config=self.config)
+ logger.info("HeadroomBundle: HeadroomHookProvider attached")
+
+ @property
+ def tools(self) -> list[Any]:
+ """MCP clients to hand to ``Agent(tools=...)``.
+
+ Returned MCPClient instances are **unstarted** — Strands' Agent
+ starts them on first use and stops them on teardown.
+ """
+ out: list[Any] = []
+ if self._headroom_mcp is not None:
+ out.append(self._headroom_mcp)
+ if self._serena_mcp is not None:
+ out.append(self._serena_mcp)
+ return out
+
+ @property
+ def hooks(self) -> list[Any]:
+ """Hook providers to hand to ``Agent(hooks=...)``."""
+ return [self._hook] if self._hook is not None else []
+
+ @property
+ def headroom_mcp(self) -> MCPClient | None:
+ """Direct handle to the Headroom MCPClient (for advanced callers)."""
+ return self._headroom_mcp
+
+ @property
+ def serena_mcp(self) -> MCPClient | None:
+ """Direct handle to the Serena MCPClient (for advanced callers)."""
+ return self._serena_mcp
diff --git a/headroom/proxy/auth_mode.py b/headroom/proxy/auth_mode.py
index 794a24b40..7b90e8cd6 100644
--- a/headroom/proxy/auth_mode.py
+++ b/headroom/proxy/auth_mode.py
@@ -207,6 +207,13 @@ CLIENT_UA_MAP: tuple[tuple[str, str], ...] = (
("github-copilot/", "copilot"),
# Google's experimental harness
("antigravity/", "antigravity"),
+ # AWS Strands Agents SDK. The default openai-python User-Agent
+ # is `OpenAI/Python <ver>` which does not embed any Strands
+ # signal, so production callers should set `X-Client: strands`
+ # explicitly (see `headroom/integrations/strands/README.md`).
+ # The UA prefix below covers any Strands runtime that injects
+ # its own segment ahead of the openai-python UA.
+ ("strands-agents/", "strands"),
)
diff --git a/headroom/proxy/cost.py b/headroom/proxy/cost.py
index 8cc51378a..9120c4583 100644
--- a/headroom/proxy/cost.py
+++ b/headroom/proxy/cost.py
@@ -330,6 +330,57 @@ def merge_cost_stats(
}
+def _aggregate_mcp_events() -> dict[str, int]:
+ """Aggregate compression / retrieval events written by Headroom MCP
+ server instances to the cross-process shared events file.
+
+ The Headroom MCP server (``headroom mcp serve``) records every
+ ``headroom_compress`` and ``headroom_retrieve`` invocation to a
+ file-locked shared log (see :func:`headroom.ccr.mcp_server._append_shared_event`).
+ This helper reads that log and aggregates within the rolling window
+ so the proxy's ``/stats`` can surface MCP-side work alongside the
+ proxy's own HTTP-path compression numbers.
+
+ Returns zeros for every key if the MCP SDK isn't installed, the
+ shared file doesn't exist yet, or any read error occurs — the
+ intent is "if there's nothing to report, report zero" so this
+ helper never blocks the summary.
+
+ Keys: ``compressions`` (count of headroom_compress calls),
+ ``tokens_removed`` (sum of input_tokens-output_tokens across
+ compress events), ``retrievals`` (count of headroom_retrieve
+ calls — the load-bearing over-compression signal).
+ """
+ zero = {"compressions": 0, "tokens_removed": 0, "retrievals": 0}
+ try:
+ from headroom.ccr.mcp_server import _read_shared_events
+ except ImportError:
+ return zero
+
+ try:
+ events = _read_shared_events()
+ except Exception: # noqa: BLE001 — never break /stats on a stats-read error
+ return zero
+
+ compressions = 0
+ tokens_removed = 0
+ retrievals = 0
+ for evt in events:
+ kind = evt.get("type")
+ if kind == "compress":
+ compressions += 1
+ in_tok = int(evt.get("input_tokens", 0) or 0)
+ out_tok = int(evt.get("output_tokens", 0) or 0)
+ tokens_removed += max(0, in_tok - out_tok)
+ elif kind == "retrieve":
+ retrievals += 1
+ return {
+ "compressions": compressions,
+ "tokens_removed": tokens_removed,
+ "retrievals": retrievals,
+ }
+
+
def build_session_summary(
proxy: Any,
metrics: Any,
@@ -450,6 +501,14 @@ def build_session_summary(
},
}
+ # MCP-side compression: events written by `headroom mcp serve`
+ # instances (one or more) to the shared stats log. Surfaces direct
+ # tool invocations the proxy HTTP path never sees, plus the
+ # `retrievals` counter — the load-bearing signal for over-compression
+ # (if it grows linearly with turn count, our lossy compressors are
+ # dropping info the model actually needs).
+ summary["mcp"] = _aggregate_mcp_events()
+
# Add tip if token mode would help
if proxy.config.mode == PROXY_MODE_CACHE and uncompressed_reasons["prefix_frozen"] > 10:
summary["tip"] = (
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 6869527b5..1ce372396 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -1806,6 +1806,8 @@ class OpenAIHandlerMixin:
optimization_latency,
pipeline_timing=pipeline_timing,
waste_signals=waste_signals_dict,
+ prefix_tracker=openai_prefix_tracker,
+ optimized_messages=optimized_messages,
)
else:
# Non-streaming: use send_openai_message() → JSON
@@ -1847,19 +1849,126 @@ class OpenAIHandlerMixin:
content=backend_response.body,
)
- # OpenAI Chat via backend (LiteLLM/AnyLLM), non-
- # streaming. No per-message live-zone tracking yet
- # — use full pre-comp request size as the attempted
- # denominator so this provider's contribution to
- # active_savings is its whole-request ratio.
- # Cache extraction from the backend response body
- # is not wired here yet (follow-up); funnel passes
- # zeros for cache fields, matching pre-refactor
- # behaviour.
+ # CCR Response Handling: intercept headroom_retrieve
+ # tool calls server-side so a Bedrock/LiteLLM
+ # OpenAI-shape response doesn't propagate a tool_call
+ # the downstream caller (e.g. Strands) can't resolve.
+ # Mirrors the Anthropic handler block (anthropic.py
+ # ~1893-2034) but on the OpenAI provider shape.
+ #
+ # NO SILENT FALLBACK: per feedback_no_silent_fallbacks
+ # we re-raise on CCR errors instead of swallowing
+ # them. The Anthropic version still swallows for
+ # legacy reasons; align it in a follow-up.
+ # TODO(#realignment): align anthropic.py CCR block to
+ # re-raise on exception so both providers fail loud.
+ if (
+ self.ccr_response_handler
+ and backend_response.body
+ and backend_response.status_code == 200
+ and self.ccr_response_handler.has_ccr_tool_calls(
+ backend_response.body, "openai"
+ )
+ ):
+ logger.info(
+ f"[{request_id}] CCR: Detected retrieval tool call "
+ f"on backend path, handling via {self.anthropic_backend.name}"
+ )
+
+ # Continuation closure — delegates transport to
+ # the backend abstraction. We strip encoding
+ # headers for safety even though the backend
+ # owns transport (mirrors the Anthropic block).
+ async def api_call_fn(
+ msgs: list[dict[str, Any]],
+ tls: list[dict[str, Any]] | None,
+ ) -> dict[str, Any]:
+ continuation_body = {**body, "messages": msgs}
+ if tls is not None:
+ continuation_body["tools"] = tls
+
+ continuation_headers = {
+ k: v
+ for k, v in headers.items()
+ if k.lower()
+ not in (
+ "content-encoding",
+ "transfer-encoding",
+ "accept-encoding",
+ "content-length",
+ )
+ }
+
+ assert self.anthropic_backend is not None
+ logger.info(
+ f"[{request_id}] CCR: Issuing continuation via "
+ f"{self.anthropic_backend.name} backend "
+ f"({len(msgs)} messages)"
+ )
+ cont_resp = await self.anthropic_backend.send_openai_message(
+ continuation_body, continuation_headers
+ )
+ return cont_resp.body
+
+ try:
+ final_resp_json = await self.ccr_response_handler.handle_response(
+ backend_response.body,
+ optimized_messages,
+ tools,
+ api_call_fn,
+ provider="openai",
+ )
+ backend_response.body = final_resp_json
+ logger.info(
+ f"[{request_id}] CCR: Retrieval handled "
+ "successfully on backend path"
+ )
+ except Exception as e:
+ import traceback
+
+ logger.error(
+ f"[{request_id}] CCR: Response handling failed on "
+ f"backend path: {e}\n"
+ f"Traceback: {traceback.format_exc()}"
+ )
+ # No silent fallback — fail loud per
+ # feedback_no_silent_fallbacks.md.
+ raise
+
+ # Extract usage from the FINAL backend body (after
+ # any CCR resolution) so the prefix tracker counts
+ # cache stats from the LAST upstream call.
total_latency = (time.time() - start_time) * 1000
usage = backend_response.body.get("usage", {})
output_tokens = usage.get("completion_tokens", 0)
total_input_tokens = usage.get("prompt_tokens", optimized_tokens)
+
+ # Cache stats: prefer the Anthropic/Bedrock top-level
+ # keys when present (authoritative). Fall back to
+ # OpenAI's `prompt_tokens_details.cached_tokens` only
+ # if the top-level keys are absent/zero.
+ cache_read_tokens = usage.get("cache_read_input_tokens", 0) or 0
+ cache_creation_input_tokens = usage.get("cache_creation_input_tokens", 0) or 0
+ if cache_read_tokens == 0:
+ prompt_details = usage.get("prompt_tokens_details") or {}
+ cache_read_tokens = prompt_details.get("cached_tokens", 0) or 0
+
+ # Bedrock reports cache creation directly. Only infer
+ # when no explicit count is available.
+ if cache_creation_input_tokens > 0:
+ cache_write_tokens = cache_creation_input_tokens
+ else:
+ cache_write_tokens = _infer_openai_cache_write_tokens(
+ total_input_tokens,
+ cache_read_tokens,
+ )
+
+ openai_prefix_tracker.update_from_response(
+ cache_read_tokens=cache_read_tokens,
+ cache_write_tokens=cache_write_tokens,
+ messages=optimized_messages,
+ )
+
await self._record_request_outcome(
RequestOutcome(
request_id=request_id,
@@ -2688,11 +2797,48 @@ class OpenAIHandlerMixin:
model or "unknown",
)
except Exception as _e:
+ _http_body_bytes = len(json.dumps(body).encode("utf-8", errors="replace"))
logger.warning(
- f"[{request_id}] /v1/responses compression failed; "
- f"forwarding original body: {type(_e).__name__}: {_e}"
+ f"[{request_id}] /v1/responses compression failed "
+ f"(bytes={_http_body_bytes}): {type(_e).__name__}: {_e}"
+ )
+ # Fail-closed protection (default): refuse to forward
+ # oversized requests after compression failure. Same
+ # decision matrix and override env var as the WS path
+ # (HEADROOM_WS_FAIL_OPEN_ON_COMPRESSION_FAILURE) — see
+ # helpers.decide_compression_failure_action.
+ from headroom.proxy.helpers import (
+ decide_compression_failure_action,
)
+ _http_action = decide_compression_failure_action(_e, _http_body_bytes)
+ if _http_action.refuse:
+ logger.error(
+ "[%s] /v1/responses REFUSING to forward request "
+ "after compression failure (reason=%s, bytes=%d); "
+ "returning HTTP 413 so the client can compact "
+ "context and retry. To restore legacy passthrough "
+ "behaviour set "
+ "HEADROOM_WS_FAIL_OPEN_ON_COMPRESSION_FAILURE=1.",
+ request_id,
+ _http_action.reason,
+ _http_action.frame_bytes,
+ )
+ raise HTTPException(
+ status_code=413,
+ detail={
+ "error": {
+ "type": "compression_refused",
+ "message": (
+ f"headroom: compression "
+ f"{_http_action.reason} on a "
+ f"{_http_body_bytes}-byte request "
+ "— please compact context and retry."
+ ),
+ }
+ },
+ ) from _e
+
capture_codex_wire_debug(
"http_upstream_request",
request_id=request_id,
@@ -3794,6 +3940,7 @@ class OpenAIHandlerMixin:
frame_type="unknown",
)
except Exception as _ce:
+ _ws_frame_bytes = len(first_msg_raw.encode("utf-8", errors="replace"))
if _first_frame_compression_elapsed_ms > 0:
record_frame = getattr(
getattr(self, "metrics", None), "record_codex_ws_frame", None
@@ -3801,22 +3948,62 @@ class OpenAIHandlerMixin:
if record_frame is not None:
record_frame(
elapsed_ms=_first_frame_compression_elapsed_ms,
- bytes_before=len(first_msg_raw.encode("utf-8", errors="replace")),
+ bytes_before=_ws_frame_bytes,
failed=True,
)
logger.warning(
- f"[{request_id}] WS /v1/responses compression failed; "
- f"forwarding original frame: {type(_ce).__name__}: {_ce}"
+ f"[{request_id}] WS /v1/responses compression failed "
+ f"(bytes={_ws_frame_bytes}): {type(_ce).__name__}: {_ce}"
)
_log_ws_passthrough(
"compression_exception",
frame_index=1,
- raw_bytes=len(first_msg_raw.encode("utf-8", errors="replace")),
+ raw_bytes=_ws_frame_bytes,
frame_type="response.create" if body else "unknown",
model=str(body.get("model") or "unknown")
if isinstance(body, dict)
else "unknown",
)
+ # Fail-closed protection (default): refuse to forward
+ # oversized frames after a compression failure. Forwarding
+ # the original to the upstream would cause a
+ # context-window-exceeded response that the client
+ # (e.g. Codex) cannot recover from, because Headroom's
+ # earlier successful compressions hid the cumulative
+ # context pressure from the client's auto-compaction
+ # heuristic. Close the client WS with 1009 instead so the
+ # client gets a clear "compact and retry" signal.
+ # See helpers.decide_compression_failure_action for the
+ # decision matrix and env-var overrides.
+ from headroom.proxy.helpers import (
+ decide_compression_failure_action,
+ )
+
+ _ws_action = decide_compression_failure_action(_ce, _ws_frame_bytes)
+ if _ws_action.refuse:
+ logger.error(
+ "[%s] WS /v1/responses REFUSING to forward "
+ "frame after compression failure "
+ "(reason=%s, bytes=%d); closing client "
+ "websocket with 1009 so client can compact "
+ "context and retry. To restore legacy "
+ "passthrough behaviour set "
+ "HEADROOM_WS_FAIL_OPEN_ON_COMPRESSION_FAILURE=1.",
+ request_id,
+ _ws_action.reason,
+ _ws_action.frame_bytes,
+ )
+ termination_cause = "compression_refused"
+ with contextlib.suppress(Exception):
+ await websocket.close(
+ code=1009,
+ reason=(
+ "headroom: compression "
+ f"{_ws_action.reason} — please "
+ "compact context and retry"
+ ),
+ )
+ return
else:
_log_ws_passthrough(
"bypass_header" if _ws_bypass else "optimize_disabled",
diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.py
index 3e76ac53e..08f68d6d2 100644
--- a/headroom/proxy/handlers/streaming.py
+++ b/headroom/proxy/handlers/streaming.py
@@ -564,6 +564,83 @@ class StreamingMixin:
except Exception as e:
logger.debug(f"[{request_id}] CCR Feedback recording failed: {e}")
+ def _record_ccr_feedback_from_openai_sse(self, full_sse_data: str, request_id: str) -> None:
+ """Record headroom_retrieve feedback from OpenAI Chat Completions SSE.
+
+ OpenAI streams tool_calls incrementally via
+ ``choices[0].delta.tool_calls[*].function.arguments`` (chunked
+ JSON string). We accumulate per-call-index and finalize on
+ stream completion. The accumulator records each completed
+ ``headroom_retrieve`` invocation as a no-op store call for the
+ TOIN feedback side effect (matches the Anthropic streaming
+ feedback path).
+ """
+ from headroom.cache.compression_store import get_compression_store
+
+ # tool_call_index -> {"name": str, "args_buf": str}
+ tool_calls: dict[int, dict[str, str]] = {}
+
+ for raw_line in full_sse_data.split("\n"):
+ line = raw_line.strip()
+ if not line.startswith("data: "):
+ continue
+ payload = line[6:]
+ if not payload or payload == "[DONE]":
+ continue
+ try:
+ data = json.loads(payload)
+ except (json.JSONDecodeError, ValueError):
+ continue
+ if not isinstance(data, dict):
+ continue
+
+ choices = data.get("choices") or []
+ if not choices:
+ continue
+ delta = (choices[0] or {}).get("delta") or {}
+ for tc in delta.get("tool_calls") or []:
+ if not isinstance(tc, dict):
+ continue
+ idx = tc.get("index", 0)
+ fn = tc.get("function") or {}
+ slot = tool_calls.setdefault(idx, {"name": "", "args_buf": ""})
+ fn_name = fn.get("name")
+ if fn_name:
+ slot["name"] = fn_name
+ fn_args = fn.get("arguments")
+ if fn_args:
+ slot["args_buf"] = slot["args_buf"] + fn_args
+
+ if not tool_calls:
+ return
+
+ store = get_compression_store()
+ for slot in tool_calls.values():
+ if slot["name"] != "headroom_retrieve":
+ continue
+ try:
+ input_data = json.loads(slot["args_buf"]) if slot["args_buf"] else {}
+ except (json.JSONDecodeError, ValueError):
+ continue
+ if not isinstance(input_data, dict):
+ continue
+ hash_key = input_data.get("hash")
+ query = input_data.get("query")
+ if not hash_key:
+ continue
+
+ logger.info(
+ f"[{request_id}] CCR Feedback (openai stream): Recording retrieval "
+ f"hash={hash_key[:8]}... query={query!r}"
+ )
+ try:
+ if query:
+ store.search(hash_key, query)
+ else:
+ store.retrieve(hash_key, query=None)
+ except Exception as e:
+ logger.debug(f"[{request_id}] CCR Feedback (openai stream) failed: {e}")
+
async def _finalize_stream_response(
self,
*,
@@ -1374,6 +1451,8 @@ class StreamingMixin:
optimization_latency: float,
pipeline_timing: dict[str, float] | None = None,
waste_signals: dict[str, int] | None = None,
+ prefix_tracker: Any | None = None,
+ optimized_messages: list[dict] | None = None,
) -> StreamingResponse:
"""Stream OpenAI chat completion response from backend.
@@ -1389,6 +1468,19 @@ class StreamingMixin:
separate cache-write counter, so the write portion is inferred
via :func:`_infer_openai_cache_write_tokens`. Memory stays O(1)
because the buffer-parser consumes whole events as they arrive.
+
+ ``prefix_tracker``/``optimized_messages`` carry the
+ :class:`PrefixCacheTracker` for the session so cache stats from
+ the FINAL usage frame can update the tracker for the next turn
+ — mirroring the direct streaming path
+ (``_stream_response``/``_finalize_stream_response``).
+
+ NOTE: CCR request-level intercept on the streaming path is
+ intentionally OUT OF SCOPE. Mirrors the Anthropic streaming
+ path, which also does not buffer-and-rewrite mid-stream — doing
+ so would require buffering the full response and would kill the
+ streaming benefit. We do still record CCR retrieval feedback
+ (cheap) for TOIN learning.
"""
from fastapi.responses import StreamingResponse
@@ -1404,12 +1496,23 @@ class StreamingMixin:
"input_tokens": None,
"output_tokens": None,
"cache_read_input_tokens": None,
+ "cache_creation_input_tokens": None,
}
+ # Bytes-level mirror of the SSE stream so we can parse the
+ # final response shape for CCR feedback after the stream
+ # closes (cheap, no buffering of in-flight chunks back to
+ # the client).
+ full_sse_bytes = bytearray()
def _absorb(usage: dict[str, int] | None) -> None:
if not usage:
return
- for key in ("input_tokens", "output_tokens", "cache_read_input_tokens"):
+ for key in (
+ "input_tokens",
+ "output_tokens",
+ "cache_read_input_tokens",
+ "cache_creation_input_tokens",
+ ):
if key in usage and not stream_state.get(key):
stream_state[key] = usage[key]
@@ -1417,6 +1520,7 @@ class StreamingMixin:
async for sse_chunk in self.anthropic_backend.stream_openai_message(body, headers):
chunk_bytes = sse_chunk.encode() if isinstance(sse_chunk, str) else sse_chunk
stream_state["sse_buffer"].extend(chunk_bytes)
+ full_sse_bytes.extend(chunk_bytes)
_absorb(self._parse_sse_usage_from_buffer(stream_state, "openai"))
# Per-chunk fallback for upstreams that emit only
# ``completion_tokens`` and not a full usage frame.
@@ -1454,10 +1558,21 @@ class StreamingMixin:
upstream_input = stream_state["input_tokens"]
output_tokens = stream_state["output_tokens"] or 0
cache_read_tokens = stream_state["cache_read_input_tokens"] or 0
+ # Prefer authoritative cache_creation_input_tokens from
+ # Bedrock/Anthropic shape when present. Fall back to
+ # inferring write count from total - read for OpenAI
+ # shape (which has no separate write counter).
+ cache_creation_input_tokens = stream_state.get("cache_creation_input_tokens") or 0
if upstream_input is None:
cache_write_tokens = 0
uncached_input_tokens = 0
cache_inferred = False
+ elif cache_creation_input_tokens > 0:
+ cache_write_tokens = cache_creation_input_tokens
+ uncached_input_tokens = max(
+ upstream_input - cache_read_tokens - cache_write_tokens, 0
+ )
+ cache_inferred = False
else:
cache_write_tokens = _infer_openai_cache_write_tokens(
upstream_input, cache_read_tokens
@@ -1465,6 +1580,35 @@ class StreamingMixin:
uncached_input_tokens = max(upstream_input - cache_read_tokens, 0)
cache_inferred = True
+ # Update prefix cache tracker for the next turn — mirrors
+ # the non-streaming sibling. Done before outcome funnel
+ # so prefix state is consistent regardless of metric
+ # path.
+ if prefix_tracker is not None:
+ tracker_messages = (
+ optimized_messages
+ if optimized_messages is not None
+ else body.get("messages", [])
+ )
+ prefix_tracker.update_from_response(
+ cache_read_tokens=cache_read_tokens,
+ cache_write_tokens=cache_write_tokens,
+ messages=tracker_messages,
+ )
+
+ # CCR Feedback: record headroom_retrieve tool calls so
+ # TOIN learns which fields matter. Streaming path can't
+ # do request-level intercept (would require buffering
+ # the full stream), so we just close the feedback loop.
+ if self.config.ccr_inject_tool and len(full_sse_bytes) > 0:
+ try:
+ full_sse_data = full_sse_bytes.decode("utf-8", errors="replace")
+ self._record_ccr_feedback_from_openai_sse(full_sse_data, request_id)
+ except Exception as e:
+ logger.debug(
+ f"[{request_id}] CCR feedback recording (openai stream) failed: {e}"
+ )
+
total_latency = (time.time() - start_time) * 1000
# Active-compression denominator for backend-routed
# streaming. No per-message live-zone tracking is wired
diff --git a/headroom/proxy/helpers.py b/headroom/proxy/helpers.py
index 622dc9170..5cbe47a34 100644
--- a/headroom/proxy/helpers.py
+++ b/headroom/proxy/helpers.py
@@ -8,6 +8,7 @@ Extracted from server.py for maintainability.
from __future__ import annotations
+import asyncio
import hashlib
import json
import logging
@@ -17,6 +18,7 @@ import subprocess
import threading
import time
from collections import OrderedDict
+from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING, Any, Literal, cast
@@ -757,6 +759,111 @@ COMPRESSION_TIMEOUT_SECONDS = 30
MAX_COMPRESSION_CACHE_SESSIONS = 500
+# ---------------------------------------------------------------------------
+# Compression-failure escape hatch
+# ---------------------------------------------------------------------------
+# When the proxy's compression stage fails (timeout, exception) on a frame
+# Headroom thought was large enough to compress, the legacy behaviour was to
+# fall through and forward the *original* uncompressed frame to the upstream.
+# That fail-open turned a recoverable timeout into a context-window overflow
+# downstream: Codex's auto-compaction reads ``total_usage_tokens`` from
+# upstream (which Headroom's earlier successful compressions shrunk), then
+# the un-compressed retry overflows the model context and the client
+# locks up.
+#
+# Default behaviour is now fail-CLOSED: refuse to forward, close the client
+# WS with code 1009 (or return HTTP 413) so the client knows to compact and
+# retry. Operators who want the old behaviour can set
+# ``HEADROOM_WS_FAIL_OPEN_ON_COMPRESSION_FAILURE=1``. The oversize threshold
+# below which transient errors still fall through to passthrough is
+# configurable via ``HEADROOM_WS_COMPRESSION_FAIL_THRESHOLD_BYTES``
+# (default 256 KiB ≈ 64K tokens).
+WS_COMPRESSION_FAIL_OPEN_ENV = "HEADROOM_WS_FAIL_OPEN_ON_COMPRESSION_FAILURE"
+WS_COMPRESSION_OVERSIZE_BYTES_ENV = "HEADROOM_WS_COMPRESSION_FAIL_THRESHOLD_BYTES"
+WS_COMPRESSION_OVERSIZE_BYTES_DEFAULT = 256 * 1024
+
+
+@dataclass(frozen=True)
+class CompressionFailureAction:
+ """Decision returned by :func:`decide_compression_failure_action`."""
+
+ refuse: bool
+ """If True, the caller MUST NOT forward the original frame. Close the
+ client connection with a clear error code instead."""
+
+ reason: str
+ """Short machine-readable label for telemetry. One of:
+ ``timeout``, ``oversize:bytes=<n>>threshold=<m>``,
+ ``small_frame_transient``, or ``env_override:fail_open``."""
+
+ frame_bytes: int
+ """Original frame size in bytes (for logging / metrics)."""
+
+
+def decide_compression_failure_action(
+ exception: BaseException,
+ frame_bytes: int,
+) -> CompressionFailureAction:
+ """Decide whether to refuse-and-close vs forward-original after the
+ proxy's compression pipeline fails on a Realtime WebSocket frame
+ (or analogous HTTP body).
+
+ Decision matrix:
+
+ * env :data:`WS_COMPRESSION_FAIL_OPEN_ENV` truthy → forward (legacy
+ behaviour, opt-in for debugging or strict compatibility).
+ * exception is :class:`asyncio.TimeoutError` → refuse (the compression
+ stage hit its own timeout, which only fires on frames Headroom
+ thought were big enough to need compression in the first place).
+ * ``frame_bytes`` > :data:`WS_COMPRESSION_OVERSIZE_BYTES_ENV`
+ (default 256 KiB) → refuse (large + any compression failure is a
+ strong signal the upstream will reject the original).
+ * otherwise → forward (a transient pipeline error on a small frame
+ shouldn't break the request).
+ """
+ fail_open = os.environ.get(WS_COMPRESSION_FAIL_OPEN_ENV, "").strip().lower() in (
+ "1",
+ "true",
+ "yes",
+ "on",
+ )
+ if fail_open:
+ return CompressionFailureAction(
+ refuse=False,
+ reason="env_override:fail_open",
+ frame_bytes=frame_bytes,
+ )
+
+ threshold = WS_COMPRESSION_OVERSIZE_BYTES_DEFAULT
+ raw_threshold = os.environ.get(WS_COMPRESSION_OVERSIZE_BYTES_ENV, "").strip()
+ if raw_threshold:
+ try:
+ parsed = int(raw_threshold)
+ if parsed > 0:
+ threshold = parsed
+ except ValueError:
+ # Operator typo'd the env value — keep the default rather than
+ # raise on every WS frame. Loud warning instead.
+ logger.warning(
+ "Ignoring non-integer %s=%r; using default %d",
+ WS_COMPRESSION_OVERSIZE_BYTES_ENV,
+ raw_threshold,
+ WS_COMPRESSION_OVERSIZE_BYTES_DEFAULT,
+ )
+
+ if isinstance(exception, asyncio.TimeoutError):
+ return CompressionFailureAction(refuse=True, reason="timeout", frame_bytes=frame_bytes)
+ if frame_bytes > threshold:
+ return CompressionFailureAction(
+ refuse=True,
+ reason=f"oversize:bytes={frame_bytes}>threshold={threshold}",
+ frame_bytes=frame_bytes,
+ )
+ return CompressionFailureAction(
+ refuse=False, reason="small_frame_transient", frame_bytes=frame_bytes
+ )
+
+
def jitter_delay_ms(base_ms: int, max_ms: int, attempt: int) -> float:
"""Exponential backoff with 50-150% jitter.
diff --git a/pyproject.toml b/pyproject.toml
index 5da91e4d3..c99333f8d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -81,6 +81,11 @@ code = [
ml = [
"torch>=2.0.0",
"transformers>=4.30.0",
+ # transformers >= 5.x requires huggingface-hub >= 1.5.0,<2.0; pinning
+ # the floor here prevents Kompress from silently falling back to
+ # "unavailable" when a sibling install (e.g. `pip install
+ # strands-agents`) drags huggingface-hub backwards.
+ "huggingface-hub>=1.5.0,<2.0",
]
# Memory system (hierarchical memory with vector search)
memory = [
diff --git a/tests/test_backends/__init__.py b/tests/test_backends/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/test_backends/test_litellm_cache_stats.py b/tests/test_backends/test_litellm_cache_stats.py
new file mode 100644
index 000000000..083a7a5d1
--- /dev/null
+++ b/tests/test_backends/test_litellm_cache_stats.py
@@ -0,0 +1,217 @@
+"""Cache-stat surfacing for `LiteLLMBackend.send_openai_message`.
+
+LiteLLM normalizes prompt-cache statistics onto its `Usage` object from
+multiple upstream dialects:
+
+* Anthropic / Bedrock-Claude → top-level attrs `cache_read_input_tokens`
+ and `cache_creation_input_tokens` (also mirrored into
+ `prompt_tokens_details.cached_tokens` / `cache_creation_tokens`).
+* OpenAI prompt-caching → only `prompt_tokens_details.cached_tokens`.
+
+Before the fix, `send_openai_message` flattened only
+`prompt_tokens / completion_tokens / total_tokens` into the response dict
+and silently dropped all cache stats on the floor — breaking
+`PrefixCacheTracker.update_from_response` for the entire backend-routed
+path (it always saw zero cache hits, so live-zone-only compression never
+engaged).
+
+These tests pin the contract for the three relevant shapes.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from tests._dotenv import importorskip_no_env_leak
+
+importorskip_no_env_leak("litellm")
+
+from headroom.backends.litellm import LiteLLMBackend # noqa: E402 (must follow importorskip)
+
+
+class _FakeUsage:
+ """Stand-in for `litellm.types.utils.Usage`.
+
+ `MagicMock` auto-creates attributes on access, which would defeat the
+ point of the "no cache fields → no keys added" test. A plain object
+ with only the attributes we explicitly set keeps `getattr(..., 0)`
+ honest.
+ """
+
+ def __init__(
+ self,
+ *,
+ prompt_tokens: int,
+ completion_tokens: int,
+ total_tokens: int,
+ cache_read_input_tokens: int | None = None,
+ cache_creation_input_tokens: int | None = None,
+ prompt_tokens_details: Any | None = None,
+ ) -> None:
+ self.prompt_tokens = prompt_tokens
+ self.completion_tokens = completion_tokens
+ self.total_tokens = total_tokens
+ if cache_read_input_tokens is not None:
+ self.cache_read_input_tokens = cache_read_input_tokens
+ if cache_creation_input_tokens is not None:
+ self.cache_creation_input_tokens = cache_creation_input_tokens
+ if prompt_tokens_details is not None:
+ self.prompt_tokens_details = prompt_tokens_details
+
+
+class _FakePromptTokensDetails:
+ """OpenAI-style nested cache shape stand-in."""
+
+ def __init__(
+ self,
+ *,
+ cached_tokens: int | None = None,
+ cache_creation_tokens: int | None = None,
+ ) -> None:
+ if cached_tokens is not None:
+ self.cached_tokens = cached_tokens
+ if cache_creation_tokens is not None:
+ self.cache_creation_tokens = cache_creation_tokens
+
+
+def _make_response(usage: _FakeUsage) -> MagicMock:
+ """Build a minimal `ModelResponse`-shaped mock with the given usage."""
+ response = MagicMock()
+ response.id = "chatcmpl-test"
+ response.created = 1_700_000_000
+ response.choices = [
+ MagicMock(
+ index=0,
+ message=MagicMock(role="assistant", content="hi", tool_calls=None),
+ finish_reason="stop",
+ )
+ ]
+ response.usage = usage
+ return response
+
+
+def _make_backend() -> LiteLLMBackend:
+ # Patch the inference-profile fetch so `__init__` doesn't try to talk to AWS.
+ with patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}):
+ return LiteLLMBackend(provider="openrouter")
+
+
+def _request_body() -> dict[str, Any]:
+ return {
+ "model": "gpt-4",
+ "messages": [{"role": "user", "content": "hello"}],
+ "max_tokens": 32,
+ }
+
+
+# =============================================================================
+# 1. Anthropic-style (top-level cache_read_input_tokens / cache_creation_input_tokens)
+# =============================================================================
+
+
+async def test_anthropic_style_cache_fields_surface_in_usage_block() -> None:
+ """Bedrock-Claude / Anthropic responses set the top-level dialect.
+
+ LiteLLM mirrors them into `prompt_tokens_details` too. Our extractor
+ must prefer the explicit top-level values (cache_read=1500, cache_write=200)
+ and also expose the OpenAI nested shape so single-dialect callers
+ don't have to branch.
+ """
+ usage = _FakeUsage(
+ prompt_tokens=2000,
+ completion_tokens=100,
+ total_tokens=2100,
+ cache_read_input_tokens=1500,
+ cache_creation_input_tokens=200,
+ prompt_tokens_details=_FakePromptTokensDetails(
+ cached_tokens=1500,
+ cache_creation_tokens=200,
+ ),
+ )
+ response = _make_response(usage)
+
+ backend = _make_backend()
+ with patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp:
+ mock_acomp.return_value = response
+ result = await backend.send_openai_message(_request_body(), {})
+
+ body_usage = result.body["usage"]
+ assert body_usage["prompt_tokens"] == 2000
+ assert body_usage["completion_tokens"] == 100
+ assert body_usage["total_tokens"] == 2100
+ assert body_usage["cache_read_input_tokens"] == 1500
+ assert body_usage["cache_creation_input_tokens"] == 200
+ assert body_usage["prompt_tokens_details"] == {"cached_tokens": 1500}
+
+
+# =============================================================================
+# 2. OpenAI-style only (prompt_tokens_details.cached_tokens, no top-level)
+# =============================================================================
+
+
+async def test_openai_nested_cache_fields_surface_when_top_level_absent() -> None:
+ """OpenAI prompt-caching responses only populate the nested dialect.
+
+ With no top-level `cache_read_input_tokens` attribute on the Usage
+ object, we must fall back to `prompt_tokens_details.cached_tokens`
+ and mirror it into the Anthropic-style top-level keys for downstream
+ consumers.
+ """
+ usage = _FakeUsage(
+ prompt_tokens=1200,
+ completion_tokens=50,
+ total_tokens=1250,
+ prompt_tokens_details=_FakePromptTokensDetails(cached_tokens=800),
+ )
+ response = _make_response(usage)
+
+ backend = _make_backend()
+ with patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp:
+ mock_acomp.return_value = response
+ result = await backend.send_openai_message(_request_body(), {})
+
+ body_usage = result.body["usage"]
+ assert body_usage["prompt_tokens"] == 1200
+ assert body_usage["completion_tokens"] == 50
+ assert body_usage["total_tokens"] == 1250
+ assert body_usage["cache_read_input_tokens"] == 800
+ assert body_usage["cache_creation_input_tokens"] == 0
+ assert body_usage["prompt_tokens_details"] == {"cached_tokens": 800}
+
+
+# =============================================================================
+# 3. Cold start — no cache fields anywhere → keep usage_block shape stable
+# =============================================================================
+
+
+async def test_no_cache_fields_means_no_cache_keys_in_usage_block() -> None:
+ """Cold-start path: no cache attributes at all on the Usage object.
+
+ We must NOT inject `cache_read_input_tokens`, `cache_creation_input_tokens`,
+ or `prompt_tokens_details` into `usage_block` — keep the dict shape
+ identical to the pre-fix behaviour so callers that key off presence
+ (rather than value) don't accidentally start seeing 0 as "we have
+ cache data, the model just didn't cache".
+ """
+ usage = _FakeUsage(
+ prompt_tokens=500,
+ completion_tokens=25,
+ total_tokens=525,
+ )
+ response = _make_response(usage)
+
+ backend = _make_backend()
+ with patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp:
+ mock_acomp.return_value = response
+ result = await backend.send_openai_message(_request_body(), {})
+
+ body_usage = result.body["usage"]
+ assert body_usage == {
+ "prompt_tokens": 500,
+ "completion_tokens": 25,
+ "total_tokens": 525,
+ }
+ assert "cache_read_input_tokens" not in body_usage
+ assert "cache_creation_input_tokens" not in body_usage
+ assert "prompt_tokens_details" not in body_usage
diff --git a/tests/test_proxy/test_compression_failure_action.py b/tests/test_proxy/test_compression_failure_action.py
new file mode 100644
index 000000000..389bbfc77
--- /dev/null
+++ b/tests/test_proxy/test_compression_failure_action.py
@@ -0,0 +1,155 @@
+"""Decision matrix for what the proxy does when compression fails.
+
+Driven by Camille's bug report (2026-05-21): Codex threads were
+locking with "ran out of room in the model's context window" because
+the WS /v1/responses handler forwarded the *original* uncompressed
+frame on compression timeout (fail-open). The upstream then rejected
+the oversized frame, and Codex's auto-compaction never fired because
+its ``total_usage_tokens`` heuristic had been hidden from cumulative
+context pressure by Headroom's earlier successful compressions.
+
+These tests pin :func:`headroom.proxy.helpers.decide_compression_failure_action`
+so the matrix is reviewable in one place and regressions are loud.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+from collections.abc import Iterator
+from contextlib import contextmanager
+
+import pytest
+
+from headroom.proxy.helpers import (
+ WS_COMPRESSION_FAIL_OPEN_ENV,
+ WS_COMPRESSION_OVERSIZE_BYTES_DEFAULT,
+ WS_COMPRESSION_OVERSIZE_BYTES_ENV,
+ decide_compression_failure_action,
+)
+
+
+@contextmanager
+def _env(**overrides: str | None) -> Iterator[None]:
+ """Temporarily set / unset env vars for a single test."""
+ saved: dict[str, str | None] = {}
+ for key, value in overrides.items():
+ saved[key] = os.environ.get(key)
+ if value is None:
+ os.environ.pop(key, None)
+ else:
+ os.environ[key] = value
+ try:
+ yield
+ finally:
+ for key, prior in saved.items():
+ if prior is None:
+ os.environ.pop(key, None)
+ else:
+ os.environ[key] = prior
+
+
+def test_timeout_always_refuses_regardless_of_frame_size() -> None:
+ """asyncio.TimeoutError → refuse, even for a tiny frame.
+
+ Compression timeout fires after ``COMPRESSION_TIMEOUT_SECONDS``, which
+ means the pipeline already started work on the frame. A small frame
+ that nevertheless timed out is a strong "something is wrong" signal —
+ safer to surface to the client than to forward.
+ """
+ with _env(**{WS_COMPRESSION_FAIL_OPEN_ENV: None, WS_COMPRESSION_OVERSIZE_BYTES_ENV: None}):
+ action = decide_compression_failure_action(asyncio.TimeoutError(), frame_bytes=128)
+ assert action.refuse is True
+ assert action.reason == "timeout"
+ assert action.frame_bytes == 128
+
+
+def test_small_transient_error_falls_through_to_passthrough() -> None:
+ """Non-timeout error on a small frame: forward original (legacy)."""
+ with _env(**{WS_COMPRESSION_FAIL_OPEN_ENV: None, WS_COMPRESSION_OVERSIZE_BYTES_ENV: None}):
+ action = decide_compression_failure_action(
+ RuntimeError("transient pipeline glitch"),
+ frame_bytes=4 * 1024,
+ )
+ assert action.refuse is False
+ assert action.reason == "small_frame_transient"
+ assert action.frame_bytes == 4 * 1024
+
+
+def test_oversize_frame_any_error_refuses() -> None:
+ """Non-timeout error on a large frame: refuse — upstream would reject."""
+ big = WS_COMPRESSION_OVERSIZE_BYTES_DEFAULT + 1024
+ with _env(**{WS_COMPRESSION_FAIL_OPEN_ENV: None, WS_COMPRESSION_OVERSIZE_BYTES_ENV: None}):
+ action = decide_compression_failure_action(
+ RuntimeError("compressor crashed"), frame_bytes=big
+ )
+ assert action.refuse is True
+ assert action.reason.startswith("oversize:")
+ assert str(big) in action.reason
+
+
+ "value",
+ ["1", "true", "yes", "on", "TRUE", "Yes"],
+)
+def test_env_fail_open_overrides_everything(value: str) -> None:
+ """Operator opt-in fail-open: don't refuse, even on timeout."""
+ big = WS_COMPRESSION_OVERSIZE_BYTES_DEFAULT + 1
+ with _env(**{WS_COMPRESSION_FAIL_OPEN_ENV: value, WS_COMPRESSION_OVERSIZE_BYTES_ENV: None}):
+ action = decide_compression_failure_action(asyncio.TimeoutError(), frame_bytes=big)
+ assert action.refuse is False
+ assert action.reason == "env_override:fail_open"
+
+
+def test_custom_threshold_via_env() -> None:
+ """Operator can lower the oversize threshold per environment."""
+ with _env(
+ **{
+ WS_COMPRESSION_FAIL_OPEN_ENV: None,
+ WS_COMPRESSION_OVERSIZE_BYTES_ENV: "1024",
+ }
+ ):
+ # Frame above the custom 1 KiB threshold → refuse
+ action = decide_compression_failure_action(RuntimeError(), frame_bytes=2048)
+ assert action.refuse is True
+ assert "threshold=1024" in action.reason
+
+ with _env(
+ **{
+ WS_COMPRESSION_FAIL_OPEN_ENV: None,
+ WS_COMPRESSION_OVERSIZE_BYTES_ENV: "1024",
+ }
+ ):
+ # Frame at/below the custom threshold → passthrough
+ action_small = decide_compression_failure_action(RuntimeError(), frame_bytes=512)
+ assert action_small.refuse is False
+ assert action_small.reason == "small_frame_transient"
+
+
+def test_invalid_threshold_env_falls_back_to_default() -> None:
+ """A typo'd env var must not blow up every WS frame."""
+ big = WS_COMPRESSION_OVERSIZE_BYTES_DEFAULT + 1
+ with _env(
+ **{
+ WS_COMPRESSION_FAIL_OPEN_ENV: None,
+ WS_COMPRESSION_OVERSIZE_BYTES_ENV: "not-an-int",
+ }
+ ):
+ action = decide_compression_failure_action(RuntimeError(), frame_bytes=big)
+ assert action.refuse is True
+ assert f"threshold={WS_COMPRESSION_OVERSIZE_BYTES_DEFAULT}" in action.reason
+
+
+def test_threshold_zero_or_negative_ignored() -> None:
+ """A bogus 0 / negative threshold must NOT silently disable the gate."""
+ big = WS_COMPRESSION_OVERSIZE_BYTES_DEFAULT + 1
+ with _env(
+ **{
+ WS_COMPRESSION_FAIL_OPEN_ENV: None,
+ WS_COMPRESSION_OVERSIZE_BYTES_ENV: "0",
+ }
+ ):
+ action = decide_compression_failure_action(RuntimeError(), frame_bytes=big)
+ # default threshold still applies
+ assert action.refuse is True
+ assert f"threshold={WS_COMPRESSION_OVERSIZE_BYTES_DEFAULT}" in action.reason
diff --git a/tests/test_proxy/test_mcp_stats_aggregation.py b/tests/test_proxy/test_mcp_stats_aggregation.py
new file mode 100644
index 000000000..9b9c4d7dc
--- /dev/null
+++ b/tests/test_proxy/test_mcp_stats_aggregation.py
@@ -0,0 +1,95 @@
+"""Coverage for the MCP-events aggregator inside the proxy /stats summary.
+
+``headroom mcp serve`` writes every compress / retrieve invocation to a
+shared file-locked log (``_append_shared_event``). Before this fix,
+``/stats`` only reported proxy-HTTP-path compressions and silently
+ignored the MCP tool work — which is exactly where Strands-style
+agents spend most of their compression budget when the LLM calls
+``headroom_compress`` directly.
+
+These tests pin the aggregation logic in
+:func:`headroom.proxy.cost._aggregate_mcp_events`.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+from unittest.mock import patch
+
+from headroom.proxy.cost import _aggregate_mcp_events
+
+
+def _mock_events(events: list[dict[str, Any]]) -> Any:
+ """Return a patch target that makes _read_shared_events yield ``events``."""
+ return patch(
+ "headroom.ccr.mcp_server._read_shared_events",
+ return_value=events,
+ )
+
+
+def test_aggregates_compress_and_retrieve_events() -> None:
+ events = [
+ {"type": "compress", "input_tokens": 1000, "output_tokens": 400},
+ {"type": "compress", "input_tokens": 500, "output_tokens": 300},
+ {"type": "retrieve", "hash": "abc123"},
+ {"type": "retrieve", "hash": "def456"},
+ {"type": "retrieve", "hash": "ghi789"},
+ ]
+ with _mock_events(events):
+ result = _aggregate_mcp_events()
+ assert result == {
+ "compressions": 2,
+ # (1000-400) + (500-300) = 600 + 200 = 800
+ "tokens_removed": 800,
+ "retrievals": 3,
+ }
+
+
+def test_returns_zeros_when_no_events() -> None:
+ with _mock_events([]):
+ assert _aggregate_mcp_events() == {
+ "compressions": 0,
+ "tokens_removed": 0,
+ "retrievals": 0,
+ }
+
+
+def test_unknown_event_types_are_ignored() -> None:
+ events = [
+ {"type": "compress", "input_tokens": 100, "output_tokens": 40},
+ {"type": "unknown_future_kind", "anything": "goes"},
+ {"type": "retrieve", "hash": "abc"},
+ {"missing_type": True}, # malformed; should skip
+ ]
+ with _mock_events(events):
+ result = _aggregate_mcp_events()
+ assert result == {"compressions": 1, "tokens_removed": 60, "retrievals": 1}
+
+
+def test_missing_token_fields_default_to_zero_without_raising() -> None:
+ events = [
+ {"type": "compress"}, # no token fields at all
+ {"type": "compress", "input_tokens": 200, "output_tokens": None}, # None coerced
+ {"type": "compress", "input_tokens": 100, "output_tokens": 100}, # zero diff
+ ]
+ with _mock_events(events):
+ result = _aggregate_mcp_events()
+ assert result["compressions"] == 3
+ # First contributes 0 (both missing); second contributes 0 (None→0,
+ # and output 0 → input - output = 200... wait that's NOT zero); third 0.
+ # Let me re-derive: max(0, 0 - 0) + max(0, 200 - 0) + max(0, 100 - 100)
+ # = 0 + 200 + 0 = 200
+ assert result["tokens_removed"] == 200
+
+
+def test_read_failure_yields_zeros() -> None:
+ """If the shared-stats reader raises, the aggregator must not crash /stats."""
+ with patch(
+ "headroom.ccr.mcp_server._read_shared_events",
+ side_effect=OSError("disk on fire"),
+ ):
+ assert _aggregate_mcp_events() == {
+ "compressions": 0,
+ "tokens_removed": 0,
+ "retrievals": 0,
+ }
diff --git a/tests/test_proxy/test_openai_backend_path.py b/tests/test_proxy/test_openai_backend_path.py
new file mode 100644
index 000000000..ccae9370e
--- /dev/null
+++ b/tests/test_proxy/test_openai_backend_path.py
@@ -0,0 +1,367 @@
+"""Tests for the OpenAI chat-completions backend (LiteLLM/Bedrock) path.
+
+Covers Fix #1 (PrefixCacheTracker.update_from_response on backend path)
+and Fix #2 (CCR response intercept for the OpenAI provider shape) on the
+non-streaming backend path of ``handle_openai_chat``.
+
+All three scenarios mock ``anthropic_backend.send_openai_message`` so we
+don't need a real provider:
+
+1. Backend response with cache_read_input_tokens > 0 → tracker.update_from_response
+ is called with the right cache_read_tokens and cache_write_tokens.
+2. Backend response with headroom_retrieve tool call → ccr_response_handler.handle_response
+ is awaited with provider="openai", and the final body returned.
+3. CCR intercept exception path → re-raises (NOT swallowed).
+"""
+
+from __future__ import annotations
+
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+fastapi = pytest.importorskip("fastapi")
+httpx = pytest.importorskip("httpx")
+
+from fastapi.testclient import TestClient # noqa: E402
+
+from headroom.backends.base import BackendResponse # noqa: E402
+from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
+
+
+class _RecordingTracker:
+ """Stub PrefixCacheTracker that records ``update_from_response`` calls."""
+
+ def __init__(self, provider: str = "openai") -> None:
+ self.provider = provider
+ self.calls: list[dict] = []
+ self._frozen = 0
+ self._last_original: list[dict] = []
+ self._last_forwarded: list[dict] = []
+
+ def update_from_response(
+ self,
+ cache_read_tokens: int,
+ cache_write_tokens: int,
+ messages: list[dict],
+ message_token_counts: list[int] | None = None,
+ original_messages: list[dict] | None = None,
+ ) -> None:
+ self.calls.append(
+ {
+ "cache_read_tokens": cache_read_tokens,
+ "cache_write_tokens": cache_write_tokens,
+ "messages": messages,
+ }
+ )
+ self._last_original = list(original_messages or messages)
+ self._last_forwarded = list(messages)
+
+ # Minimal surface used by handle_openai_chat — return 0 so we never freeze.
+ def get_frozen_message_count(self) -> int:
+ return self._frozen
+
+ def get_last_original_messages(self) -> list[dict]:
+ return list(self._last_original)
+
+ def get_last_forwarded_messages(self) -> list[dict]:
+ return list(self._last_forwarded)
+
+
+def _make_config() -> ProxyConfig:
+ return ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ backend="anyllm",
+ anyllm_provider="openai",
+ )
+
+
+def _make_mock_backend(response_body: dict, status_code: int = 200) -> MagicMock:
+ backend = MagicMock()
+ backend.name = "anyllm-openai"
+ backend.send_openai_message = AsyncMock(
+ return_value=BackendResponse(
+ body=response_body,
+ status_code=status_code,
+ headers={"content-type": "application/json"},
+ )
+ )
+ return backend
+
+
+def _install_tracker_stub(client: TestClient) -> _RecordingTracker:
+ """Force the session_tracker_store to hand out our recording tracker."""
+ tracker = _RecordingTracker(provider="openai")
+ # Find the proxy instance behind the app — it's stored as app.state.proxy.
+ proxy = client.app.state.proxy
+ proxy.session_tracker_store.get_or_create = MagicMock(return_value=tracker)
+ return tracker
+
+
+def test_backend_response_updates_prefix_tracker_with_bedrock_cache_fields():
+ """Bedrock/Anthropic-shape cache fields → tracker sees authoritative read/write counts."""
+ config = _make_config()
+ response_body = {
+ "id": "chatcmpl-bedrock-1",
+ "object": "chat.completion",
+ "model": "anthropic.claude-3-5-sonnet-20241022-v2:0",
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": "Hi!"},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": {
+ "prompt_tokens": 1000,
+ "completion_tokens": 20,
+ "total_tokens": 1020,
+ # Bedrock/Anthropic top-level keys
+ "cache_read_input_tokens": 700,
+ "cache_creation_input_tokens": 100,
+ # OpenAI shape (always populated by the LiteLLM normalizer)
+ "prompt_tokens_details": {"cached_tokens": 700},
+ },
+ }
+
+ mock_backend = _make_mock_backend(response_body)
+ with patch("headroom.proxy.server.AnyLLMBackend", return_value=mock_backend):
+ app = create_app(config)
+ with TestClient(app) as client:
+ tracker = _install_tracker_stub(client)
+ resp = client.post(
+ "/v1/chat/completions",
+ json={
+ "model": "anthropic.claude-3-5-sonnet-20241022-v2:0",
+ "messages": [{"role": "user", "content": "hi"}],
+ "stream": False,
+ },
+ headers={"Authorization": "Bearer test-key"},
+ )
+
+ assert resp.status_code == 200, resp.text
+ assert mock_backend.send_openai_message.await_count == 1
+ assert len(tracker.calls) == 1, tracker.calls
+ call = tracker.calls[0]
+ # Prefer the Bedrock authoritative top-level read/write counts.
+ assert call["cache_read_tokens"] == 700
+ assert call["cache_write_tokens"] == 100
+
+
+def test_backend_response_falls_back_to_openai_cached_tokens_when_bedrock_keys_absent():
+ """Pure OpenAI shape (no top-level Anthropic keys) → fall back to prompt_tokens_details + infer write."""
+ config = _make_config()
+ response_body = {
+ "id": "chatcmpl-openai-1",
+ "object": "chat.completion",
+ "model": "gpt-4o-mini",
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": "Hi!"},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": {
+ "prompt_tokens": 500,
+ "completion_tokens": 10,
+ "total_tokens": 510,
+ # No top-level Anthropic keys, only OpenAI shape
+ "prompt_tokens_details": {"cached_tokens": 200},
+ },
+ }
+
+ mock_backend = _make_mock_backend(response_body)
+ with patch("headroom.proxy.server.AnyLLMBackend", return_value=mock_backend):
+ app = create_app(config)
+ with TestClient(app) as client:
+ tracker = _install_tracker_stub(client)
+ resp = client.post(
+ "/v1/chat/completions",
+ json={
+ "model": "gpt-4o-mini",
+ "messages": [{"role": "user", "content": "hi"}],
+ "stream": False,
+ },
+ headers={"Authorization": "Bearer test-key"},
+ )
+
+ assert resp.status_code == 200, resp.text
+ assert len(tracker.calls) == 1
+ call = tracker.calls[0]
+ assert call["cache_read_tokens"] == 200
+ # No cache_creation_input_tokens → inferred = prompt_tokens - cache_read = 500 - 200 = 300
+ assert call["cache_write_tokens"] == 300
+
+
+def test_backend_response_with_ccr_tool_call_is_intercepted_and_resolved():
+ """OpenAI-shape response carrying headroom_retrieve → CCR handler resolves it."""
+ config = _make_config()
+ # First response: tool_call for headroom_retrieve
+ tool_call_response = {
+ "id": "chatcmpl-ccr-1",
+ "object": "chat.completion",
+ "model": "anthropic.claude-3-5-sonnet-20241022-v2:0",
+ "choices": [
+ {
+ "index": 0,
+ "message": {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "call_abc",
+ "type": "function",
+ "function": {
+ "name": "headroom_retrieve",
+ "arguments": '{"hash": "deadbeef"}',
+ },
+ }
+ ],
+ },
+ "finish_reason": "tool_calls",
+ }
+ ],
+ "usage": {
+ "prompt_tokens": 100,
+ "completion_tokens": 10,
+ "total_tokens": 110,
+ },
+ }
+ final_resp_json = {
+ "id": "chatcmpl-ccr-final",
+ "object": "chat.completion",
+ "model": "anthropic.claude-3-5-sonnet-20241022-v2:0",
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": "Resolved!"},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": {
+ "prompt_tokens": 100,
+ "completion_tokens": 5,
+ "total_tokens": 105,
+ },
+ }
+
+ mock_backend = _make_mock_backend(tool_call_response)
+ with patch("headroom.proxy.server.AnyLLMBackend", return_value=mock_backend):
+ app = create_app(config)
+ with TestClient(app) as client:
+ _install_tracker_stub(client)
+ proxy = client.app.state.proxy
+ # Replace the response handler with a recording mock.
+ recording_handler = MagicMock()
+ recording_handler.has_ccr_tool_calls = MagicMock(return_value=True)
+ recording_handler.handle_response = AsyncMock(return_value=final_resp_json)
+ proxy.ccr_response_handler = recording_handler
+
+ resp = client.post(
+ "/v1/chat/completions",
+ json={
+ "model": "anthropic.claude-3-5-sonnet-20241022-v2:0",
+ "messages": [{"role": "user", "content": "hi"}],
+ "stream": False,
+ },
+ headers={"Authorization": "Bearer test-key"},
+ )
+
+ assert resp.status_code == 200, resp.text
+ # handle_response was awaited with provider="openai"
+ recording_handler.handle_response.assert_awaited_once()
+ _args, kwargs = recording_handler.handle_response.call_args
+ assert kwargs.get("provider") == "openai"
+ # Resolved body propagated back to the client
+ assert resp.json()["choices"][0]["message"]["content"] == "Resolved!"
+
+
+def test_backend_ccr_intercept_exception_is_reraised_not_swallowed():
+ """CCR resolution failure on the backend path → 500, NOT silent fallback to original body."""
+ config = _make_config()
+ tool_call_response = {
+ "id": "chatcmpl-ccr-fail",
+ "object": "chat.completion",
+ "model": "anthropic.claude-3-5-sonnet-20241022-v2:0",
+ "choices": [
+ {
+ "index": 0,
+ "message": {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "call_bad",
+ "type": "function",
+ "function": {
+ "name": "headroom_retrieve",
+ "arguments": '{"hash": "badhash"}',
+ },
+ }
+ ],
+ },
+ "finish_reason": "tool_calls",
+ }
+ ],
+ "usage": {"prompt_tokens": 50, "completion_tokens": 5, "total_tokens": 55},
+ }
+
+ mock_backend = _make_mock_backend(tool_call_response)
+ with patch("headroom.proxy.server.AnyLLMBackend", return_value=mock_backend):
+ app = create_app(config)
+ with TestClient(app) as client:
+ _install_tracker_stub(client)
+ proxy = client.app.state.proxy
+ failing_handler = MagicMock()
+ failing_handler.has_ccr_tool_calls = MagicMock(return_value=True)
+ failing_handler.handle_response = AsyncMock(
+ side_effect=RuntimeError("ccr-store-blew-up")
+ )
+ proxy.ccr_response_handler = failing_handler
+
+ resp = client.post(
+ "/v1/chat/completions",
+ json={
+ "model": "anthropic.claude-3-5-sonnet-20241022-v2:0",
+ "messages": [{"role": "user", "content": "hi"}],
+ "stream": False,
+ },
+ headers={"Authorization": "Bearer test-key"},
+ )
+
+ # The outer `try/except Exception` on the backend block converts the
+ # re-raise into a 500 response. The critical assertion is that the
+ # original tool_call body is NOT returned to the client — which is
+ # what a silent fallback would do.
+ failing_handler.handle_response.assert_awaited_once()
+ assert resp.status_code == 500, (
+ f"expected 500 (CCR error re-raised), got {resp.status_code}: {resp.text[:200]}"
+ )
+ body = resp.json()
+ # Confirm we didn't propagate the original tool_call body.
+ assert (
+ "choices" not in body
+ or body.get("choices", [{}])[0].get("message", {}).get("tool_calls") is None
+ )
+ assert "error" in body
+ assert "ccr-store-blew-up" in body["error"]["message"]
+
+
+def test_backend_streaming_passes_prefix_tracker_through():
+ """Streaming backend path should accept and use prefix_tracker — non-regression smoke."""
+ # The wiring contract is structural — just confirm the parameter exists.
+ import inspect
+
+ from headroom.proxy.handlers.streaming import StreamingMixin
+
+ sig = inspect.signature(StreamingMixin._stream_openai_via_backend)
+ assert "prefix_tracker" in sig.parameters, (
+ "_stream_openai_via_backend must accept prefix_tracker to match the direct path"
+ )
+ assert "optimized_messages" in sig.parameters, (
+ "_stream_openai_via_backend must accept optimized_messages so the "
+ "tracker can record the messages that were sent"
+ )