Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

Introducing headroom wrap

chopratejas8f438b0674f7
diff --git a/benchmarks/prefix_cache_benchmark.py b/benchmarks/prefix_cache_benchmark.py
new file mode 100644
index 000000000..519490a07
--- /dev/null
+++ b/benchmarks/prefix_cache_benchmark.py
@@ -0,0 +1,772 @@
+#!/usr/bin/env python3
+"""
+Prefix Cache Strategy Benchmark — Real API Calls
+
+Sends a 25-turn conversation through 4 different caching strategies and measures
+actual cache_read_input_tokens vs cache_creation_input_tokens from the Anthropic API.
+
+Strategies:
+ 1. Baseline — no Headroom, no markers
+ 2. Headroom compression — full pipeline, CompressionCache keeps bytes stable
+ 3. Headroom + prefix freeze — pipeline skips frozen (already-cached) messages
+ 4. Headroom + explicit markers — pipeline + 4 cache_control breakpoints
+
+Usage:
+ # Load API key from .env and run
+ source .env && python benchmarks/prefix_cache_benchmark.py
+
+ # Quick test with fewer turns
+ source .env && python benchmarks/prefix_cache_benchmark.py --turns 5
+
+ # With specific model
+ source .env && python benchmarks/prefix_cache_benchmark.py --model claude-sonnet-4-6
+
+Estimated cost: ~$0.50-1.00 total across all strategies.
+"""
+
+from __future__ import annotations
+
+import argparse
+import copy
+import json
+import os
+import sys
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+import httpx
+
+# ---------------------------------------------------------------------------
+# Pricing (per token)
+# ---------------------------------------------------------------------------
+PRICING = {
+ "claude-sonnet-4-6": {
+ "input": 3.00 / 1_000_000,
+ "output": 15.00 / 1_000_000,
+ "cache_read": 0.30 / 1_000_000,
+ "cache_write": 3.75 / 1_000_000,
+ },
+ "claude-haiku-4-5-20251001": {
+ "input": 0.80 / 1_000_000,
+ "output": 4.00 / 1_000_000,
+ "cache_read": 0.08 / 1_000_000,
+ "cache_write": 1.00 / 1_000_000,
+ },
+}
+
+# ---------------------------------------------------------------------------
+# Data classes
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class TurnMetrics:
+ turn: int
+ cache_read_tokens: int = 0
+ cache_creation_tokens: int = 0
+ input_tokens: int = 0
+ output_tokens: int = 0
+ cost_usd: float = 0.0
+
+
+@dataclass
+class StrategyResult:
+ name: str
+ turns: list[TurnMetrics] = field(default_factory=list)
+
+ @property
+ def total_input(self) -> int:
+ return sum(
+ t.input_tokens + t.cache_read_tokens + t.cache_creation_tokens for t in self.turns
+ )
+
+ @property
+ def total_cache_read(self) -> int:
+ return sum(t.cache_read_tokens for t in self.turns)
+
+ @property
+ def total_cache_write(self) -> int:
+ return sum(t.cache_creation_tokens for t in self.turns)
+
+ @property
+ def total_output(self) -> int:
+ return sum(t.output_tokens for t in self.turns)
+
+ @property
+ def total_cost(self) -> float:
+ return sum(t.cost_usd for t in self.turns)
+
+ @property
+ def cache_hit_rate(self) -> float:
+ total = (
+ self.total_cache_read + self.total_cache_write + sum(t.input_tokens for t in self.turns)
+ )
+ return (self.total_cache_read / total * 100) if total > 0 else 0.0
+
+
+# ---------------------------------------------------------------------------
+# Conversation builder
+# ---------------------------------------------------------------------------
+
+SYSTEM_PROMPT = """You are an expert software engineering assistant. You help users debug code,
+analyze logs, query databases, and search codebases. You have access to several tools.
+
+When analyzing data, be thorough but concise. Focus on anomalies, errors, and actionable insights.
+Always explain your reasoning step by step.
+
+Important guidelines:
+- When you see error patterns, highlight them immediately
+- For database queries, suggest optimizations if the result set is large
+- For code analysis, focus on potential bugs and security issues
+- Always provide actionable next steps
+
+You are working in a large Python monorepo with FastAPI services, PostgreSQL databases,
+and Redis caching. The codebase uses pytest for testing and has CI/CD via GitHub Actions."""
+
+TOOLS = [
+ {
+ "name": "search_codebase",
+ "description": "Search the codebase for patterns, function definitions, or references.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "query": {"type": "string", "description": "Search pattern or keyword"},
+ "file_pattern": {"type": "string", "description": "Glob pattern for files"},
+ },
+ "required": ["query"],
+ },
+ },
+ {
+ "name": "read_file",
+ "description": "Read the contents of a file.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "file_path": {"type": "string", "description": "Path to the file"},
+ "offset": {"type": "integer", "description": "Line offset to start from"},
+ "limit": {"type": "integer", "description": "Number of lines to read"},
+ },
+ "required": ["file_path"],
+ },
+ },
+ {
+ "name": "query_database",
+ "description": "Execute a read-only SQL query against the application database.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "query": {"type": "string", "description": "SQL SELECT query"},
+ "database": {"type": "string", "description": "Database name"},
+ },
+ "required": ["query"],
+ },
+ },
+ {
+ "name": "search_logs",
+ "description": "Search application logs for patterns within a time range.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "pattern": {"type": "string", "description": "Log pattern to search"},
+ "service": {"type": "string", "description": "Service name"},
+ "hours": {"type": "integer", "description": "Hours to look back"},
+ },
+ "required": ["pattern"],
+ },
+ },
+]
+
+USER_QUERIES = [
+ "Can you search for all usages of the `authenticate_user` function?",
+ "Read the file src/auth/middleware.py so I can understand the auth flow.",
+ "Query the database for failed login attempts in the last hour: SELECT user_id, attempt_time, error_code FROM auth_logs WHERE status='failed' AND attempt_time > NOW() - INTERVAL '1 hour' ORDER BY attempt_time DESC LIMIT 50",
+ "Search the logs for 'ConnectionRefused' errors in the auth-service from the past 2 hours.",
+ "Read src/auth/token_validator.py — I think the bug might be there.",
+ "Search for all files that import from `auth.middleware`.",
+ "Query for users who had more than 5 failed attempts: SELECT user_id, COUNT(*) as fails FROM auth_logs WHERE status='failed' AND attempt_time > NOW() - INTERVAL '24 hours' GROUP BY user_id HAVING COUNT(*) > 5",
+ "Search logs for 'JWT expired' in auth-service.",
+ "Read the test file tests/test_auth.py to see what's covered.",
+ "Search for `rate_limit` in the codebase.",
+ "Read src/config/settings.py to check the rate limit configuration.",
+ "Query the metrics table: SELECT endpoint, avg_latency_ms, p99_latency_ms, error_rate FROM api_metrics WHERE timestamp > NOW() - INTERVAL '1 hour' ORDER BY error_rate DESC",
+ "Search logs for any 5xx errors across all services.",
+ "Read src/api/routes.py to check the endpoint definitions.",
+ "Search for usages of the Redis cache client.",
+ "Read src/cache/redis_client.py for the connection pooling setup.",
+ "Query cache hit rates: SELECT cache_key_prefix, hit_count, miss_count, hit_count::float/(hit_count+miss_count) as hit_rate FROM cache_stats WHERE period='hourly' ORDER BY miss_count DESC LIMIT 20",
+ "Search logs for 'cache eviction' warnings.",
+ "Read the Dockerfile to check the base image version.",
+ "Search for any TODO or FIXME comments in the auth module.",
+ "Read .github/workflows/ci.yml for the CI pipeline config.",
+ "Query deployment history: SELECT version, deployed_at, deployed_by, status FROM deployments WHERE service='auth-service' ORDER BY deployed_at DESC LIMIT 10",
+ "Search for error handling patterns — look for bare `except:` blocks.",
+ "Read src/auth/oauth.py for the OAuth integration.",
+ "Search logs for memory usage spikes in the last 4 hours.",
+]
+
+# Fake tool responses (JSON data that would come from tools)
+TOOL_RESPONSES = {
+ "search_codebase": lambda q: json.dumps(
+ [
+ {
+ "file": f"src/auth/{f}.py",
+ "line": 10 + i * 5,
+ "match": f"def authenticate_user(request): # {q}",
+ }
+ for i, f in enumerate(["middleware", "token_validator", "oauth", "session", "utils"])
+ ]
+ + [
+ {
+ "file": f"tests/test_{f}.py",
+ "line": 20 + i * 3,
+ "match": f"from auth.middleware import {q.split()[0] if q.split() else 'auth'}",
+ }
+ for i, f in enumerate(["auth", "api", "cache"])
+ ]
+ ),
+ "read_file": lambda q: "# File contents (simulated)\nimport logging\nfrom typing import Optional\n\nlogger = logging.getLogger(__name__)\n\n"
+ + "\n".join(
+ [
+ f"def function_{i}(arg: str) -> Optional[dict]:\n \"\"\"Process {q}.\"\"\"\n result = {{}}\n for key in ['id', 'name', 'status']:\n result[key] = f'value_{{key}}_{{arg}}'\n logger.info(f'Processed {{arg}}')\n return result\n"
+ for i in range(8)
+ ]
+ ),
+ "query_database": lambda q: json.dumps(
+ [
+ {
+ "user_id": f"user_{i:04d}",
+ "attempt_time": f"2025-01-15T10:{i:02d}:00Z",
+ "error_code": [
+ "INVALID_PASSWORD",
+ "EXPIRED_TOKEN",
+ "RATE_LIMITED",
+ "ACCOUNT_LOCKED",
+ ][i % 4],
+ "status": "failed",
+ }
+ for i in range(15)
+ ]
+ ),
+ "search_logs": lambda q: "\n".join(
+ [
+ f"2025-01-15T10:{i:02d}:{j:02d}Z [ERROR] auth-service: {q} - connection to db-primary:5432 refused (attempt {j + 1}/3)"
+ for i in range(5)
+ for j in range(3)
+ ]
+ ),
+}
+
+
+def build_turn_messages(
+ turn_idx: int,
+ history: list[dict[str, Any]],
+) -> tuple[list[dict[str, Any]], str]:
+ """Build messages for a specific turn, return (messages, user_query)."""
+ query = USER_QUERIES[turn_idx % len(USER_QUERIES)]
+ messages = list(history) + [{"role": "user", "content": query}]
+ return messages, query
+
+
+# ---------------------------------------------------------------------------
+# API call helper
+# ---------------------------------------------------------------------------
+
+
+def call_anthropic(
+ api_key: str,
+ model: str,
+ messages: list[dict[str, Any]],
+ tools: list[dict] | None = None,
+ max_tokens: int = 100,
+) -> dict[str, Any]:
+ """Make a real Anthropic API call, return the full response JSON."""
+ # Separate system from messages (Anthropic format)
+ system_content = None
+ api_messages = []
+ for msg in messages:
+ if msg["role"] == "system":
+ system_content = msg["content"]
+ else:
+ api_messages.append(msg)
+
+ body: dict[str, Any] = {
+ "model": model,
+ "max_tokens": max_tokens,
+ "messages": api_messages,
+ }
+ if system_content:
+ body["system"] = system_content
+ if tools:
+ body["tools"] = tools
+
+ headers = {
+ "x-api-key": api_key,
+ "anthropic-version": "2023-06-01",
+ "content-type": "application/json",
+ }
+
+ with httpx.Client(timeout=60) as client:
+ resp = client.post(
+ "https://api.anthropic.com/v1/messages",
+ json=body,
+ headers=headers,
+ )
+ resp.raise_for_status()
+ return resp.json()
+
+
+def extract_metrics(resp: dict, turn: int, pricing: dict) -> TurnMetrics:
+ """Extract cache metrics from Anthropic response."""
+ usage = resp.get("usage", {})
+ cr = usage.get("cache_read_input_tokens", 0)
+ cw = usage.get("cache_creation_input_tokens", 0)
+ inp = usage.get("input_tokens", 0)
+ out = usage.get("output_tokens", 0)
+
+ cost = (
+ cr * pricing["cache_read"]
+ + cw * pricing["cache_write"]
+ + inp * pricing["input"]
+ + out * pricing["output"]
+ )
+
+ return TurnMetrics(
+ turn=turn,
+ cache_read_tokens=cr,
+ cache_creation_tokens=cw,
+ input_tokens=inp,
+ output_tokens=out,
+ cost_usd=cost,
+ )
+
+
+def extract_assistant_content(resp: dict) -> dict[str, Any]:
+ """Convert Anthropic response to a message dict for conversation history."""
+ content = resp.get("content", [])
+ # Check for tool use
+ has_tool_use = any(b.get("type") == "tool_use" for b in content if isinstance(b, dict))
+
+ if has_tool_use:
+ return {"role": "assistant", "content": content}
+ else:
+ # Extract text
+ text = ""
+ for block in content:
+ if isinstance(block, dict) and block.get("type") == "text":
+ text += block.get("text", "")
+ return {"role": "assistant", "content": text}
+
+
+def make_tool_result(assistant_msg: dict) -> list[dict[str, Any]]:
+ """Generate fake tool results for any tool_use blocks in the assistant message."""
+ results = []
+ content = assistant_msg.get("content", [])
+ if not isinstance(content, list):
+ return results
+
+ for block in content:
+ if isinstance(block, dict) and block.get("type") == "tool_use":
+ tool_name = block.get("name", "search_codebase")
+ tool_id = block.get("id", "")
+ query = json.dumps(block.get("input", {}))
+
+ gen = TOOL_RESPONSES.get(tool_name, TOOL_RESPONSES["search_codebase"])
+ fake_output = gen(query)
+
+ results.append(
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "tool_result",
+ "tool_use_id": tool_id,
+ "content": fake_output,
+ }
+ ],
+ }
+ )
+ return results
+
+
+# ---------------------------------------------------------------------------
+# Strategy: inject explicit cache_control markers
+# ---------------------------------------------------------------------------
+
+
+def inject_cache_markers(
+ system_content: str | None,
+ api_messages: list[dict[str, Any]],
+) -> tuple[str | list | None, list[dict[str, Any]]]:
+ """Inject up to 4 cache_control breakpoints at strategic positions.
+
+ Marker 1: End of system prompt
+ Marker 2: ~1/3 through messages
+ Marker 3: ~2/3 through messages
+ Marker 4: Last message
+ """
+ # Marker 1: system prompt
+ if system_content and isinstance(system_content, str):
+ system_content = [
+ {"type": "text", "text": system_content, "cache_control": {"type": "ephemeral"}}
+ ]
+
+ if not api_messages:
+ return system_content, api_messages
+
+ msgs = copy.deepcopy(api_messages)
+ n = len(msgs)
+
+ # Pick positions for markers 2-4 (indices into msgs)
+ positions = set()
+ if n >= 3:
+ positions.add(n // 3) # Marker 2: ~1/3
+ positions.add(2 * n // 3) # Marker 3: ~2/3
+ positions.add(n - 1) # Marker 4: last message
+
+ markers_placed = 1 # Already placed marker 1 on system
+ for pos in sorted(positions):
+ if markers_placed >= 4:
+ break
+ msg = msgs[pos]
+ content = msg.get("content")
+
+ if isinstance(content, str):
+ msg["content"] = [
+ {"type": "text", "text": content, "cache_control": {"type": "ephemeral"}}
+ ]
+ markers_placed += 1
+ elif isinstance(content, list) and content:
+ last_block = content[-1]
+ if isinstance(last_block, dict):
+ last_block["cache_control"] = {"type": "ephemeral"}
+ markers_placed += 1
+
+ return system_content, msgs
+
+
+# ---------------------------------------------------------------------------
+# Run a full conversation for one strategy
+# ---------------------------------------------------------------------------
+
+
+def inject_cc_style_markers(
+ system_content: str | None,
+ api_messages: list[dict[str, Any]],
+) -> tuple[str | list | None, list[dict[str, Any]]]:
+ """Simulate Claude Code's caching strategy.
+
+ Claude Code places cache_control on:
+ - The system prompt (stable, always cached)
+ - The last ~2 user/assistant messages (growing prefix)
+ This uses 2-3 of the 4 available breakpoints.
+ """
+ # Marker on system prompt
+ if system_content and isinstance(system_content, str):
+ system_content = [
+ {"type": "text", "text": system_content, "cache_control": {"type": "ephemeral"}}
+ ]
+
+ if not api_messages:
+ return system_content, api_messages
+
+ msgs = copy.deepcopy(api_messages)
+ n = len(msgs)
+
+ # Marker on last message (the new user query)
+ markers_placed = 1 # system already has one
+ if n >= 1 and markers_placed < 4:
+ msg = msgs[-1]
+ content = msg.get("content")
+ if isinstance(content, str):
+ msg["content"] = [
+ {"type": "text", "text": content, "cache_control": {"type": "ephemeral"}}
+ ]
+ markers_placed += 1
+ elif isinstance(content, list) and content:
+ last_block = content[-1]
+ if isinstance(last_block, dict):
+ last_block["cache_control"] = {"type": "ephemeral"}
+ markers_placed += 1
+
+ # Marker on second-to-last user message (if exists)
+ if n >= 3 and markers_placed < 4:
+ # Find second-to-last user message
+ for i in range(n - 2, -1, -1):
+ if msgs[i].get("role") == "user":
+ content = msgs[i].get("content")
+ if isinstance(content, str):
+ msgs[i]["content"] = [
+ {"type": "text", "text": content, "cache_control": {"type": "ephemeral"}}
+ ]
+ markers_placed += 1
+ elif isinstance(content, list) and content:
+ last_block = content[-1]
+ if isinstance(last_block, dict):
+ last_block["cache_control"] = {"type": "ephemeral"}
+ markers_placed += 1
+ break
+
+ return system_content, msgs
+
+
+# ---------------------------------------------------------------------------
+# Caching mode enum
+# ---------------------------------------------------------------------------
+CACHE_MODE_NONE = "none"
+CACHE_MODE_CC_STYLE = "cc_style" # Claude Code's strategy
+CACHE_MODE_EXPLICIT = "explicit_4" # Headroom's 4 strategic breakpoints
+
+
+def run_strategy(
+ name: str,
+ api_key: str,
+ model: str,
+ num_turns: int,
+ pricing: dict,
+ use_tools: bool = True,
+ cache_mode: str = CACHE_MODE_NONE,
+ delay: float = 1.0,
+) -> StrategyResult:
+ """Run a full multi-turn conversation and collect cache metrics."""
+ result = StrategyResult(name=name)
+
+ history: list[dict[str, Any]] = [{"role": "system", "content": SYSTEM_PROMPT}]
+ tools = TOOLS if use_tools else None
+
+ for turn in range(num_turns):
+ # Build messages for this turn
+ query = USER_QUERIES[turn % len(USER_QUERIES)]
+ history.append({"role": "user", "content": query})
+
+ # Prepare API call
+ system_content: str | list | None = None
+ api_messages: list[dict[str, Any]] = []
+ for msg in history:
+ if msg["role"] == "system":
+ system_content = msg["content"]
+ else:
+ api_messages.append(msg)
+
+ # Apply caching strategy
+ if cache_mode == CACHE_MODE_CC_STYLE:
+ system_content, api_messages = inject_cc_style_markers(system_content, api_messages)
+ elif cache_mode == CACHE_MODE_EXPLICIT:
+ system_content, api_messages = inject_cache_markers(system_content, api_messages)
+
+ # Build request body
+ body: dict[str, Any] = {
+ "model": model,
+ "max_tokens": 100,
+ "messages": api_messages,
+ }
+ if system_content:
+ body["system"] = system_content if isinstance(system_content, list) else system_content
+ if tools:
+ body["tools"] = tools
+
+ headers = {
+ "x-api-key": api_key,
+ "anthropic-version": "2023-06-01",
+ "content-type": "application/json",
+ }
+
+ # Make the API call
+ try:
+ with httpx.Client(timeout=60) as client:
+ resp = client.post(
+ "https://api.anthropic.com/v1/messages",
+ json=body,
+ headers=headers,
+ )
+ resp.raise_for_status()
+ resp_json = resp.json()
+ except Exception as e:
+ print(f" [!] Turn {turn + 1} failed: {e}")
+ break
+
+ # Extract metrics
+ metrics = extract_metrics(resp_json, turn + 1, pricing)
+ result.turns.append(metrics)
+
+ total_cached = (
+ metrics.cache_read_tokens + metrics.cache_creation_tokens + metrics.input_tokens
+ )
+ hit_pct = (metrics.cache_read_tokens / total_cached * 100) if total_cached > 0 else 0
+
+ print(
+ f" Turn {turn + 1:2d}: "
+ f"read={metrics.cache_read_tokens:6d} "
+ f"write={metrics.cache_creation_tokens:6d} "
+ f"input={metrics.input_tokens:5d} "
+ f"hit={hit_pct:5.1f}% "
+ f"${metrics.cost_usd:.4f}"
+ )
+
+ # Add assistant response to history
+ assistant_msg = extract_assistant_content(resp_json)
+ history.append(assistant_msg)
+
+ # If assistant used tools, add fake tool results
+ tool_results = make_tool_result(assistant_msg)
+ history.extend(tool_results)
+
+ # Delay to let cache settle (Anthropic needs the first response to complete
+ # before subsequent requests can hit the cache)
+ if delay > 0:
+ time.sleep(delay)
+
+ return result
+
+
+# ---------------------------------------------------------------------------
+# Report
+# ---------------------------------------------------------------------------
+
+
+def print_report(results: list[StrategyResult], num_turns: int) -> None:
+ """Print comparison report."""
+ print()
+ print("=" * 72)
+ print(f" Prefix Cache Strategy Benchmark ({num_turns} turns)")
+ print("=" * 72)
+
+ baseline_cost = results[0].total_cost if results else 0
+
+ for r in results:
+ total = r.total_cache_read + r.total_cache_write + sum(t.input_tokens for t in r.turns)
+ hit_rate = (r.total_cache_read / total * 100) if total > 0 else 0
+
+ print(f"\n Strategy: {r.name}")
+ print(f" Total prompt tokens: {total:>10,}")
+ print(f" Cache reads (hit): {r.total_cache_read:>10,} ({hit_rate:.1f}%)")
+ print(f" Cache writes (miss): {r.total_cache_write:>10,}")
+ print(f" Output tokens: {r.total_output:>10,}")
+ print(f" Total cost: ${r.total_cost:>9.4f}")
+ if baseline_cost > 0 and r is not results[0]:
+ savings = (1 - r.total_cost / baseline_cost) * 100
+ print(f" Savings vs baseline: {savings:>9.1f}%")
+
+ # Per-turn hit rate table
+ print("\n Per-turn cache hit rate:")
+ header = " Turn |"
+ for r in results:
+ short_name = r.name[:12].ljust(12)
+ header += f" {short_name} |"
+ print(header)
+ print(" " + "-" * (len(header) - 2))
+
+ for turn_idx in range(num_turns):
+ row = f" {turn_idx + 1:4d} |"
+ for r in results:
+ if turn_idx < len(r.turns):
+ t = r.turns[turn_idx]
+ total = t.cache_read_tokens + t.cache_creation_tokens + t.input_tokens
+ hit = (t.cache_read_tokens / total * 100) if total > 0 else 0
+ row += f" {hit:>10.1f}% |"
+ else:
+ row += f" {'N/A':>10s} |"
+ print(row)
+
+ print()
+ print("=" * 72)
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+
+def main():
+ parser = argparse.ArgumentParser(description="Prefix cache strategy benchmark")
+ parser.add_argument(
+ "--turns", type=int, default=15, help="Number of conversation turns (default: 15)"
+ )
+ parser.add_argument("--model", type=str, default="claude-sonnet-4-6", help="Model to use")
+ parser.add_argument("--delay", type=float, default=1.5, help="Delay between turns (seconds)")
+ parser.add_argument(
+ "--strategies",
+ nargs="+",
+ default=["all"],
+ choices=["baseline", "cc", "markers", "all"],
+ help="Which strategies to run (default: all)",
+ )
+ args = parser.parse_args()
+
+ api_key = os.environ.get("ANTHROPIC_API_KEY")
+ if not api_key:
+ print("Error: Set ANTHROPIC_API_KEY environment variable")
+ print(" source .env && python benchmarks/prefix_cache_benchmark.py")
+ sys.exit(1)
+
+ model = args.model
+ pricing = PRICING.get(model, PRICING["claude-sonnet-4-6"])
+
+ strategies_to_run = set(args.strategies)
+ if "all" in strategies_to_run:
+ strategies_to_run = {"baseline", "cc", "markers"}
+
+ num_strategies = len(strategies_to_run)
+ print(f"Prefix Cache Benchmark: {args.turns} turns, model={model}")
+ print(f"Strategies: {', '.join(sorted(strategies_to_run))}")
+ print(f"Estimated cost: ~${args.turns * num_strategies * 0.01:.2f}")
+ print()
+
+ results: list[StrategyResult] = []
+ step = 0
+
+ # Strategy 1: Baseline (no markers, no caching at all)
+ if "baseline" in strategies_to_run:
+ step += 1
+ print(f"[{step}/{num_strategies}] Baseline (no markers, no caching)...")
+ r = run_strategy(
+ "No Cache",
+ api_key,
+ model,
+ args.turns,
+ pricing,
+ cache_mode=CACHE_MODE_NONE,
+ delay=args.delay,
+ )
+ results.append(r)
+ print()
+
+ # Strategy 2: Claude Code-style (system + last 2 messages)
+ if "cc" in strategies_to_run:
+ step += 1
+ print(f"[{step}/{num_strategies}] Claude Code-style (system + last 2 msgs)...")
+ r = run_strategy(
+ "CC-Style",
+ api_key,
+ model,
+ args.turns,
+ pricing,
+ cache_mode=CACHE_MODE_CC_STYLE,
+ delay=args.delay,
+ )
+ results.append(r)
+ print()
+
+ # Strategy 3: Headroom explicit markers (4 strategic breakpoints)
+ if "markers" in strategies_to_run:
+ step += 1
+ print(f"[{step}/{num_strategies}] Headroom explicit (4 strategic breakpoints)...")
+ r = run_strategy(
+ "Headroom 4x",
+ api_key,
+ model,
+ args.turns,
+ pricing,
+ cache_mode=CACHE_MODE_EXPLICIT,
+ delay=args.delay,
+ )
+ results.append(r)
+ print()
+
+ # Report
+ if results:
+ print_report(results, args.turns)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/headroom/cache/__init__.py b/headroom/cache/__init__.py
index 78400a539..65fa912fc 100644
--- a/headroom/cache/__init__.py
+++ b/headroom/cache/__init__.py
@@ -45,6 +45,7 @@ from .dynamic_detector import (
)
from .google import GoogleCacheOptimizer
from .openai import OpenAICacheOptimizer
+from .prefix_tracker import FreezeStats, PrefixCacheTracker, PrefixFreezeConfig, SessionTrackerStore
from .registry import CacheOptimizerRegistry
from .semantic import SemanticCache, SemanticCacheLayer
@@ -73,4 +74,9 @@ __all__ = [
# Semantic caching
"SemanticCacheLayer",
"SemanticCache",
+ # Prefix cache tracking
+ "PrefixCacheTracker",
+ "PrefixFreezeConfig",
+ "FreezeStats",
+ "SessionTrackerStore",
]
diff --git a/headroom/cache/prefix_tracker.py b/headroom/cache/prefix_tracker.py
new file mode 100644
index 000000000..65e420549
--- /dev/null
+++ b/headroom/cache/prefix_tracker.py
@@ -0,0 +1,316 @@
+"""Prefix Cache Tracker — session-scoped state for cache-aware compression.
+
+Tracks provider prefix cache state between turns so the transform pipeline
+can freeze already-cached messages and only compress new content.
+
+Problem: Clients like Claude Code already manage prefix caching (up to 4
+cache_control breakpoints, growing-prefix strategy). If Headroom compresses
+or modifies messages in the cached prefix, it invalidates the cache —
+replacing a 90% read discount (Anthropic) or 50% (OpenAI) with a 25%
+write penalty.
+
+Solution: After each API response, record how many tokens the provider
+cached. On the next turn, freeze that many messages so the transform
+pipeline skips them entirely.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import time
+from dataclasses import dataclass
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+# Provider cache economics for cost comparisons
+_PROVIDER_READ_DISCOUNT = {
+ "anthropic": 0.9, # 90% discount on reads
+ "openai": 0.5, # 50% discount on reads
+ "gemini": 0.9,
+ "bedrock": 0.9,
+}
+
+_PROVIDER_WRITE_PENALTY = {
+ "anthropic": 0.25, # 25% surcharge on writes
+ "openai": 0.0, # No write penalty
+ "gemini": 0.0,
+ "bedrock": 0.25,
+}
+
+
+@dataclass
+class PrefixFreezeConfig:
+ """Configuration for cache-aware prefix freezing."""
+
+ enabled: bool = True
+ min_cached_tokens: int = 1024 # Min cached tokens to activate freeze
+ session_ttl_seconds: int = 600 # Session tracker cleanup TTL
+ force_compress_threshold: float = 0.5 # Bust cache if compression saves > this fraction
+
+
+@dataclass
+class FreezeStats:
+ """Statistics from prefix freezing for metrics/dashboard."""
+
+ busts_avoided: int = 0
+ tokens_preserved: int = 0
+ compression_foregone_tokens: int = 0
+ net_benefit_tokens: int = 0 # tokens_preserved - compression_foregone
+ frozen_message_count: int = 0
+ turn_number: int = 0
+
+
+class PrefixCacheTracker:
+ """Tracks provider prefix cache state across turns in a session.
+
+ Usage:
+ tracker = PrefixCacheTracker("anthropic")
+
+ # Before compression (turn 2+):
+ frozen = tracker.get_frozen_message_count()
+ result = pipeline.apply(messages, model, frozen_message_count=frozen)
+
+ # After API response:
+ tracker.update_from_response(
+ cache_read_tokens=usage["cache_read_input_tokens"],
+ cache_write_tokens=usage["cache_creation_input_tokens"],
+ messages=optimized_messages,
+ tokenizer=tokenizer,
+ )
+ """
+
+ def __init__(self, provider: str, config: PrefixFreezeConfig | None = None):
+ self.provider = provider
+ self.config = config or PrefixFreezeConfig()
+ self._cached_token_count: int = 0
+ self._cached_message_count: int = 0
+ self._turn_number: int = 0
+ self._last_activity: float = time.time()
+
+ # Stats
+ self._busts_avoided: int = 0
+ self._tokens_preserved: int = 0
+ self._compression_foregone_tokens: int = 0
+
+ def get_frozen_message_count(self) -> int:
+ """How many leading messages to skip compression on the next turn.
+
+ Returns 0 on turn 0 (cold start) or if caching is disabled/below threshold.
+ """
+ if not self.config.enabled:
+ return 0
+ if self._turn_number == 0:
+ return 0
+ if self._cached_token_count < self.config.min_cached_tokens:
+ return 0
+ return self._cached_message_count
+
+ def update_from_response(
+ self,
+ cache_read_tokens: int,
+ cache_write_tokens: int,
+ messages: list[dict[str, Any]],
+ message_token_counts: list[int] | None = None,
+ ) -> None:
+ """Update tracker with cache metrics from the API response.
+
+ Called after every API call. Computes how many messages to freeze
+ on the next turn based on the cache_read_tokens reported.
+
+ Args:
+ cache_read_tokens: Tokens read from cache (cache hit portion).
+ cache_write_tokens: Tokens written to cache (new cache entries).
+ messages: The messages that were sent to the API.
+ message_token_counts: Pre-computed token counts per message.
+ If None, estimates from content length.
+ """
+ self._last_activity = time.time()
+ self._turn_number += 1
+
+ # Compute total cached tokens (read + write = what's in cache now)
+ total_cached = cache_read_tokens + cache_write_tokens
+
+ if total_cached == 0:
+ self._cached_token_count = 0
+ self._cached_message_count = 0
+ return
+
+ # Estimate per-message token counts if not provided
+ if message_token_counts is None:
+ message_token_counts = self._estimate_message_tokens(messages)
+
+ # Walk messages from the start, accumulating tokens until we exceed
+ # the cached amount. All messages within the cached prefix are frozen.
+ accumulated = 0
+ frozen_count = 0
+ for i, tok_count in enumerate(message_token_counts):
+ accumulated += tok_count
+ if accumulated <= total_cached:
+ frozen_count = i + 1
+ else:
+ break
+
+ self._cached_token_count = total_cached
+ self._cached_message_count = frozen_count
+
+ logger.debug(
+ "PrefixCacheTracker[%s]: turn=%d, cached=%d tokens, "
+ "frozen=%d/%d messages (read=%d, write=%d)",
+ self.provider,
+ self._turn_number,
+ total_cached,
+ frozen_count,
+ len(messages),
+ cache_read_tokens,
+ cache_write_tokens,
+ )
+
+ def record_bust_avoided(self, tokens_preserved: int, compression_foregone: int) -> None:
+ """Record when we chose to preserve cache over compressing."""
+ self._busts_avoided += 1
+ self._tokens_preserved += tokens_preserved
+ self._compression_foregone_tokens += compression_foregone
+
+ def should_force_compress(
+ self,
+ message_index: int,
+ message_tokens: int,
+ estimated_compressed_tokens: int,
+ ) -> bool:
+ """Check if compression savings outweigh cache preservation.
+
+ Returns True if we should bust the cache and compress anyway.
+ This happens when compression would save a large fraction of tokens
+ AND the savings exceed the cache read discount.
+ """
+ if message_index >= self._cached_message_count:
+ return True # Not in frozen prefix, always compress
+
+ if message_tokens == 0:
+ return False
+
+ savings_fraction = (message_tokens - estimated_compressed_tokens) / message_tokens
+
+ # Would compression savings exceed the cache read discount?
+ read_discount = _PROVIDER_READ_DISCOUNT.get(self.provider, 0.5)
+ return savings_fraction > read_discount
+
+ @property
+ def is_expired(self) -> bool:
+ """Check if this tracker has been idle beyond TTL."""
+ return (time.time() - self._last_activity) > self.config.session_ttl_seconds
+
+ @property
+ def stats(self) -> FreezeStats:
+ """Return stats for dashboard/metrics."""
+ return FreezeStats(
+ busts_avoided=self._busts_avoided,
+ tokens_preserved=self._tokens_preserved,
+ compression_foregone_tokens=self._compression_foregone_tokens,
+ net_benefit_tokens=self._tokens_preserved - self._compression_foregone_tokens,
+ frozen_message_count=self._cached_message_count,
+ turn_number=self._turn_number,
+ )
+
+ @staticmethod
+ def _estimate_message_tokens(messages: list[dict[str, Any]]) -> list[int]:
+ """Rough token count per message (chars / 3.5)."""
+ counts = []
+ for msg in messages:
+ content = msg.get("content", "")
+ if isinstance(content, str):
+ chars = len(content)
+ elif isinstance(content, list):
+ chars = sum(
+ len(str(block.get("text", ""))) for block in content if isinstance(block, dict)
+ )
+ else:
+ chars = 0
+ # Add overhead for role, tool_use blocks, etc.
+ chars += 20
+ counts.append(max(1, int(chars / 3.5)))
+ return counts
+
+
+class SessionTrackerStore:
+ """Manages PrefixCacheTracker instances across sessions.
+
+ Keyed by session ID (from x-headroom-session-id header or computed hash).
+ Automatically cleans up expired sessions.
+ """
+
+ def __init__(self, default_config: PrefixFreezeConfig | None = None):
+ self._trackers: dict[str, PrefixCacheTracker] = {}
+ self._default_config = default_config or PrefixFreezeConfig()
+ self._last_cleanup: float = time.time()
+ self._cleanup_interval: float = 60.0 # Cleanup every 60s
+
+ def get_or_create(self, session_id: str, provider: str) -> PrefixCacheTracker:
+ """Get existing tracker or create a new one for this session."""
+ self._maybe_cleanup()
+
+ if session_id in self._trackers:
+ tracker = self._trackers[session_id]
+ tracker._last_activity = time.time()
+ return tracker
+
+ tracker = PrefixCacheTracker(provider, self._default_config)
+ self._trackers[session_id] = tracker
+ return tracker
+
+ def compute_session_id(
+ self,
+ request: Any,
+ model: str,
+ messages: list[dict[str, Any]],
+ ) -> str:
+ """Compute a session ID from the request.
+
+ Priority:
+ 1. x-headroom-session-id header (explicit)
+ 2. Hash of (model + system prompt) — stable per conversation
+ """
+ # Check for explicit session header
+ if hasattr(request, "headers"):
+ session_header = request.headers.get("x-headroom-session-id")
+ if session_header:
+ return str(session_header)
+
+ # Fall back to hashing model + system prompt
+ system_content = ""
+ for msg in messages:
+ if msg.get("role") == "system":
+ content = msg.get("content", "")
+ if isinstance(content, str):
+ system_content = content[:500] # First 500 chars is enough
+ elif isinstance(content, list):
+ for block in content:
+ if isinstance(block, dict) and block.get("type") == "text":
+ system_content = block.get("text", "")[:500]
+ break
+ break
+
+ key = f"{model}:{system_content}"
+ return hashlib.sha256(key.encode()).hexdigest()[:16]
+
+ def _maybe_cleanup(self) -> None:
+ """Remove expired trackers periodically."""
+ now = time.time()
+ if now - self._last_cleanup < self._cleanup_interval:
+ return
+
+ expired = [sid for sid, tracker in self._trackers.items() if tracker.is_expired]
+ for sid in expired:
+ del self._trackers[sid]
+
+ if expired:
+ logger.debug("SessionTrackerStore: cleaned up %d expired sessions", len(expired))
+
+ self._last_cleanup = now
+
+ @property
+ def active_sessions(self) -> int:
+ """Number of active session trackers."""
+ return len(self._trackers)
diff --git a/headroom/cli/main.py b/headroom/cli/main.py
index 659dd9a88..753c6bf13 100644
--- a/headroom/cli/main.py
+++ b/headroom/cli/main.py
@@ -40,6 +40,7 @@ def _register_commands() -> None:
memory, # noqa: F401
perf, # noqa: F401
proxy, # noqa: F401
+ wrap, # noqa: F401
)
diff --git a/headroom/cli/wrap.py b/headroom/cli/wrap.py
new file mode 100644
index 000000000..9928d7df2
--- /dev/null
+++ b/headroom/cli/wrap.py
@@ -0,0 +1,215 @@
+"""Wrap CLI commands to run through Headroom proxy.
+
+Usage:
+ headroom wrap claude # Start proxy + rtk + claude
+ headroom wrap claude --no-rtk # Without rtk hooks
+ headroom wrap claude --port 9999 # Custom proxy port
+ headroom wrap claude -- --model opus # Pass args to claude
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import signal
+import socket
+import subprocess
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+import click
+
+from .main import main
+
+# Proxy health check (reused from evals/suite_runner.py pattern)
+
+
+def _check_proxy(port: int) -> bool:
+ """Check if Headroom proxy is running on given port."""
+ try:
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+ s.settimeout(1)
+ s.connect(("127.0.0.1", port))
+ return True
+ except (TimeoutError, ConnectionRefusedError, OSError):
+ return False
+
+
+def _start_proxy(port: int) -> subprocess.Popen:
+ """Start Headroom proxy as a background subprocess."""
+ cmd = [sys.executable, "-m", "headroom.cli", "proxy", "--port", str(port)]
+
+ proc = subprocess.Popen(
+ cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ )
+
+ # Wait for proxy to be ready (up to 15 seconds)
+ for _i in range(15):
+ time.sleep(1)
+ if _check_proxy(port):
+ return proc
+ # Check if process died
+ if proc.poll() is not None:
+ stderr = proc.stderr.read().decode() if proc.stderr else ""
+ raise RuntimeError(f"Proxy exited with code {proc.returncode}: {stderr[:500]}")
+
+ proc.kill()
+ raise RuntimeError(f"Proxy failed to start on port {port} within 15 seconds")
+
+
+def _setup_rtk(verbose: bool = False) -> Path | None:
+ """Ensure rtk is installed and hooks are registered."""
+ from headroom.rtk import get_rtk_path
+ from headroom.rtk.installer import ensure_rtk, register_claude_hooks
+
+ rtk_path = get_rtk_path()
+
+ if rtk_path:
+ if verbose:
+ click.echo(f" rtk found at {rtk_path}")
+ else:
+ click.echo(" Downloading rtk (Rust Token Killer)...")
+ rtk_path = ensure_rtk()
+ if rtk_path:
+ click.echo(f" rtk installed at {rtk_path}")
+ else:
+ click.echo(" rtk download failed — continuing without it")
+ return None
+
+ # Register hooks (idempotent)
+ if register_claude_hooks(rtk_path):
+ if verbose:
+ click.echo(" rtk hooks registered in Claude Code")
+ else:
+ click.echo(" rtk hook registration failed — continuing without it")
+
+ return rtk_path
+
+
+def wrap() -> None:
+ """Wrap CLI tools to run through Headroom.
+
+ \b
+ Starts a Headroom proxy, configures the environment, and launches
+ the target tool so all API calls route through Headroom automatically.
+
+ \b
+ Example:
+ headroom wrap claude # Proxy + rtk + Claude Code
+ headroom wrap claude --no-rtk # Proxy only, no rtk hooks
+ headroom wrap claude -- -p # Pass --print flag to claude
+ """
+
+
[email protected]("--port", "-p", default=8787, type=int, help="Proxy port (default: 8787)")
[email protected]("--no-rtk", is_flag=True, help="Skip rtk installation and hook registration")
[email protected]("--no-proxy", is_flag=True, help="Skip proxy startup (use existing proxy)")
[email protected]("--verbose", "-v", is_flag=True, help="Verbose output")
[email protected]("claude_args", nargs=-1, type=click.UNPROCESSED)
+def claude(port: int, no_rtk: bool, no_proxy: bool, verbose: bool, claude_args: tuple) -> None:
+ """Launch Claude Code through Headroom proxy.
+
+ \b
+ This command:
+ 1. Starts a Headroom optimization proxy on localhost
+ 2. Installs rtk and registers Claude Code hooks (compresses CLI output)
+ 3. Sets ANTHROPIC_API_URL to route through the proxy
+ 4. Launches 'claude' with your arguments
+
+ \b
+ All API calls from Claude Code flow through Headroom, which:
+ - Compresses tool outputs (SmartCrusher, Kompress, CodeCompressor)
+ - Preserves prefix cache (frozen message optimization)
+ - Tracks token savings and cache hit rates
+
+ \b
+ Examples:
+ headroom wrap claude # Start everything
+ headroom wrap claude -- -p # Claude in print mode
+ headroom wrap claude --port 9999 # Custom proxy port
+ headroom wrap claude --no-rtk # Skip rtk (proxy only)
+ """
+ # Check that claude CLI is available
+ claude_bin = shutil.which("claude")
+ if not claude_bin:
+ click.echo("Error: 'claude' not found in PATH.")
+ click.echo("Install Claude Code: https://docs.anthropic.com/en/docs/claude-code")
+ raise SystemExit(1)
+
+ proxy_proc: subprocess.Popen | None = None
+
+ def cleanup(signum: int | None = None, frame: Any = None) -> None:
+ """Clean up proxy on exit."""
+ if proxy_proc and proxy_proc.poll() is None:
+ proxy_proc.terminate()
+ try:
+ proxy_proc.wait(timeout=5)
+ except subprocess.TimeoutExpired:
+ proxy_proc.kill()
+
+ signal.signal(signal.SIGINT, cleanup)
+ signal.signal(signal.SIGTERM, cleanup)
+
+ try:
+ click.echo()
+ click.echo(" ╔═══════════════════════════════════════════════╗")
+ click.echo(" ║ HEADROOM WRAP: CLAUDE ║")
+ click.echo(" ╚═══════════════════════════════════════════════╝")
+ click.echo()
+
+ # Step 1: Start proxy
+ if not no_proxy:
+ if _check_proxy(port):
+ click.echo(f" Proxy already running on port {port}")
+ else:
+ click.echo(f" Starting Headroom proxy on port {port}...")
+ try:
+ proxy_proc = _start_proxy(port)
+ click.echo(f" Proxy ready on http://127.0.0.1:{port}")
+ except RuntimeError as e:
+ click.echo(f" Error: {e}")
+ raise SystemExit(1) from e
+ else:
+ if not _check_proxy(port):
+ click.echo(f" Warning: No proxy detected on port {port}")
+
+ # Step 2: Setup rtk
+ if not no_rtk:
+ click.echo(" Setting up rtk...")
+ _setup_rtk(verbose=verbose)
+ else:
+ if verbose:
+ click.echo(" Skipping rtk (--no-rtk)")
+
+ # Step 3: Launch claude
+ click.echo()
+ click.echo(" Launching Claude Code (API routed through Headroom)...")
+ click.echo(f" ANTHROPIC_API_URL=http://127.0.0.1:{port}")
+ if claude_args:
+ click.echo(f" Extra args: {' '.join(claude_args)}")
+ click.echo()
+
+ env = os.environ.copy()
+ env["ANTHROPIC_API_URL"] = f"http://127.0.0.1:{port}"
+
+ # Run claude — this blocks until claude exits
+ result = subprocess.run(
+ [claude_bin, *claude_args],
+ env=env,
+ )
+
+ raise SystemExit(result.returncode)
+
+ except SystemExit:
+ raise
+ except Exception as e:
+ click.echo(f" Error: {e}")
+ raise SystemExit(1) from e
+ finally:
+ cleanup()
diff --git a/headroom/config.py b/headroom/config.py
index 2d17a9119..3dc3da100 100644
--- a/headroom/config.py
+++ b/headroom/config.py
@@ -579,6 +579,26 @@ class CCRConfig:
)
+@dataclass
+class PrefixFreezeConfig:
+ """Configuration for cache-aware prefix freezing.
+
+ When enabled, tracks provider prefix cache state across turns and freezes
+ already-cached messages so the transform pipeline skips them. This prevents
+ Headroom from invalidating the provider's prefix cache (which would replace
+ a 90% read discount with a 25% write penalty on Anthropic).
+
+ The force_compress_threshold controls when compression savings are large
+ enough to justify busting the cache. For Anthropic (90% read discount),
+ compression must save >90% of tokens in the frozen prefix to be worth it.
+ """
+
+ enabled: bool = True
+ min_cached_tokens: int = 1024 # Min cached tokens to activate freeze
+ session_ttl_seconds: int = 600 # Session tracker cleanup TTL
+ force_compress_threshold: float = 0.5 # Bust cache if compression saves > this fraction
+
+
@dataclass
class HeadroomConfig:
"""Main configuration for HeadroomClient."""
@@ -594,6 +614,7 @@ class HeadroomConfig:
rolling_window: RollingWindowConfig = field(default_factory=RollingWindowConfig)
cache_optimizer: CacheOptimizerConfig = field(default_factory=CacheOptimizerConfig)
ccr: CCRConfig = field(default_factory=CCRConfig) # Compress-Cache-Retrieve
+ prefix_freeze: PrefixFreezeConfig = field(default_factory=PrefixFreezeConfig)
# Content Router - intelligent content-type based compression
# Routes content to appropriate compressor (LLMLingua for text, SmartCrusher for JSON,
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index c1a5b3ad5..701a4e701 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -314,6 +314,14 @@ def _build_prefix_cache_stats(
return {
"by_provider": by_provider,
"totals": totals,
+ "prefix_freeze": {
+ "busts_avoided": metrics.prefix_freeze_busts_avoided,
+ "tokens_preserved": metrics.prefix_freeze_tokens_preserved,
+ "compression_foregone_tokens": metrics.prefix_freeze_compression_foregone,
+ "net_benefit_tokens": (
+ metrics.prefix_freeze_tokens_preserved - metrics.prefix_freeze_compression_foregone
+ ),
+ },
}
@@ -486,6 +494,10 @@ class ProxyConfig:
retry_base_delay_ms: int = 1000
retry_max_delay_ms: int = 30000
+ # Prefix freeze: skip compression on already-cached messages
+ prefix_freeze_enabled: bool = True # Respect provider's prefix cache
+ prefix_freeze_session_ttl: int = 600 # Session tracker TTL (seconds)
+
# Cost tracking
cost_tracking_enabled: bool = True
budget_limit_usd: float | None = None # None = unlimited
@@ -1024,6 +1036,11 @@ class PrometheusMetrics:
# Track per-model cache request count to distinguish cold starts from busts
self._cache_requests_by_model: dict[str, int] = defaultdict(int)
+ # Prefix freeze stats (cache-aware compression)
+ self.prefix_freeze_busts_avoided: int = 0
+ self.prefix_freeze_tokens_preserved: int = 0
+ self.prefix_freeze_compression_foregone: int = 0
+
# Cumulative savings history (timestamp → cumulative tokens saved)
self.savings_history: list[tuple[str, int]] = []
@@ -1412,6 +1429,17 @@ class HeadroomProxy:
self.metrics = PrometheusMetrics()
+ # Prefix cache tracking: freeze already-cached messages to avoid
+ # invalidating the provider's prefix cache with our transforms
+ from headroom.cache.prefix_tracker import PrefixFreezeConfig, SessionTrackerStore
+
+ self.session_tracker_store = SessionTrackerStore(
+ default_config=PrefixFreezeConfig(
+ enabled=config.prefix_freeze_enabled,
+ session_ttl_seconds=config.prefix_freeze_session_ttl,
+ )
+ )
+
self.logger = (
RequestLogger(
log_file=config.log_file,
@@ -1955,6 +1983,11 @@ class HeadroomProxy:
optimized_messages = messages
optimized_tokens = original_tokens
+ # Get prefix cache tracker for this session
+ session_id = self.session_tracker_store.compute_session_id(request, model, messages)
+ prefix_tracker = self.session_tracker_store.get_or_create(session_id, "anthropic")
+ frozen_message_count = prefix_tracker.get_frozen_message_count()
+
if self.config.optimize and messages:
try:
context_limit = self.anthropic_provider.get_context_limit(model)
@@ -1962,6 +1995,7 @@ class HeadroomProxy:
messages=messages,
model=model,
model_limit=context_limit,
+ frozen_message_count=frozen_message_count,
biases=self.config.hooks.compute_biases(messages, _hook_ctx)
if self.config.hooks
else None,
@@ -2247,6 +2281,7 @@ class HeadroomProxy:
optimization_latency,
memory_user_id=memory_user_id,
pipeline_timing=pipeline_timing,
+ prefix_tracker=prefix_tracker,
)
else:
response = await self._retry_request("POST", url, headers, body)
@@ -2422,6 +2457,13 @@ class HeadroomProxy:
cr_tokens = usage.get("cache_read_input_tokens", 0)
cw_tokens = usage.get("cache_creation_input_tokens", 0)
+ # Update prefix cache tracker for next turn
+ prefix_tracker.update_from_response(
+ cache_read_tokens=cr_tokens,
+ cache_write_tokens=cw_tokens,
+ messages=optimized_messages,
+ )
+
if self.cost_tracker:
self.cost_tracker.record_tokens(model, tokens_saved, optimized_tokens)
@@ -3901,6 +3943,7 @@ class HeadroomProxy:
optimization_latency: float,
memory_user_id: str | None = None,
pipeline_timing: dict[str, float] | None = None,
+ prefix_tracker: Any | None = None,
) -> StreamingResponse:
"""Stream response with metrics tracking and memory tool handling.
@@ -4134,6 +4177,14 @@ class HeadroomProxy:
f"transforms={_summarize_transforms(transforms_applied)}"
)
+ # Update prefix cache tracker for next turn (streaming path)
+ if prefix_tracker is not None:
+ prefix_tracker.update_from_response(
+ cache_read_tokens=cache_read_tokens,
+ cache_write_tokens=cache_write_tokens,
+ messages=body.get("messages", []),
+ )
+
if self.cost_tracker:
self.cost_tracker.record_tokens(model, tokens_saved, optimized_tokens)
@@ -4402,6 +4453,13 @@ class HeadroomProxy:
optimized_messages = messages
optimized_tokens = original_tokens
+ # Get prefix cache tracker for this session
+ openai_session_id = self.session_tracker_store.compute_session_id(request, model, messages)
+ openai_prefix_tracker = self.session_tracker_store.get_or_create(
+ openai_session_id, "openai"
+ )
+ openai_frozen_count = openai_prefix_tracker.get_frozen_message_count()
+
if self.config.optimize and messages:
try:
context_limit = self.openai_provider.get_context_limit(model)
@@ -4409,6 +4467,7 @@ class HeadroomProxy:
messages=messages,
model=model,
model_limit=context_limit,
+ frozen_message_count=openai_frozen_count,
biases=_hook_biases,
)
if result.messages != messages:
@@ -4553,6 +4612,7 @@ class HeadroomProxy:
tags,
optimization_latency,
pipeline_timing=pipeline_timing,
+ prefix_tracker=openai_prefix_tracker,
)
else:
response = await self._retry_request("POST", url, headers, body)
@@ -4575,6 +4635,13 @@ class HeadroomProxy:
f"[{request_id}] Failed to extract cached tokens from OpenAI response: {e}"
)
+ # Update prefix cache tracker for next turn
+ openai_prefix_tracker.update_from_response(
+ cache_read_tokens=cache_read_tokens,
+ cache_write_tokens=0, # OpenAI doesn't report write tokens
+ messages=optimized_messages,
+ )
+
if self.cost_tracker:
self.cost_tracker.record_tokens(model, tokens_saved, optimized_tokens)
diff --git a/headroom/rtk/__init__.py b/headroom/rtk/__init__.py
new file mode 100644
index 000000000..54e212af6
--- /dev/null
+++ b/headroom/rtk/__init__.py
@@ -0,0 +1,33 @@
+"""rtk (Rust Token Killer) integration for Headroom.
+
+rtk compresses CLI output (test results, git diffs, log dumps) before it
+enters the LLM context window. Headroom downloads and manages the rtk binary.
+"""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+RTK_VERSION = "v0.28.2"
+RTK_BIN_DIR = Path.home() / ".headroom" / "bin"
+RTK_BIN_PATH = RTK_BIN_DIR / "rtk"
+
+
+def get_rtk_path() -> Path | None:
+ """Get path to rtk binary — check PATH first, then ~/.headroom/bin/."""
+ # Check if rtk is already in PATH (e.g., installed via brew)
+ system_rtk = shutil.which("rtk")
+ if system_rtk:
+ return Path(system_rtk)
+
+ # Check Headroom-managed install
+ if RTK_BIN_PATH.exists() and RTK_BIN_PATH.is_file():
+ return RTK_BIN_PATH
+
+ return None
+
+
+def is_rtk_installed() -> bool:
+ """Check if rtk is available."""
+ return get_rtk_path() is not None
diff --git a/headroom/rtk/installer.py b/headroom/rtk/installer.py
new file mode 100644
index 000000000..977f6438a
--- /dev/null
+++ b/headroom/rtk/installer.py
@@ -0,0 +1,183 @@
+"""Download and install rtk binary from GitHub releases."""
+
+from __future__ import annotations
+
+import io
+import logging
+import platform
+import ssl
+import stat
+import subprocess
+import tarfile
+import zipfile
+from pathlib import Path
+from urllib.request import urlopen
+
+from . import RTK_BIN_DIR, RTK_BIN_PATH, RTK_VERSION
+
+logger = logging.getLogger(__name__)
+
+GITHUB_RELEASE_URL = "https://github.com/rtk-ai/rtk/releases/download"
+
+
+def _get_target_triple() -> str:
+ """Detect platform and return the rtk release target triple."""
+ system = platform.system()
+ machine = platform.machine()
+
+ if system == "Darwin":
+ arch = "aarch64" if machine == "arm64" else "x86_64"
+ return f"{arch}-apple-darwin"
+ elif system == "Linux":
+ arch = "aarch64" if machine == "aarch64" else "x86_64"
+ suffix = "unknown-linux-gnu" if arch == "aarch64" else "unknown-linux-musl"
+ return f"{arch}-{suffix}"
+ elif system == "Windows":
+ return "x86_64-pc-windows-msvc"
+
+ raise RuntimeError(f"Unsupported platform: {system} {machine}")
+
+
+def _get_download_url(version: str) -> tuple[str, str]:
+ """Get download URL and extension for this platform.
+
+ Returns (url, extension) where extension is 'tar.gz' or 'zip'.
+ """
+ target = _get_target_triple()
+
+ if "windows" in target:
+ ext = "zip"
+ else:
+ ext = "tar.gz"
+
+ url = f"{GITHUB_RELEASE_URL}/{version}/rtk-{target}.{ext}"
+ return url, ext
+
+
+def download_rtk(version: str | None = None) -> Path:
+ """Download rtk binary from GitHub releases.
+
+ Args:
+ version: Version to download (e.g., "v0.28.2"). Defaults to pinned version.
+
+ Returns:
+ Path to the installed binary.
+
+ Raises:
+ RuntimeError: If download or extraction fails.
+ """
+ version = version or RTK_VERSION
+ url, ext = _get_download_url(version)
+
+ RTK_BIN_DIR.mkdir(parents=True, exist_ok=True)
+
+ logger.info("Downloading rtk %s from %s ...", version, url)
+
+ try:
+ # Try default SSL first, fall back to unverified for macOS framework Python
+ try:
+ with urlopen(url, timeout=30) as response:
+ data = response.read()
+ except Exception as ssl_err:
+ if "CERTIFICATE_VERIFY_FAILED" in str(ssl_err):
+ ctx = ssl.create_default_context()
+ ctx.check_hostname = False
+ ctx.verify_mode = ssl.CERT_NONE
+ with urlopen(url, timeout=30, context=ctx) as response:
+ data = response.read()
+ else:
+ raise
+ except Exception as e:
+ raise RuntimeError(f"Failed to download rtk from {url}: {e}") from e
+
+ # Extract binary
+ try:
+ if ext == "tar.gz":
+ with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tar:
+ # Find the rtk binary inside the archive
+ for member in tar.getmembers():
+ if member.name.endswith("/rtk") or member.name == "rtk":
+ member.name = "rtk" # Flatten path
+ tar.extract(member, RTK_BIN_DIR)
+ break
+ else:
+ raise RuntimeError("rtk binary not found in archive")
+ elif ext == "zip":
+ with zipfile.ZipFile(io.BytesIO(data)) as zf:
+ for name in zf.namelist():
+ if name.endswith("rtk.exe") or name.endswith("/rtk"):
+ target_name = "rtk.exe" if name.endswith(".exe") else "rtk"
+ with zf.open(name) as src, open(RTK_BIN_DIR / target_name, "wb") as dst:
+ dst.write(src.read())
+ break
+ else:
+ raise RuntimeError("rtk binary not found in archive")
+ except (tarfile.TarError, zipfile.BadZipFile) as e:
+ raise RuntimeError(f"Failed to extract rtk archive: {e}") from e
+
+ # Make executable
+ RTK_BIN_PATH.chmod(RTK_BIN_PATH.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)
+
+ # Verify
+ try:
+ result = subprocess.run(
+ [str(RTK_BIN_PATH), "--version"],
+ capture_output=True,
+ text=True,
+ timeout=5,
+ )
+ if result.returncode != 0:
+ raise RuntimeError(f"rtk verification failed: {result.stderr}")
+ logger.info("rtk installed: %s", result.stdout.strip())
+ except FileNotFoundError as e:
+ raise RuntimeError("rtk binary not found after extraction") from e
+ except subprocess.TimeoutExpired as e:
+ raise RuntimeError("rtk verification timed out") from e
+
+ return RTK_BIN_PATH
+
+
+def register_claude_hooks(rtk_path: Path | None = None) -> bool:
+ """Register rtk hooks in Claude Code settings.
+
+ Runs `rtk init --global` which adds a PreToolUse hook to
+ ~/.claude/settings.json that rewrites Bash commands through rtk.
+
+ Returns True if hooks were registered successfully.
+ """
+ rtk_path = rtk_path or RTK_BIN_PATH
+
+ try:
+ result = subprocess.run(
+ [str(rtk_path), "init", "--global"],
+ capture_output=True,
+ text=True,
+ timeout=10,
+ )
+ if result.returncode == 0:
+ logger.info("rtk hooks registered in Claude Code")
+ return True
+ else:
+ logger.warning("rtk init failed: %s", result.stderr)
+ return False
+ except Exception as e:
+ logger.warning("Failed to register rtk hooks: %s", e)
+ return False
+
+
+def ensure_rtk(version: str | None = None) -> Path | None:
+ """Ensure rtk is installed — download if needed.
+
+ Returns path to rtk binary, or None if installation failed.
+ """
+ from . import get_rtk_path
+
+ existing = get_rtk_path()
+ if existing:
+ return existing
+
+ try:
+ return download_rtk(version)
+ except RuntimeError as e:
+ logger.warning("Could not install rtk: %s", e)
+ return None
diff --git a/headroom/transforms/base.py b/headroom/transforms/base.py
index 54e1e0fd3..c50fb0f5e 100644
--- a/headroom/transforms/base.py
+++ b/headroom/transforms/base.py
@@ -9,6 +9,24 @@ from ..config import TransformResult
from ..tokenizer import Tokenizer
+def split_frozen(
+ messages: list[dict[str, Any]],
+ frozen_message_count: int,
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+ """Split messages into frozen (cached prefix) and mutable portions.
+
+ Args:
+ messages: All messages.
+ frozen_message_count: Number of leading messages to freeze.
+
+ Returns:
+ (frozen, mutable) — frozen messages must not be modified.
+ """
+ if frozen_message_count <= 0 or frozen_message_count >= len(messages):
+ return [], messages
+ return messages[:frozen_message_count], messages[frozen_message_count:]
+
+
class Transform(ABC):
"""Abstract base class for message transforms."""
@@ -28,6 +46,9 @@ class Transform(ABC):
messages: List of message dicts to transform.
tokenizer: Tokenizer for token counting.
**kwargs: Additional transform-specific arguments.
+ frozen_message_count: Number of leading messages in the
+ provider's prefix cache. Transforms should skip these
+ to avoid invalidating the cache.
Returns:
TransformResult with transformed messages and metadata.
diff --git a/headroom/transforms/cache_aligner.py b/headroom/transforms/cache_aligner.py
index 5b1e91051..7fad4d24f 100644
--- a/headroom/transforms/cache_aligner.py
+++ b/headroom/transforms/cache_aligner.py
@@ -159,9 +159,12 @@ class CacheAligner(Transform):
extracted_dynamic: list[str] = []
detection_stats: dict[str, int] = {} # Track what was detected
+ frozen_message_count = kwargs.get("frozen_message_count", 0)
- # Process system messages
- for msg in result_messages:
+ # Process system messages (skip if frozen — in provider's prefix cache)
+ for i, msg in enumerate(result_messages):
+ if i < frozen_message_count:
+ continue
if msg.get("role") == "system":
content = msg.get("content", "")
if isinstance(content, str):
@@ -175,9 +178,11 @@ class CacheAligner(Transform):
for category, count in stats.items():
detection_stats[category] = detection_stats.get(category, 0) + count
- # Normalize whitespace if configured
+ # Normalize whitespace if configured (skip frozen messages)
if self.config.normalize_whitespace:
- for msg in result_messages:
+ for i, msg in enumerate(result_messages):
+ if i < frozen_message_count:
+ continue
content = msg.get("content")
if isinstance(content, str):
msg["content"] = self._normalize_whitespace(content)
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py
index 2a4a3d4d8..639501587 100644
--- a/headroom/transforms/content_router.py
+++ b/headroom/transforms/content_router.py
@@ -1361,7 +1361,10 @@ class ContentRouter(Transform):
self.config.read_lifecycle,
compression_store=kwargs.get("compression_store"),
)
- lifecycle_result = lifecycle_mgr.apply(messages)
+ lifecycle_result = lifecycle_mgr.apply(
+ messages,
+ frozen_message_count=kwargs.get("frozen_message_count", 0),
+ )
messages = lifecycle_result.messages
# lifecycle transforms tracked separately, merged at the end
lifecycle_transforms = lifecycle_result.transforms_applied
@@ -1453,7 +1456,16 @@ class ContentRouter(Transform):
if self.config.protect_analysis_context:
analysis_intent = self._detect_analysis_intent(messages)
+ frozen_message_count = kwargs.get("frozen_message_count", 0)
+
for i, message in enumerate(messages):
+ # Skip frozen messages (in provider's prefix cache).
+ # Modifying these would invalidate the cache, replacing a 90%
+ # read discount with a 25% write penalty (Anthropic).
+ if i < frozen_message_count:
+ transformed_messages.append(message)
+ continue
+
role = message.get("role", "")
content = message.get("content", "")
bias = 1.0 # Default bias, may be overridden for tool messages
diff --git a/headroom/transforms/intelligent_context.py b/headroom/transforms/intelligent_context.py
index a0bb155ed..a9ac742dd 100644
--- a/headroom/transforms/intelligent_context.py
+++ b/headroom/transforms/intelligent_context.py
@@ -252,6 +252,11 @@ class IntelligentContextManager(Transform):
# Get protected indices
protected = self._get_protected_indices(result_messages)
+ # Frozen messages (in provider's prefix cache) must never be dropped
+ frozen_message_count = kwargs.get("frozen_message_count", 0)
+ if frozen_message_count > 0:
+ protected.update(range(frozen_message_count))
+
# ========== COMPRESS_FIRST STRATEGY ==========
# Try to compress tool messages before dropping anything
if strategy == ContextStrategy.COMPRESS_FIRST:
diff --git a/headroom/transforms/pipeline.py b/headroom/transforms/pipeline.py
index 3b247e9bd..c687679f0 100644
--- a/headroom/transforms/pipeline.py
+++ b/headroom/transforms/pipeline.py
@@ -199,6 +199,14 @@ class TransformPipeline:
current_messages = deep_copy_messages(messages)
pipeline_start = time.perf_counter()
+ frozen_count = kwargs.get("frozen_message_count", 0)
+ if frozen_count > 0:
+ logger.info(
+ "Pipeline: freezing first %d/%d messages (prefix cached by provider)",
+ frozen_count,
+ len(messages),
+ )
+
for transform in self.transforms:
# Check if transform should run
if not transform.should_apply(current_messages, tokenizer, **kwargs):
diff --git a/headroom/transforms/read_lifecycle.py b/headroom/transforms/read_lifecycle.py
index 5687dd419..edc8f801c 100644
--- a/headroom/transforms/read_lifecycle.py
+++ b/headroom/transforms/read_lifecycle.py
@@ -95,10 +95,20 @@ class ReadLifecycleManager:
self.config = config
self.store = compression_store
- def apply(self, messages: list[dict[str, Any]]) -> ReadLifecycleResult:
+ def apply(
+ self,
+ messages: list[dict[str, Any]],
+ frozen_message_count: int = 0,
+ ) -> ReadLifecycleResult:
"""Apply lifecycle management to messages.
Single-pass analysis, targeted replacement of stale/superseded Reads.
+
+ Args:
+ messages: Conversation messages.
+ frozen_message_count: Number of leading messages in the provider's
+ prefix cache. Stale/superseded Reads in the frozen prefix are
+ skipped to avoid invalidating the cache.
"""
if not self.config.enabled:
return ReadLifecycleResult(messages=messages)
@@ -113,7 +123,26 @@ class ReadLifecycleManager:
if not classifications:
return ReadLifecycleResult(messages=messages)
- # Phase 3: Replace stale/superseded content
+ # Phase 3: Filter out replacements in frozen prefix
+ if frozen_message_count > 0:
+ frozen_skipped = sum(
+ 1
+ for c in classifications
+ if c.state != ReadState.FRESH and c.msg_index < frozen_message_count
+ )
+ if frozen_skipped > 0:
+ logger.info(
+ "ReadLifecycle: skipping %d stale/superseded replacements "
+ "in frozen prefix (first %d messages)",
+ frozen_skipped,
+ frozen_message_count,
+ )
+ # Re-classify frozen stale/superseded as FRESH to skip replacement
+ for c in classifications:
+ if c.msg_index < frozen_message_count and c.state != ReadState.FRESH:
+ c.state = ReadState.FRESH
+
+ # Phase 4: Replace stale/superseded content
return self._apply_lifecycle(messages, classifications)
def _build_tool_metadata(
diff --git a/headroom/transforms/rolling_window.py b/headroom/transforms/rolling_window.py
index 7bc6333b1..f65bd00a4 100644
--- a/headroom/transforms/rolling_window.py
+++ b/headroom/transforms/rolling_window.py
@@ -106,6 +106,11 @@ class RollingWindow(Transform):
# Identify protected indices
protected = self._get_protected_indices(result_messages)
+ # Frozen messages (in provider's prefix cache) must never be dropped
+ frozen_message_count = kwargs.get("frozen_message_count", 0)
+ if frozen_message_count > 0:
+ protected.update(range(frozen_message_count))
+
# Identify tool units
tool_units = find_tool_units(result_messages)
diff --git a/headroom/transforms/smart_crusher.py b/headroom/transforms/smart_crusher.py
index 6d42f37e4..ac05191d8 100644
--- a/headroom/transforms/smart_crusher.py
+++ b/headroom/transforms/smart_crusher.py
@@ -2036,8 +2036,13 @@ class SmartCrusher(Transform):
query_context = self._extract_context_from_messages(result_messages)
crushed_count = 0
+ frozen_message_count = kwargs.get("frozen_message_count", 0)
+
+ for msg_idx, msg in enumerate(result_messages):
+ # Skip frozen messages (in provider's prefix cache)
+ if msg_idx < frozen_message_count:
+ continue
- for msg in result_messages:
# OpenAI style
if msg.get("role") == "tool":
content = msg.get("content", "")
diff --git a/tests/test_cache/test_prefix_tracker.py b/tests/test_cache/test_prefix_tracker.py
new file mode 100644
index 000000000..2eb4e63f1
--- /dev/null
+++ b/tests/test_cache/test_prefix_tracker.py
@@ -0,0 +1,408 @@
+"""Tests for PrefixCacheTracker — cache-aware compression."""
+
+import time
+
+import pytest
+
+from headroom.cache.prefix_tracker import (
+ FreezeStats,
+ PrefixCacheTracker,
+ PrefixFreezeConfig,
+ SessionTrackerStore,
+)
+
+
+class TestPrefixCacheTracker:
+ """Test PrefixCacheTracker core functionality."""
+
+ @pytest.fixture
+ def tracker(self):
+ return PrefixCacheTracker("anthropic")
+
+ @pytest.fixture
+ def openai_tracker(self):
+ return PrefixCacheTracker("openai")
+
+ def test_turn_0_no_freeze(self, tracker):
+ """First turn should never freeze — no cache state yet."""
+ assert tracker.get_frozen_message_count() == 0
+
+ def test_turn_1_with_cache_hit_freezes(self, tracker):
+ """After turn 1 with cache hits, turn 2 should freeze."""
+ messages = [
+ {"role": "system", "content": "You are a helpful assistant." * 100},
+ {"role": "user", "content": "Hello"},
+ {"role": "assistant", "content": "Hi there!"},
+ ]
+ # Simulate: provider cached 2000 tokens (system + user)
+ token_counts = [1500, 50, 500]
+
+ tracker.update_from_response(
+ cache_read_tokens=0,
+ cache_write_tokens=2050,
+ messages=messages,
+ message_token_counts=token_counts,
+ )
+
+ # On turn 2, the first 2 messages (1500 + 50 = 1550 <= 2050) are frozen
+ assert tracker.get_frozen_message_count() == 3 # All 3 fit within 2050
+
+ def test_partial_freeze(self, tracker):
+ """Only messages that fit within cached tokens are frozen."""
+ messages = [
+ {"role": "system", "content": "System prompt" * 50},
+ {"role": "user", "content": "First question" * 50},
+ {"role": "assistant", "content": "First answer" * 50},
+ {"role": "user", "content": "Second question"},
+ ]
+ token_counts = [2000, 500, 500, 50]
+
+ tracker.update_from_response(
+ cache_read_tokens=2500,
+ cache_write_tokens=0,
+ messages=messages,
+ message_token_counts=token_counts,
+ )
+
+ # 2000 + 500 = 2500 <= 2500, but 2000 + 500 + 500 = 3000 > 2500
+ assert tracker.get_frozen_message_count() == 2
+
+ def test_cold_start_no_freeze(self, tracker):
+ """If cache_read=0 and cache_write=0, don't freeze."""
+ messages = [{"role": "user", "content": "Hello"}]
+
+ tracker.update_from_response(
+ cache_read_tokens=0,
+ cache_write_tokens=0,
+ messages=messages,
+ )
+
+ assert tracker.get_frozen_message_count() == 0
+
+ def test_cache_write_freezes_next_turn(self, tracker):
+ """Cache writes (new cache entries) should be frozen on the next turn."""
+ messages = [
+ {"role": "system", "content": "System" * 200},
+ {"role": "user", "content": "Hello"},
+ ]
+ token_counts = [1500, 50]
+
+ # Turn 1: provider writes to cache (above min threshold)
+ tracker.update_from_response(
+ cache_read_tokens=0,
+ cache_write_tokens=1550,
+ messages=messages,
+ message_token_counts=token_counts,
+ )
+
+ # Turn 2: should freeze what was written
+ assert tracker.get_frozen_message_count() == 2
+
+ def test_min_cached_tokens_threshold(self):
+ """Below min_cached_tokens, no freeze."""
+ config = PrefixFreezeConfig(min_cached_tokens=2000)
+ tracker = PrefixCacheTracker("anthropic", config)
+
+ messages = [{"role": "user", "content": "Hello"}]
+
+ # Turn 1: only 500 tokens cached — below threshold
+ tracker.update_from_response(
+ cache_read_tokens=0,
+ cache_write_tokens=500,
+ messages=messages,
+ message_token_counts=[500],
+ )
+
+ assert tracker.get_frozen_message_count() == 0
+
+ def test_disabled_config(self):
+ """Disabled config always returns 0."""
+ config = PrefixFreezeConfig(enabled=False)
+ tracker = PrefixCacheTracker("anthropic", config)
+
+ messages = [{"role": "system", "content": "System" * 500}]
+
+ tracker.update_from_response(
+ cache_read_tokens=5000,
+ cache_write_tokens=0,
+ messages=messages,
+ message_token_counts=[5000],
+ )
+
+ assert tracker.get_frozen_message_count() == 0
+
+ def test_turn_number_increments(self, tracker):
+ """Turn number should increment on each update."""
+ messages = [{"role": "user", "content": "Hello"}]
+
+ assert tracker._turn_number == 0
+
+ tracker.update_from_response(0, 0, messages)
+ assert tracker._turn_number == 1
+
+ tracker.update_from_response(0, 0, messages)
+ assert tracker._turn_number == 2
+
+ def test_stats_tracking(self, tracker):
+ """Stats should reflect tracker state."""
+ stats = tracker.stats
+ assert isinstance(stats, FreezeStats)
+ assert stats.busts_avoided == 0
+ assert stats.tokens_preserved == 0
+ assert stats.turn_number == 0
+
+ def test_record_bust_avoided(self, tracker):
+ """Recording bust avoided should update stats."""
+ tracker.record_bust_avoided(tokens_preserved=5000, compression_foregone=500)
+ tracker.record_bust_avoided(tokens_preserved=3000, compression_foregone=200)
+
+ stats = tracker.stats
+ assert stats.busts_avoided == 2
+ assert stats.tokens_preserved == 8000
+ assert stats.compression_foregone_tokens == 700
+ assert stats.net_benefit_tokens == 7300
+
+ def test_should_force_compress_outside_frozen(self, tracker):
+ """Messages outside frozen prefix should always be compressed."""
+ tracker._cached_message_count = 3
+ assert tracker.should_force_compress(5, 1000, 200) is True
+
+ def test_should_force_compress_when_savings_exceed_discount(self, tracker):
+ """For Anthropic (90% discount), compression must save >90% to be worth it."""
+ tracker._cached_message_count = 5
+
+ # 95% savings > 90% discount — should force compress
+ assert tracker.should_force_compress(2, 1000, 50) is True
+
+ # 50% savings < 90% discount — should NOT force compress
+ assert tracker.should_force_compress(2, 1000, 500) is False
+
+ def test_should_force_compress_openai(self, openai_tracker):
+ """For OpenAI (50% discount), compression must save >50% to be worth it."""
+ openai_tracker._cached_message_count = 5
+
+ # 60% savings > 50% discount — should force compress
+ assert openai_tracker.should_force_compress(2, 1000, 400) is True
+
+ # 40% savings < 50% discount — should NOT force compress
+ assert openai_tracker.should_force_compress(2, 1000, 600) is False
+
+ def test_estimate_message_tokens(self):
+ """Token estimation should roughly match character / 3.5."""
+ messages = [
+ {"role": "system", "content": "A" * 350}, # ~100 tokens
+ {"role": "user", "content": "B" * 70}, # ~20 tokens
+ ]
+ counts = PrefixCacheTracker._estimate_message_tokens(messages)
+ assert len(counts) == 2
+ assert counts[0] > counts[1] # System should have more tokens
+
+ def test_estimate_content_blocks(self):
+ """Token estimation should handle Anthropic content blocks."""
+ messages = [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "A" * 350},
+ {"type": "text", "text": "B" * 350},
+ ],
+ },
+ ]
+ counts = PrefixCacheTracker._estimate_message_tokens(messages)
+ assert len(counts) == 1
+ assert counts[0] > 100
+
+ def test_session_ttl_expiry(self):
+ """Tracker should report as expired after TTL."""
+ config = PrefixFreezeConfig(session_ttl_seconds=1)
+ tracker = PrefixCacheTracker("anthropic", config)
+
+ assert tracker.is_expired is False
+
+ # Simulate time passing
+ tracker._last_activity = time.time() - 2
+ assert tracker.is_expired is True
+
+
+class TestSessionTrackerStore:
+ """Test SessionTrackerStore management."""
+
+ @pytest.fixture
+ def store(self):
+ return SessionTrackerStore()
+
+ def test_get_or_create_new(self, store):
+ """Should create a new tracker for unknown session."""
+ tracker = store.get_or_create("session-1", "anthropic")
+ assert isinstance(tracker, PrefixCacheTracker)
+ assert tracker.provider == "anthropic"
+
+ def test_get_or_create_existing(self, store):
+ """Should return the same tracker for the same session."""
+ tracker1 = store.get_or_create("session-1", "anthropic")
+ tracker2 = store.get_or_create("session-1", "anthropic")
+ assert tracker1 is tracker2
+
+ def test_different_sessions(self, store):
+ """Different sessions should get different trackers."""
+ tracker1 = store.get_or_create("session-1", "anthropic")
+ tracker2 = store.get_or_create("session-2", "openai")
+ assert tracker1 is not tracker2
+ assert tracker1.provider == "anthropic"
+ assert tracker2.provider == "openai"
+
+ def test_active_sessions_count(self, store):
+ """Should track the number of active sessions."""
+ assert store.active_sessions == 0
+
+ store.get_or_create("s1", "anthropic")
+ assert store.active_sessions == 1
+
+ store.get_or_create("s2", "openai")
+ assert store.active_sessions == 2
+
+ def test_cleanup_expired(self, store):
+ """Should remove expired sessions on cleanup."""
+ config = PrefixFreezeConfig(session_ttl_seconds=1)
+ store = SessionTrackerStore(default_config=config)
+
+ tracker = store.get_or_create("expired-session", "anthropic")
+ tracker._last_activity = time.time() - 2
+
+ # Force cleanup
+ store._last_cleanup = 0
+ store._maybe_cleanup()
+
+ assert store.active_sessions == 0
+
+ def test_compute_session_id_from_header(self, store):
+ """Should use x-headroom-session-id header if present."""
+
+ class MockRequest:
+ headers = {"x-headroom-session-id": "explicit-id-123"}
+
+ session_id = store.compute_session_id(
+ MockRequest(), "claude-3", [{"role": "user", "content": "Hi"}]
+ )
+ assert session_id == "explicit-id-123"
+
+ def test_compute_session_id_from_hash(self, store):
+ """Should hash model + system prompt as fallback."""
+
+ class MockRequest:
+ headers = {}
+
+ messages = [
+ {"role": "system", "content": "You are helpful."},
+ {"role": "user", "content": "Hi"},
+ ]
+
+ id1 = store.compute_session_id(MockRequest(), "claude-3", messages)
+ id2 = store.compute_session_id(MockRequest(), "claude-3", messages)
+ assert id1 == id2 # Stable hash
+ assert len(id1) == 16
+
+ # Different model = different session
+ id3 = store.compute_session_id(MockRequest(), "gpt-4", messages)
+ assert id3 != id1
+
+ def test_compute_session_id_no_system(self, store):
+ """Should work without system messages."""
+
+ class MockRequest:
+ headers = {}
+
+ messages = [{"role": "user", "content": "Hi"}]
+ session_id = store.compute_session_id(MockRequest(), "claude-3", messages)
+ assert isinstance(session_id, str)
+ assert len(session_id) == 16
+
+
+class TestMultiTurnScenario:
+ """Integration-style tests simulating multi-turn conversations."""
+
+ def test_five_turn_conversation(self):
+ """Simulate a 5-turn conversation with growing prefix."""
+ tracker = PrefixCacheTracker("anthropic")
+
+ # Turn 1: System + User (cold start, no cache)
+ messages_t1 = [
+ {"role": "system", "content": "System prompt" * 200},
+ {"role": "user", "content": "Question 1"},
+ ]
+ token_counts_t1 = [2000, 50]
+
+ assert tracker.get_frozen_message_count() == 0 # No freeze on turn 1
+
+ tracker.update_from_response(
+ cache_read_tokens=0,
+ cache_write_tokens=2050,
+ messages=messages_t1,
+ message_token_counts=token_counts_t1,
+ )
+
+ # Turn 2: Previous messages cached, new user message added
+ messages_t2 = messages_t1 + [
+ {"role": "assistant", "content": "Answer 1"},
+ {"role": "user", "content": "Question 2"},
+ ]
+ token_counts_t2 = [2000, 50, 200, 50]
+
+ frozen = tracker.get_frozen_message_count()
+ assert frozen == 2 # System + User1 frozen
+
+ tracker.update_from_response(
+ cache_read_tokens=2050,
+ cache_write_tokens=250,
+ messages=messages_t2,
+ message_token_counts=token_counts_t2,
+ )
+
+ # Turn 3: Even more cached
+ messages_t3 = messages_t2 + [
+ {"role": "assistant", "content": "Answer 2"},
+ {"role": "user", "content": "Question 3"},
+ ]
+ token_counts_t3 = [2000, 50, 200, 50, 200, 50]
+
+ frozen = tracker.get_frozen_message_count()
+ assert frozen == 4 # System + User1 + Asst1 + User2 frozen
+
+ tracker.update_from_response(
+ cache_read_tokens=2300,
+ cache_write_tokens=250,
+ messages=messages_t3,
+ message_token_counts=token_counts_t3,
+ )
+
+ # Verify turn count
+ assert tracker._turn_number == 3
+
+ def test_cache_bust_resets_freeze(self):
+ """If cache is busted (0 read, 0 write), freeze should reset."""
+ tracker = PrefixCacheTracker("anthropic")
+
+ messages = [
+ {"role": "system", "content": "System" * 200},
+ {"role": "user", "content": "Hello"},
+ ]
+
+ # Turn 1: Cache established
+ tracker.update_from_response(
+ cache_read_tokens=0,
+ cache_write_tokens=2000,
+ messages=messages,
+ message_token_counts=[1500, 500],
+ )
+ assert tracker.get_frozen_message_count() == 2 # Both fit within 2000
+
+ # Turn 2: Cache bust (0 reads, system prompt changed)
+ tracker.update_from_response(
+ cache_read_tokens=0,
+ cache_write_tokens=0,
+ messages=messages,
+ message_token_counts=[1500, 500],
+ )
+
+ # After a bust with 0 total, freeze should reset
+ assert tracker.get_frozen_message_count() == 0