Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

Add cache optimization module with scalable dynamic content detection

chopratejas7a05808e0f92
diff --git a/benchmarks/dynamic_detector_benchmark.py b/benchmarks/dynamic_detector_benchmark.py
new file mode 100644
index 000000000..4a8d307e9
--- /dev/null
+++ b/benchmarks/dynamic_detector_benchmark.py
@@ -0,0 +1,375 @@
+#!/usr/bin/env python3
+"""
+Real-world benchmark for DynamicContentDetector.
+
+Tests the detector against realistic system prompts from AI coding agents,
+chatbots, and enterprise applications.
+"""
+
+import time
+import statistics
+from dataclasses import dataclass
+from typing import Any
+
+from headroom.cache.dynamic_detector import (
+ DetectorConfig,
+ DynamicContentDetector,
+ DynamicCategory,
+)
+
+
+@dataclass
+class BenchmarkResult:
+ """Result of a single benchmark run."""
+ name: str
+ content_length: int
+ spans_found: int
+ categories: list[str]
+ static_length: int
+ dynamic_length: int
+ latency_ms: float
+ tiers_used: list[str]
+ warnings: list[str]
+
+
+# Real-world system prompts
+REAL_WORLD_PROMPTS = {
+ "claude_code_style": """You are Claude, an AI assistant created by Anthropic to be helpful, harmless, and honest.
+
+Today is Tuesday, January 7, 2026.
+Current time: 10:30:45 AM PST.
+
+You are operating in a software development environment with access to:
+- File system operations
+- Terminal commands
+- Web search
+
+Session ID: sess_abc123def456ghi789jkl012
+Request ID: req_xyz789abc123def456ghi789
+User: tchopra
+Workspace: /Users/tchopra/claude-projects/headroom
+
+Be concise, accurate, and helpful. Follow the user's instructions carefully.""",
+
+ "enterprise_assistant": """You are an enterprise AI assistant for Acme Corporation.
+
+Current Date: 2026-01-07T10:30:00Z
+Last Updated: 2026-01-07T09:00:00Z
+
+User Profile:
+- Name: John Smith
+- Employee ID: EMP-2024-00542
+- Department: Engineering
+- Manager: Sarah Johnson
+- Location: San Francisco, CA
+- Hire Date: March 15, 2023
+
+System Status:
+- API Version: v2.3.1-beta
+- Server Load: 45%
+- Active Users: 1,247
+- Queue Length: 23
+
+Budget Information:
+- Monthly Allowance: $5,000.00
+- Used This Month: $2,341.67
+- Remaining: $2,658.33
+
+Help the user with their work tasks while following company policies.""",
+
+ "coding_agent": """You are an autonomous coding agent with access to tools.
+
+Environment:
+- OS: macOS Darwin 25.1.0
+- Working Directory: /Users/developer/projects/myapp
+- Git Branch: feature/JIRA-1234-add-auth
+- Last Commit: a1b2c3d4e5f6 (2 hours ago)
+- Node Version: v20.10.0
+- Python Version: 3.11.7
+
+Current Task Context:
+- Task ID: 550e8400-e29b-41d4-a716-446655440000
+- Created: 2026-01-07T08:15:30Z
+- Priority: High
+- Estimated Time: 2 hours
+
+API Keys Available:
+- OPENAI_API_KEY: sk-proj-xxxxxxxxxxxxxxxxxxxxxxxxxxxx
+- ANTHROPIC_API_KEY: sk-ant-xxxxxxxxxxxxxxxxxxxxxxxxxxxx
+- DATABASE_URL: postgresql://user:pass@localhost:5432/mydb
+
+Execute tasks step by step, verify each action, and report progress.""",
+
+ "customer_support": """You are a customer support agent for TechStore Inc.
+
+Current Time: January 7, 2026, 3:45 PM EST
+Support Ticket: #TKT-2026-0107-4521
+
+Customer Information:
+- Name: Alice Chen
+- Phone: (555) 123-4567
+- Customer Since: August 2021
+- Loyalty Tier: Gold
+- Total Purchases: $12,456.78
+
+Recent Orders:
+- Order #ORD-2026-0105-7823 - iPhone 15 Pro - $1,199.00 - Delivered
+- Order #ORD-2025-1220-3456 - AirPods Pro - $249.00 - Delivered
+- Order #ORD-2025-1115-9012 - MacBook Air - $1,299.00 - Returned
+
+Active Issues:
+- Case #CS-2026-0107-001 - Battery drain issue - Open since today
+
+Provide helpful, empathetic support while following company guidelines.""",
+
+ "data_analysis": """You are a data analysis assistant.
+
+Report Generated: 2026-01-07 10:30:00 UTC
+Report ID: RPT-550e8400-e29b-41d4-a716-446655440000
+Data Range: 2025-12-01 to 2025-12-31
+
+Summary Statistics:
+- Total Revenue: $1,234,567.89
+- Total Orders: 45,678
+- Average Order Value: $27.03
+- Top Product: Widget Pro ($234,567.00)
+- Top Region: California (23.4%)
+
+Key Metrics:
+- DAU: 125,000
+- MAU: 890,000
+- Churn Rate: 2.3%
+- NPS Score: 67
+
+Anomalies Detected:
+- Spike on Dec 15: 3.2x normal traffic
+- Drop on Dec 25: 0.4x normal (expected - holiday)
+
+Help analyze the data and provide insights.""",
+
+ "minimal_static": """You are a helpful AI assistant.
+
+Your role is to:
+1. Answer questions accurately
+2. Be concise and clear
+3. Follow instructions carefully
+4. Admit when you don't know something
+
+Always be helpful, harmless, and honest.""",
+
+ "heavy_dynamic": """Session started at 2026-01-07T10:30:45.123Z
+Request ID: req_abc123def456ghi789jkl012mno345pqr678
+Trace ID: 550e8400-e29b-41d4-a716-446655440000
+Parent Span: span_xyz789abc123
+User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)
+IP Address: 192.168.1.100
+Geo: San Francisco, CA, USA (37.7749, -122.4194)
+
+Auth Token: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...
+Token Expires: 2026-01-07T11:30:45Z
+Refresh Token: rt_abc123def456
+
+Last Login: 2026-01-06T18:45:30Z
+Login Count: 1,247
+Account Balance: $5,432.10
+Credit Limit: $10,000.00
+
+Real-time Stock Prices (as of 10:30 AM):
+- AAPL: $185.42 (+1.2%)
+- GOOGL: $142.89 (-0.5%)
+- MSFT: $378.23 (+0.8%)
+- AMZN: $156.78 (+2.1%)
+
+Process this request.""",
+}
+
+
+def run_benchmark(
+ prompts: dict[str, str],
+ tiers: list[str],
+ iterations: int = 10,
+) -> dict[str, Any]:
+ """Run benchmark on prompts with specified tiers."""
+
+ config = DetectorConfig(tiers=tiers) # type: ignore
+ detector = DynamicContentDetector(config)
+
+ results: dict[str, list[BenchmarkResult]] = {}
+
+ for name, content in prompts.items():
+ results[name] = []
+
+ for _ in range(iterations):
+ start = time.perf_counter()
+ result = detector.detect(content)
+ elapsed = (time.perf_counter() - start) * 1000
+
+ categories = list(set(s.category.value for s in result.spans))
+
+ results[name].append(BenchmarkResult(
+ name=name,
+ content_length=len(content),
+ spans_found=len(result.spans),
+ categories=categories,
+ static_length=len(result.static_content),
+ dynamic_length=len(result.dynamic_content),
+ latency_ms=elapsed,
+ tiers_used=result.tiers_used,
+ warnings=result.warnings,
+ ))
+
+ return results
+
+
+def print_results(
+ results: dict[str, list[BenchmarkResult]],
+ tier_name: str,
+):
+ """Print benchmark results."""
+
+ print(f"\n{'='*80}")
+ print(f"BENCHMARK RESULTS: {tier_name}")
+ print(f"{'='*80}")
+
+ for name, runs in results.items():
+ latencies = [r.latency_ms for r in runs]
+ avg_latency = statistics.mean(latencies)
+ std_latency = statistics.stdev(latencies) if len(latencies) > 1 else 0
+
+ # Use first run for span info (consistent across runs)
+ first = runs[0]
+
+ compression = (1 - first.static_length / first.content_length) * 100 if first.content_length > 0 else 0
+
+ print(f"\n📄 {name}")
+ print(f" Content: {first.content_length:,} chars")
+ print(f" Spans found: {first.spans_found}")
+ print(f" Categories: {', '.join(first.categories) if first.categories else 'none'}")
+ print(f" Static: {first.static_length:,} chars | Dynamic: {first.dynamic_length:,} chars")
+ print(f" Compression: {compression:.1f}% removed")
+ print(f" Latency: {avg_latency:.2f}ms ± {std_latency:.2f}ms")
+ print(f" Tiers used: {', '.join(first.tiers_used)}")
+ if first.warnings:
+ print(f" ⚠️ Warnings: {len(first.warnings)}")
+
+
+def print_comparison(all_results: dict[str, dict[str, list[BenchmarkResult]]]):
+ """Print comparison across tiers."""
+
+ print(f"\n{'='*80}")
+ print("TIER COMPARISON")
+ print(f"{'='*80}")
+
+ prompts = list(REAL_WORLD_PROMPTS.keys())
+ tiers = list(all_results.keys())
+
+ # Header
+ header = f"{'Prompt':<25}"
+ for tier in tiers:
+ header += f" | {tier:>12} spans | {'latency':>8}"
+ print(header)
+ print("-" * len(header))
+
+ for prompt in prompts:
+ row = f"{prompt:<25}"
+ for tier in tiers:
+ if prompt in all_results[tier]:
+ runs = all_results[tier][prompt]
+ spans = runs[0].spans_found
+ latency = statistics.mean([r.latency_ms for r in runs])
+ row += f" | {spans:>12} | {latency:>7.2f}ms"
+ else:
+ row += f" | {'N/A':>12} | {'N/A':>8}"
+ print(row)
+
+ # Summary
+ print(f"\n{'='*80}")
+ print("SUMMARY")
+ print(f"{'='*80}")
+
+ for tier in tiers:
+ all_latencies = []
+ total_spans = 0
+ for runs in all_results[tier].values():
+ all_latencies.extend([r.latency_ms for r in runs])
+ total_spans += runs[0].spans_found
+
+ avg = statistics.mean(all_latencies)
+ p50 = statistics.median(all_latencies)
+ p99 = sorted(all_latencies)[int(len(all_latencies) * 0.99)] if len(all_latencies) > 1 else avg
+
+ print(f"\n{tier}:")
+ print(f" Total spans detected: {total_spans}")
+ print(f" Avg latency: {avg:.2f}ms")
+ print(f" P50 latency: {p50:.2f}ms")
+ print(f" P99 latency: {p99:.2f}ms")
+
+
+def show_detection_details(prompt_name: str, content: str):
+ """Show detailed detection for a specific prompt."""
+
+ print(f"\n{'='*80}")
+ print(f"DETECTION DETAILS: {prompt_name}")
+ print(f"{'='*80}")
+
+ config = DetectorConfig(tiers=["regex"])
+ detector = DynamicContentDetector(config)
+ result = detector.detect(content)
+
+ print(f"\nOriginal content ({len(content)} chars):")
+ print("-" * 40)
+ print(content[:500] + "..." if len(content) > 500 else content)
+
+ print(f"\n\nDetected spans ({len(result.spans)}):")
+ print("-" * 40)
+ for span in result.spans:
+ print(f" [{span.category.value:12}] '{span.text[:50]}{'...' if len(span.text) > 50 else ''}'")
+
+ print(f"\n\nStatic content ({len(result.static_content)} chars):")
+ print("-" * 40)
+ print(result.static_content[:500] + "..." if len(result.static_content) > 500 else result.static_content)
+
+ print(f"\n\nDynamic content ({len(result.dynamic_content)} chars):")
+ print("-" * 40)
+ print(result.dynamic_content if result.dynamic_content else "(none)")
+
+
+def main():
+ """Run the benchmark."""
+
+ print("🚀 Dynamic Content Detector - Real World Benchmark")
+ print("=" * 80)
+
+ iterations = 20
+
+ # Test each tier configuration
+ tier_configs = {
+ "regex_only": ["regex"],
+ # "regex+ner": ["regex", "ner"], # Uncomment if spacy installed
+ # "all_tiers": ["regex", "ner", "semantic"], # Uncomment if all deps installed
+ }
+
+ all_results: dict[str, dict[str, list[BenchmarkResult]]] = {}
+
+ for tier_name, tiers in tier_configs.items():
+ print(f"\n⏱️ Running {tier_name} ({iterations} iterations per prompt)...")
+ results = run_benchmark(REAL_WORLD_PROMPTS, tiers, iterations)
+ all_results[tier_name] = results
+ print_results(results, tier_name)
+
+ # Print comparison if multiple tiers tested
+ if len(all_results) > 1:
+ print_comparison(all_results)
+
+ # Show detailed detection for a few prompts
+ print("\n" + "=" * 80)
+ print("DETAILED DETECTION EXAMPLES")
+ print("=" * 80)
+
+ for name in ["claude_code_style", "enterprise_assistant", "heavy_dynamic"]:
+ show_detection_details(name, REAL_WORLD_PROMPTS[name])
+
+
+if __name__ == "__main__":
+ main()
diff --git a/headroom/__init__.py b/headroom/__init__.py
index fe02d8e5b..4868c09bf 100644
--- a/headroom/__init__.py
+++ b/headroom/__init__.py
@@ -38,10 +38,25 @@ Example usage:
print(f"Would save {plan.tokens_saved} tokens")
"""
+from .cache import (
+ AnthropicCacheOptimizer,
+ BaseCacheOptimizer,
+ CacheConfig,
+ CacheMetrics,
+ CacheOptimizerRegistry,
+ CacheResult,
+ CacheStrategy,
+ GoogleCacheOptimizer,
+ OpenAICacheOptimizer,
+ OptimizationContext,
+ SemanticCache,
+ SemanticCacheLayer,
+)
from .client import HeadroomClient
from .config import (
Block,
CacheAlignerConfig,
+ CacheOptimizerConfig,
CachePrefixMetrics,
DiffArtifact,
HeadroomConfig,
@@ -92,6 +107,7 @@ __all__ = [
"ToolCrusherConfig",
"SmartCrusherConfig",
"CacheAlignerConfig",
+ "CacheOptimizerConfig",
"RollingWindowConfig",
"RelevanceScorerConfig",
# Data models
@@ -109,6 +125,19 @@ __all__ = [
"CacheAligner",
"RollingWindow",
"TransformPipeline",
+ # Cache optimizers
+ "BaseCacheOptimizer",
+ "CacheConfig",
+ "CacheMetrics",
+ "CacheResult",
+ "CacheStrategy",
+ "OptimizationContext",
+ "CacheOptimizerRegistry",
+ "AnthropicCacheOptimizer",
+ "OpenAICacheOptimizer",
+ "GoogleCacheOptimizer",
+ "SemanticCache",
+ "SemanticCacheLayer",
# Relevance scoring
"RelevanceScore",
"RelevanceScorer",
diff --git a/headroom/cache/__init__.py b/headroom/cache/__init__.py
new file mode 100644
index 000000000..88f5998c9
--- /dev/null
+++ b/headroom/cache/__init__.py
@@ -0,0 +1,76 @@
+"""
+Headroom Cache Optimization Module.
+
+This module provides a plugin-based architecture for cache optimization
+across different LLM providers. Each provider has different caching
+mechanisms and this module abstracts those differences.
+
+Provider Caching Differences:
+- Anthropic: Explicit cache_control blocks, 90% savings, 5-min TTL
+- OpenAI: Automatic prefix caching, 50% savings, no user control
+- Google: Separate CachedContent API, 75% savings + storage costs
+
+Usage:
+ from headroom.cache import CacheOptimizerRegistry, SemanticCacheLayer
+
+ # Get provider-specific optimizer
+ optimizer = CacheOptimizerRegistry.get("anthropic")
+ result = optimizer.optimize(messages, context)
+
+ # With semantic caching layer
+ semantic = SemanticCacheLayer(optimizer, similarity_threshold=0.95)
+ result = semantic.process(messages, context)
+
+ # Register custom optimizer
+ CacheOptimizerRegistry.register("my-provider", MyOptimizer)
+"""
+
+from .base import (
+ BaseCacheOptimizer,
+ CacheBreakpoint,
+ CacheConfig,
+ CacheMetrics,
+ CacheOptimizer,
+ CacheResult,
+ CacheStrategy,
+ OptimizationContext,
+)
+from .dynamic_detector import (
+ DetectorConfig,
+ DynamicCategory,
+ DynamicContentDetector,
+ DynamicSpan,
+ detect_dynamic_content,
+)
+from .registry import CacheOptimizerRegistry
+from .anthropic import AnthropicCacheOptimizer
+from .openai import OpenAICacheOptimizer
+from .google import GoogleCacheOptimizer
+from .semantic import SemanticCacheLayer, SemanticCache
+
+__all__ = [
+ # Base types
+ "BaseCacheOptimizer",
+ "CacheBreakpoint",
+ "CacheConfig",
+ "CacheMetrics",
+ "CacheOptimizer",
+ "CacheResult",
+ "CacheStrategy",
+ "OptimizationContext",
+ # Dynamic content detection
+ "DetectorConfig",
+ "DynamicCategory",
+ "DynamicContentDetector",
+ "DynamicSpan",
+ "detect_dynamic_content",
+ # Registry
+ "CacheOptimizerRegistry",
+ # Provider implementations
+ "AnthropicCacheOptimizer",
+ "OpenAICacheOptimizer",
+ "GoogleCacheOptimizer",
+ # Semantic caching
+ "SemanticCacheLayer",
+ "SemanticCache",
+]
diff --git a/headroom/cache/anthropic.py b/headroom/cache/anthropic.py
new file mode 100644
index 000000000..670ffa5b3
--- /dev/null
+++ b/headroom/cache/anthropic.py
@@ -0,0 +1,518 @@
+"""
+Anthropic Cache Optimizer.
+
+Implements cache optimization for Anthropic's explicit cache_control mechanism.
+Anthropic uses ephemeral cache breakpoints to mark content that should be cached.
+
+Anthropic Caching Characteristics:
+- Explicit cache_control: {"type": "ephemeral"} blocks
+- Minimum 1024 tokens for caching to be effective
+- Maximum 4 cache breakpoints per request
+- 5-minute TTL (extended on cache hit)
+- Cost: 25% MORE to write to cache, 90% LESS to read
+
+Usage:
+ from headroom.cache import AnthropicCacheOptimizer, OptimizationContext
+
+ optimizer = AnthropicCacheOptimizer()
+ context = OptimizationContext(provider="anthropic", model="claude-3-opus")
+
+ result = optimizer.optimize(messages, context)
+ # result.messages now contains cache_control blocks
+"""
+
+from __future__ import annotations
+
+import copy
+import re
+from dataclasses import dataclass, field
+from typing import Any
+
+from .base import (
+ BaseCacheOptimizer,
+ BreakpointLocation,
+ CacheBreakpoint,
+ CacheConfig,
+ CacheMetrics,
+ CacheResult,
+ CacheStrategy,
+ OptimizationContext,
+)
+
+
+# Anthropic-specific constants
+ANTHROPIC_MIN_CACHEABLE_TOKENS = 1024
+ANTHROPIC_MAX_BREAKPOINTS = 4
+ANTHROPIC_CACHE_TTL_SECONDS = 300 # 5 minutes
+ANTHROPIC_WRITE_COST_MULTIPLIER = 1.25 # 25% more to write
+ANTHROPIC_READ_COST_MULTIPLIER = 0.10 # 90% less to read
+
+
+@dataclass
+class ContentSection:
+ """Represents a section of content that may be cacheable."""
+
+ content: str | list[dict[str, Any]]
+ section_type: str # "system", "tools", "examples", "user", "assistant"
+ message_index: int
+ content_index: int | None = None
+ token_count: int = 0
+ is_cacheable: bool = False
+ reason: str = ""
+
+
+@dataclass
+class BreakpointPlan:
+ """Plan for where to insert cache breakpoints."""
+
+ breakpoints: list[CacheBreakpoint] = field(default_factory=list)
+ total_cacheable_tokens: int = 0
+ estimated_savings_percent: float = 0.0
+ warnings: list[str] = field(default_factory=list)
+
+
+class AnthropicCacheOptimizer(BaseCacheOptimizer):
+ """
+ Cache optimizer for Anthropic's explicit cache_control mechanism.
+
+ This optimizer analyzes messages and inserts cache_control blocks at
+ optimal positions to maximize cache hit rates and minimize costs.
+
+ Key features:
+ - Detects cacheable sections (system prompt, tools, few-shot examples)
+ - Respects Anthropic's 1024 token minimum and 4 breakpoint maximum
+ - Stabilizes prefixes by moving dates and normalizing whitespace
+ - Tracks metrics for monitoring and debugging
+ """
+
+ def __init__(self, config: CacheConfig | None = None):
+ super().__init__(config)
+ if self.config.min_cacheable_tokens < ANTHROPIC_MIN_CACHEABLE_TOKENS:
+ self.config.min_cacheable_tokens = ANTHROPIC_MIN_CACHEABLE_TOKENS
+ if self.config.max_breakpoints > ANTHROPIC_MAX_BREAKPOINTS:
+ self.config.max_breakpoints = ANTHROPIC_MAX_BREAKPOINTS
+
+ @property
+ def name(self) -> str:
+ return "anthropic-cache-optimizer"
+
+ @property
+ def provider(self) -> str:
+ return "anthropic"
+
+ @property
+ def strategy(self) -> CacheStrategy:
+ return CacheStrategy.EXPLICIT_BREAKPOINTS
+
+ def optimize(
+ self,
+ messages: list[dict[str, Any]],
+ context: OptimizationContext,
+ config: CacheConfig | None = None,
+ ) -> CacheResult:
+ """
+ Optimize messages for Anthropic's cache.
+
+ Steps:
+ 1. Analyze messages to identify cacheable sections
+ 2. Stabilize the prefix (moves dates, normalizes whitespace)
+ 3. Plan breakpoint placement
+ 4. Insert cache_control blocks at optimal positions
+ 5. Record metrics for monitoring
+ """
+ effective_config = config or self.config
+
+ if not effective_config.enabled:
+ return CacheResult(
+ messages=messages,
+ metrics=CacheMetrics(),
+ transforms_applied=[],
+ )
+
+ optimized_messages = copy.deepcopy(messages)
+ transforms_applied: list[str] = []
+ warnings: list[str] = []
+
+ # Step 1: Analyze content sections
+ sections = self._analyze_sections(optimized_messages)
+
+ # Step 2: Stabilize prefix
+ optimized_messages, stabilization_applied = self._stabilize_prefix(
+ optimized_messages, effective_config
+ )
+ transforms_applied.extend(stabilization_applied)
+
+ # Step 3: Plan breakpoint placement
+ plan = self._plan_breakpoints(sections, effective_config)
+ warnings.extend(plan.warnings)
+
+ # Step 4: Insert cache_control blocks
+ optimized_messages = self._insert_breakpoints(
+ optimized_messages, plan.breakpoints
+ )
+ if plan.breakpoints:
+ transforms_applied.append(
+ f"inserted_{len(plan.breakpoints)}_cache_breakpoints"
+ )
+
+ # Step 5: Compute metrics
+ prefix_content = self._extract_cacheable_content(optimized_messages)
+ prefix_hash = self._compute_prefix_hash(prefix_content)
+
+ cache_hit = False
+ if context.previous_prefix_hash:
+ cache_hit = prefix_hash == context.previous_prefix_hash
+ elif self._previous_prefix_hash:
+ cache_hit = prefix_hash == self._previous_prefix_hash
+
+ total_tokens = sum(s.token_count for s in sections)
+ cacheable_tokens = plan.total_cacheable_tokens
+
+ metrics = CacheMetrics(
+ stable_prefix_tokens=cacheable_tokens,
+ stable_prefix_hash=prefix_hash,
+ breakpoints_inserted=len(plan.breakpoints),
+ breakpoint_locations=plan.breakpoints,
+ prefix_changed_from_previous=not cache_hit,
+ previous_prefix_hash=self._previous_prefix_hash,
+ estimated_cache_hit=cache_hit,
+ estimated_savings_percent=plan.estimated_savings_percent if cache_hit else 0.0,
+ cacheable_tokens=cacheable_tokens,
+ non_cacheable_tokens=total_tokens - cacheable_tokens,
+ cache_ttl_remaining_seconds=ANTHROPIC_CACHE_TTL_SECONDS if cache_hit else None,
+ )
+
+ self._previous_prefix_hash = prefix_hash
+ self._record_metrics(metrics)
+
+ return CacheResult(
+ messages=optimized_messages,
+ metrics=metrics,
+ tokens_before=total_tokens,
+ tokens_after=total_tokens,
+ transforms_applied=transforms_applied,
+ warnings=warnings,
+ )
+
+ def _analyze_sections(
+ self, messages: list[dict[str, Any]]
+ ) -> list[ContentSection]:
+ """Analyze messages to identify distinct content sections."""
+ sections: list[ContentSection] = []
+
+ for idx, message in enumerate(messages):
+ role = message.get("role", "")
+ content = message.get("content", "")
+
+ if role == "system":
+ section_type = "system"
+ elif role == "user":
+ section_type = "examples" if self._looks_like_example(message, messages, idx) else "user"
+ elif role == "assistant":
+ section_type = "examples" if self._looks_like_example(message, messages, idx) else "assistant"
+ else:
+ section_type = role
+
+ # Handle tools
+ if "tools" in message:
+ tool_section = ContentSection(
+ content=message["tools"],
+ section_type="tools",
+ message_index=idx,
+ token_count=self._estimate_tools_tokens(message["tools"]),
+ is_cacheable=True,
+ reason="Tool definitions are static and cacheable",
+ )
+ sections.append(tool_section)
+
+ if isinstance(content, str):
+ token_count = self._count_tokens_estimate(content)
+ is_cacheable, reason = self._assess_cacheability(
+ section_type, token_count, content
+ )
+ sections.append(ContentSection(
+ content=content,
+ section_type=section_type,
+ message_index=idx,
+ token_count=token_count,
+ is_cacheable=is_cacheable,
+ reason=reason,
+ ))
+
+ elif isinstance(content, list):
+ for block_idx, block in enumerate(content):
+ if isinstance(block, dict) and block.get("type") == "text":
+ text = block.get("text", "")
+ token_count = self._count_tokens_estimate(text)
+ is_cacheable, reason = self._assess_cacheability(
+ section_type, token_count, text
+ )
+ sections.append(ContentSection(
+ content=block,
+ section_type=section_type,
+ message_index=idx,
+ content_index=block_idx,
+ token_count=token_count,
+ is_cacheable=is_cacheable,
+ reason=reason,
+ ))
+
+ return sections
+
+ def _assess_cacheability(
+ self, section_type: str, token_count: int, content: str
+ ) -> tuple[bool, str]:
+ """Assess whether a section is cacheable."""
+ if token_count < self.config.min_cacheable_tokens:
+ return False, f"Below minimum tokens ({token_count} < {self.config.min_cacheable_tokens})"
+
+ if section_type == "system":
+ return True, "System prompts are highly cacheable"
+ if section_type == "tools":
+ return True, "Tool definitions are static and cacheable"
+ if section_type == "examples":
+ return True, "Few-shot examples are typically static"
+ if self._has_dynamic_content(content):
+ return False, "Contains dynamic content (dates, times, etc.)"
+ if section_type == "user":
+ return False, "User messages are typically dynamic"
+
+ return True, "Content is large enough for caching"
+
+ def _has_dynamic_content(self, content: str) -> bool:
+ """Check if content has dynamic elements."""
+ for pattern in self.config.date_patterns:
+ if re.search(pattern, content):
+ return True
+ return False
+
+ def _looks_like_example(
+ self,
+ message: dict[str, Any],
+ messages: list[dict[str, Any]],
+ idx: int,
+ ) -> bool:
+ """Determine if a message looks like a few-shot example."""
+ system_idx = -1
+ for i, msg in enumerate(messages):
+ if msg.get("role") == "system":
+ system_idx = i
+ break
+
+ if system_idx >= 0 and idx <= system_idx + 4:
+ role = message.get("role")
+ if role == "user" and idx + 1 < len(messages):
+ if messages[idx + 1].get("role") == "assistant":
+ return True
+ elif role == "assistant" and idx > 0:
+ if messages[idx - 1].get("role") == "user":
+ return True
+
+ content = message.get("content", "")
+ if isinstance(content, str):
+ example_markers = ["example:", "for example", "e.g.", "sample:"]
+ return any(marker in content.lower() for marker in example_markers)
+
+ return False
+
+ def _estimate_tools_tokens(self, tools: Any) -> int:
+ """Estimate token count for tool definitions."""
+ import json
+ try:
+ return self._count_tokens_estimate(json.dumps(tools))
+ except (TypeError, ValueError):
+ return 0
+
+ def _stabilize_prefix(
+ self,
+ messages: list[dict[str, Any]],
+ config: CacheConfig,
+ ) -> tuple[list[dict[str, Any]], list[str]]:
+ """Stabilize the prefix by moving dynamic content."""
+ transforms: list[str] = []
+
+ for message in messages:
+ if message.get("role") != "system":
+ continue
+
+ content = message.get("content", "")
+ if isinstance(content, str):
+ new_content, applied = self._stabilize_text(content, config)
+ if new_content != content:
+ message["content"] = new_content
+ transforms.extend(applied)
+
+ elif isinstance(content, list):
+ for block in content:
+ if isinstance(block, dict) and block.get("type") == "text":
+ text = block.get("text", "")
+ new_text, applied = self._stabilize_text(text, config)
+ if new_text != text:
+ block["text"] = new_text
+ transforms.extend(applied)
+
+ return messages, transforms
+
+ def _stabilize_text(
+ self, text: str, config: CacheConfig
+ ) -> tuple[str, list[str]]:
+ """Stabilize a text string."""
+ transforms: list[str] = []
+ result = text
+
+ extracted_dates: list[str] = []
+ for pattern in config.date_patterns:
+ matches = re.findall(pattern, result)
+ if matches:
+ extracted_dates.extend(matches)
+ result = re.sub(pattern, "", result)
+ transforms.append("extracted_dates")
+
+ if config.normalize_whitespace:
+ new_result = re.sub(r"[ \t]+", " ", result)
+ if new_result != result:
+ result = new_result
+ transforms.append("normalized_spaces")
+
+ if config.collapse_blank_lines:
+ new_result = re.sub(r"\n{3,}", "\n\n", result)
+ if new_result != result:
+ result = new_result
+ transforms.append("collapsed_blank_lines")
+
+ result = result.strip()
+
+ if extracted_dates:
+ result = result + config.dynamic_separator + " ".join(extracted_dates)
+
+ return result, list(set(transforms))
+
+ def _plan_breakpoints(
+ self,
+ sections: list[ContentSection],
+ config: CacheConfig,
+ ) -> BreakpointPlan:
+ """Plan where to place cache breakpoints."""
+ plan = BreakpointPlan()
+
+ cacheable = [s for s in sections if s.is_cacheable]
+ if not cacheable:
+ plan.warnings.append("No sections meet caching requirements")
+ return plan
+
+ priority_order = {"system": 0, "tools": 1, "examples": 2}
+ cacheable.sort(key=lambda s: priority_order.get(s.section_type, 3))
+
+ selected: list[ContentSection] = []
+ accumulated_tokens = 0
+
+ for section in cacheable:
+ if len(selected) >= config.max_breakpoints:
+ plan.warnings.append(
+ f"Reached maximum breakpoints ({config.max_breakpoints})"
+ )
+ break
+
+ selected.append(section)
+ accumulated_tokens += section.token_count
+
+ for section in selected:
+ location = self._section_type_to_location(section.section_type)
+ breakpoint = CacheBreakpoint(
+ message_index=section.message_index,
+ location=location,
+ content_index=section.content_index,
+ tokens_at_breakpoint=section.token_count,
+ reason=section.reason,
+ )
+ plan.breakpoints.append(breakpoint)
+
+ plan.total_cacheable_tokens = accumulated_tokens
+ if accumulated_tokens > 0:
+ plan.estimated_savings_percent = 90.0
+
+ return plan
+
+ def _section_type_to_location(self, section_type: str) -> BreakpointLocation:
+ """Convert section type to breakpoint location enum."""
+ mapping = {
+ "system": BreakpointLocation.AFTER_SYSTEM,
+ "tools": BreakpointLocation.AFTER_TOOLS,
+ "examples": BreakpointLocation.AFTER_EXAMPLES,
+ }
+ return mapping.get(section_type, BreakpointLocation.CUSTOM)
+
+ def _insert_breakpoints(
+ self,
+ messages: list[dict[str, Any]],
+ breakpoints: list[CacheBreakpoint],
+ ) -> list[dict[str, Any]]:
+ """Insert cache_control blocks at specified positions."""
+ for bp in breakpoints:
+ if bp.message_index >= len(messages):
+ continue
+
+ message = messages[bp.message_index]
+ content = message.get("content", "")
+
+ if isinstance(content, str):
+ message["content"] = [
+ {
+ "type": "text",
+ "text": content,
+ "cache_control": {"type": "ephemeral"},
+ }
+ ]
+ elif isinstance(content, list):
+ if bp.content_index is not None and bp.content_index < len(content):
+ block = content[bp.content_index]
+ if isinstance(block, dict):
+ block["cache_control"] = {"type": "ephemeral"}
+ elif content:
+ last_block = content[-1]
+ if isinstance(last_block, dict):
+ last_block["cache_control"] = {"type": "ephemeral"}
+
+ return messages
+
+ def _extract_cacheable_content(self, messages: list[dict[str, Any]]) -> str:
+ """Extract content that has cache_control markers for hashing."""
+ parts: list[str] = []
+
+ for message in messages:
+ content = message.get("content", "")
+ if isinstance(content, list):
+ for block in content:
+ if isinstance(block, dict) and "cache_control" in block:
+ text = block.get("text", "")
+ if text:
+ parts.append(text)
+ elif isinstance(content, str) and message.get("role") == "system":
+ parts.append(content)
+
+ return "\n".join(parts)
+
+ def estimate_savings(
+ self,
+ messages: list[dict[str, Any]],
+ context: OptimizationContext,
+ ) -> float:
+ """Estimate potential savings from caching."""
+ sections = self._analyze_sections(messages)
+ plan = self._plan_breakpoints(sections, self.config)
+
+ if plan.total_cacheable_tokens == 0:
+ return 0.0
+
+ total_tokens = sum(s.token_count for s in sections)
+ cacheable_ratio = plan.total_cacheable_tokens / total_tokens
+ return 90.0 * cacheable_ratio
+
+ def get_cache_write_cost_multiplier(self) -> float:
+ return ANTHROPIC_WRITE_COST_MULTIPLIER
+
+ def get_cache_read_cost_multiplier(self) -> float:
+ return ANTHROPIC_READ_COST_MULTIPLIER
+
+ def get_cache_ttl_seconds(self) -> int:
+ return ANTHROPIC_CACHE_TTL_SECONDS
diff --git a/headroom/cache/base.py b/headroom/cache/base.py
new file mode 100644
index 000000000..617c961fa
--- /dev/null
+++ b/headroom/cache/base.py
@@ -0,0 +1,339 @@
+"""
+Base types and interfaces for cache optimization.
+
+This module defines the core abstractions that all cache optimizers implement.
+The design allows for provider-specific implementations while maintaining a
+consistent interface for users.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import Any, Literal, Protocol, runtime_checkable
+
+
+class CacheStrategy(Enum):
+ """Cache optimization strategy."""
+
+ # Just stabilize prefix (move dates, normalize whitespace)
+ PREFIX_STABILIZATION = "prefix_stabilization"
+
+ # Insert explicit cache breakpoints (Anthropic)
+ EXPLICIT_BREAKPOINTS = "explicit_breakpoints"
+
+ # Manage separate cached content objects (Google)
+ CACHED_CONTENT = "cached_content"
+
+ # No optimization possible (provider doesn't support caching)
+ NONE = "none"
+
+
+class BreakpointLocation(Enum):
+ """Where to insert cache breakpoints."""
+
+ AFTER_SYSTEM = "after_system"
+ AFTER_TOOLS = "after_tools"
+ AFTER_EXAMPLES = "after_examples"
+ CUSTOM = "custom"
+
+
+@dataclass
+class CacheBreakpoint:
+ """
+ Represents a cache breakpoint location.
+
+ For Anthropic, this maps to cache_control blocks.
+ For other providers, this is informational.
+ """
+
+ # Message index where breakpoint should be inserted
+ message_index: int
+
+ # Location type
+ location: BreakpointLocation
+
+ # For content arrays, index within the content
+ content_index: int | None = None
+
+ # Token count at this breakpoint
+ tokens_at_breakpoint: int = 0
+
+ # Reason for this breakpoint
+ reason: str = ""
+
+
+@dataclass
+class CacheConfig:
+ """Configuration for cache optimization."""
+
+ # Whether to optimize at all
+ enabled: bool = True
+
+ # Strategy to use (auto-detected if None)
+ strategy: CacheStrategy | None = None
+
+ # Minimum tokens before caching makes sense
+ min_cacheable_tokens: int = 1024
+
+ # Maximum number of breakpoints (Anthropic limit is 4)
+ max_breakpoints: int = 4
+
+ # Patterns to extract and move to dynamic section
+ date_patterns: list[str] = field(default_factory=lambda: [
+ r"Today is \w+ \d{1,2},? \d{4}\.?",
+ r"Current date: \d{4}-\d{2}-\d{2}",
+ r"The current time is .+\.",
+ ])
+
+ # Whether to normalize whitespace
+ normalize_whitespace: bool = True
+
+ # Collapse multiple blank lines
+ collapse_blank_lines: bool = True
+
+ # Separator between static and dynamic content
+ dynamic_separator: str = "\n\n---\n\n"
+
+ # Dynamic content detection tiers (for OpenAI prefix stabilization)
+ # - "regex": Fast pattern matching (~0ms) - always recommended
+ # - "ner": Named Entity Recognition via spaCy (~5-10ms) - catches names, money, etc.
+ # - "semantic": Embedding similarity (~20-50ms) - catches volatile patterns
+ # Default is regex-only for speed. Add tiers for better detection at cost of latency.
+ dynamic_detection_tiers: list[Literal["regex", "ner", "semantic"]] = field(
+ default_factory=lambda: ["regex"]
+ )
+
+ # For semantic caching
+ semantic_cache_enabled: bool = False
+ semantic_similarity_threshold: float = 0.95
+ semantic_cache_ttl_seconds: int = 300
+
+
+@dataclass
+class CacheMetrics:
+ """Metrics about cache optimization."""
+
+ # Prefix analysis
+ stable_prefix_tokens: int = 0
+ stable_prefix_hash: str = ""
+
+ # Breakpoint info
+ breakpoints_inserted: int = 0
+ breakpoint_locations: list[CacheBreakpoint] = field(default_factory=list)
+
+ # Cache hit estimation
+ prefix_changed_from_previous: bool = False
+ previous_prefix_hash: str | None = None
+ estimated_cache_hit: bool = False
+
+ # Savings estimation
+ estimated_savings_percent: float = 0.0
+ cacheable_tokens: int = 0
+ non_cacheable_tokens: int = 0
+
+ # Provider-specific
+ provider_cache_id: str | None = None # For Google's CachedContent
+ cache_ttl_remaining_seconds: int | None = None
+
+
+@dataclass
+class OptimizationContext:
+ """Context for optimization request."""
+
+ # Request tracking
+ request_id: str = ""
+ timestamp: datetime = field(default_factory=datetime.now)
+
+ # Provider info
+ provider: str = ""
+ model: str = ""
+
+ # Query for relevance (used by semantic cache)
+ query: str | None = None
+
+ # Previous request info (for cache hit detection)
+ previous_prefix_hash: str | None = None
+
+ # Additional metadata
+ metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class CacheResult:
+ """Result of cache optimization."""
+
+ # Optimized messages
+ messages: list[dict[str, Any]]
+
+ # Whether this was a semantic cache hit
+ semantic_cache_hit: bool = False
+
+ # Cached response (if semantic cache hit)
+ cached_response: Any | None = None
+
+ # Optimization metrics
+ metrics: CacheMetrics = field(default_factory=CacheMetrics)
+
+ # Tokens before/after
+ tokens_before: int = 0
+ tokens_after: int = 0
+
+ # Transforms applied
+ transforms_applied: list[str] = field(default_factory=list)
+
+ # Warnings
+ warnings: list[str] = field(default_factory=list)
+
+
+@runtime_checkable
+class CacheOptimizer(Protocol):
+ """
+ Protocol for cache optimizers.
+
+ All provider-specific optimizers must implement this interface.
+ This allows for easy swapping of implementations and plugin registration.
+ """
+
+ @property
+ def name(self) -> str:
+ """Name of this optimizer."""
+ ...
+
+ @property
+ def provider(self) -> str:
+ """Provider this optimizer is for."""
+ ...
+
+ @property
+ def strategy(self) -> CacheStrategy:
+ """The caching strategy this optimizer uses."""
+ ...
+
+ def optimize(
+ self,
+ messages: list[dict[str, Any]],
+ context: OptimizationContext,
+ config: CacheConfig | None = None,
+ ) -> CacheResult:
+ """
+ Optimize messages for caching.
+
+ Args:
+ messages: The messages to optimize.
+ context: Optimization context with request info.
+ config: Optional configuration override.
+
+ Returns:
+ CacheResult with optimized messages and metrics.
+ """
+ ...
+
+ def get_metrics(self) -> CacheMetrics:
+ """Get aggregated metrics from this optimizer."""
+ ...
+
+ def estimate_savings(
+ self,
+ messages: list[dict[str, Any]],
+ context: OptimizationContext,
+ ) -> float:
+ """
+ Estimate potential savings from optimization.
+
+ Returns:
+ Estimated savings as a percentage (0-100).
+ """
+ ...
+
+
+class BaseCacheOptimizer(ABC):
+ """
+ Abstract base class for cache optimizers.
+
+ Provides common functionality for all optimizers.
+ """
+
+ def __init__(self, config: CacheConfig | None = None):
+ self.config = config or CacheConfig()
+ self._metrics_history: list[CacheMetrics] = []
+ self._previous_prefix_hash: str | None = None
+
+ @property
+ @abstractmethod
+ def name(self) -> str:
+ """Name of this optimizer."""
+ ...
+
+ @property
+ @abstractmethod
+ def provider(self) -> str:
+ """Provider this optimizer is for."""
+ ...
+
+ @property
+ @abstractmethod
+ def strategy(self) -> CacheStrategy:
+ """The caching strategy this optimizer uses."""
+ ...
+
+ @abstractmethod
+ def optimize(
+ self,
+ messages: list[dict[str, Any]],
+ context: OptimizationContext,
+ config: CacheConfig | None = None,
+ ) -> CacheResult:
+ """Optimize messages for caching."""
+ ...
+
+ def get_metrics(self) -> CacheMetrics:
+ """Get aggregated metrics."""
+ if not self._metrics_history:
+ return CacheMetrics()
+
+ # Return most recent metrics
+ return self._metrics_history[-1]
+
+ def estimate_savings(
+ self,
+ messages: list[dict[str, Any]],
+ context: OptimizationContext,
+ ) -> float:
+ """Estimate potential savings."""
+ # Default implementation - subclasses can override
+ result = self.optimize(messages, context)
+ return result.metrics.estimated_savings_percent
+
+ def _record_metrics(self, metrics: CacheMetrics) -> None:
+ """Record metrics for history."""
+ self._metrics_history.append(metrics)
+ # Keep only last 100 entries
+ if len(self._metrics_history) > 100:
+ self._metrics_history = self._metrics_history[-100:]
+
+ def _compute_prefix_hash(self, content: str) -> str:
+ """Compute a short hash of content."""
+ import hashlib
+ return hashlib.sha256(content.encode()).hexdigest()[:12]
+
+ def _extract_system_content(self, messages: list[dict[str, Any]]) -> str:
+ """Extract content from system messages."""
+ parts = []
+ for msg in messages:
+ if msg.get("role") == "system":
+ content = msg.get("content", "")
+ if isinstance(content, str):
+ parts.append(content)
+ elif isinstance(content, list):
+ # Handle content blocks
+ for block in content:
+ if isinstance(block, dict) and block.get("type") == "text":
+ parts.append(block.get("text", ""))
+ return "\n".join(parts)
+
+ def _count_tokens_estimate(self, text: str) -> int:
+ """Rough token count estimate (4 chars per token)."""
+ return len(text) // 4
diff --git a/headroom/cache/dynamic_detector.py b/headroom/cache/dynamic_detector.py
new file mode 100644
index 000000000..01a2d60ab
--- /dev/null
+++ b/headroom/cache/dynamic_detector.py
@@ -0,0 +1,951 @@
+"""
+Dynamic Content Detector for Cache Optimization.
+
+This module provides a scalable, language-agnostic approach to detecting dynamic
+content in prompts. Dynamic content (dates, prices, user data, session info) breaks
+cache prefixes. By detecting and moving dynamic content to the end, we maximize
+cache hits.
+
+Design Philosophy:
+ - NO HARDCODED PATTERNS for locale-specific content (no month names, etc.)
+ - Structural detection: "Label: value" patterns where LABEL indicates dynamism
+ - Entropy-based detection: High entropy = likely dynamic (UUIDs, tokens, hashes)
+ - Universal patterns only: ISO 8601, UUIDs, Unix timestamps (truly universal)
+
+Tiers (configurable, each adds latency):
+ Tier 1: Regex (~0ms) - Structural patterns, universal formats, entropy-based
+ Tier 2: NER (~5-10ms) - Named Entity Recognition for names, money, orgs
+ Tier 3: Semantic (~20-50ms) - Embedding similarity to known dynamic patterns
+
+Usage:
+ from headroom.cache.dynamic_detector import DynamicContentDetector
+
+ detector = DynamicContentDetector(tiers=["regex", "ner"])
+ result = detector.detect("Session: abc123. User: John paid $500.")
+
+ # result.spans = [
+ # DynamicSpan(text="Session: abc123", category="session", tier="regex", ...),
+ # DynamicSpan(text="John", category="person", tier="ner", ...),
+ # DynamicSpan(text="$500", category="money", tier="ner", ...),
+ # ]
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Literal
+
+# Optional imports - graceful degradation
+_SPACY_AVAILABLE = False
+_SENTENCE_TRANSFORMERS_AVAILABLE = False
+
+try:
+ import spacy
+ _SPACY_AVAILABLE = True
+except ImportError:
+ spacy = None # type: ignore
+
+try:
+ from sentence_transformers import SentenceTransformer
+ import numpy as np
+ _SENTENCE_TRANSFORMERS_AVAILABLE = True
+except ImportError:
+ SentenceTransformer = None # type: ignore
+ np = None # type: ignore
+
+
+class DynamicCategory(str, Enum):
+ """Categories of dynamic content."""
+
+ # Tier 1: Structural/Regex detectable
+ DATE = "date"
+ TIME = "time"
+ DATETIME = "datetime"
+ TIMESTAMP = "timestamp"
+ UUID = "uuid"
+ REQUEST_ID = "request_id"
+ VERSION = "version"
+ SESSION = "session"
+ USER_DATA = "user_data"
+ IDENTIFIER = "identifier" # Generic high-entropy ID
+
+ # Tier 2: NER detectable
+ PERSON = "person"
+ MONEY = "money"
+ ORG = "org"
+ LOCATION = "location"
+
+ # Tier 3: Semantic
+ VOLATILE = "volatile" # Semantically detected as changing
+ REALTIME = "realtime"
+
+ # Fallback
+ UNKNOWN = "unknown"
+
+
+@dataclass
+class DynamicSpan:
+ """A span of dynamic content detected in text."""
+
+ # The actual text matched
+ text: str
+
+ # Position in original content
+ start: int
+ end: int
+
+ # What category of dynamic content
+ category: DynamicCategory
+
+ # Which tier detected it
+ tier: Literal["regex", "ner", "semantic"]
+
+ # Confidence score (0-1)
+ confidence: float = 1.0
+
+ # Additional metadata (pattern name, entity type, etc.)
+ metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class DetectionResult:
+ """Result of dynamic content detection."""
+
+ # All detected spans
+ spans: list[DynamicSpan]
+
+ # Content with dynamic parts removed
+ static_content: str
+
+ # Content that was extracted (for reinsertion at end)
+ dynamic_content: str
+
+ # Which tiers were used
+ tiers_used: list[str]
+
+ # Processing time in milliseconds
+ processing_time_ms: float = 0.0
+
+ # Any warnings (e.g., "spaCy not available, skipping NER")
+ warnings: list[str] = field(default_factory=list)
+
+
+@dataclass
+class DetectorConfig:
+ """Configuration for the dynamic content detector."""
+
+ # Which tiers to enable (order matters - later tiers can use earlier results)
+ tiers: list[Literal["regex", "ner", "semantic"]] = field(
+ default_factory=lambda: ["regex"]
+ )
+
+ # Tier 1: Structural labels that indicate dynamic content
+ # These are the KEY names that hint the VALUE is dynamic
+ # Users can add domain-specific labels
+ dynamic_labels: list[str] = field(default_factory=lambda: [
+ # Time-related
+ "date", "time", "timestamp", "datetime", "created", "updated",
+ "modified", "expires", "last", "current", "today", "now",
+ # Identifiers
+ "id", "uuid", "guid", "session", "request", "trace", "span",
+ "transaction", "correlation", "token", "key", "secret",
+ # User-related
+ "user", "username", "email", "name", "phone", "address",
+ "customer", "client", "employee", "member",
+ # System state
+ "version", "build", "commit", "branch", "revision",
+ "status", "state", "count", "total", "balance", "remaining",
+ "load", "queue", "active", "pending",
+ # Order/ticket related
+ "order", "ticket", "case", "invoice", "reference",
+ ])
+
+ # Tier 1: Custom regex patterns (user-provided)
+ custom_patterns: list[tuple[str, DynamicCategory]] = field(
+ default_factory=list
+ )
+
+ # Entropy threshold for detecting random strings (0-1 scale normalized)
+ # Higher = more selective (only very random strings)
+ entropy_threshold: float = 0.7
+
+ # Minimum length for entropy-based detection
+ min_entropy_length: int = 8
+
+ # Tier 2: NER config
+ spacy_model: str = "en_core_web_sm"
+ ner_entity_types: list[str] = field(
+ default_factory=lambda: ["DATE", "TIME", "MONEY", "PERSON", "ORG", "GPE"]
+ )
+
+ # Tier 3: Semantic config
+ embedding_model: str = "all-MiniLM-L6-v2"
+ semantic_threshold: float = 0.7
+
+ # General
+ min_span_length: int = 2
+ merge_overlapping: bool = True
+
+
+def calculate_entropy(s: str) -> float:
+ """
+ Calculate Shannon entropy of a string, normalized to 0-1.
+
+ Higher entropy = more random/unpredictable = likely dynamic.
+ - "aaaaaaa" -> ~0 (low entropy, predictable)
+ - "a1b2c3d4" -> ~0.7 (medium entropy)
+ - "550e8400-e29b-41d4" -> ~0.9 (high entropy, random-looking)
+
+ Returns:
+ Normalized entropy (0-1). Higher = more likely dynamic.
+ """
+ if not s:
+ return 0.0
+
+ # Count character frequencies
+ freq: dict[str, int] = {}
+ for char in s:
+ freq[char] = freq.get(char, 0) + 1
+
+ # Calculate entropy
+ length = len(s)
+ entropy = 0.0
+ for count in freq.values():
+ p = count / length
+ entropy -= p * math.log2(p)
+
+ # Normalize: max entropy for string of length n with k unique chars
+ # is log2(min(n, alphabet_size)). We'll normalize by log2(length)
+ # to get a 0-1 scale
+ max_entropy = math.log2(length) if length > 1 else 1.0
+ return entropy / max_entropy if max_entropy > 0 else 0.0
+
+
+class RegexDetector:
+ """
+ Tier 1: Scalable pattern detection.
+
+ Uses THREE strategies (no hardcoded month names!):
+ 1. Structural: "Label: value" patterns where label indicates dynamic content
+ 2. Universal: Truly universal formats (ISO 8601, UUID, Unix timestamps)
+ 3. Entropy: High-entropy strings (tokens, hashes, IDs)
+ """
+
+ # Universal patterns (these formats are language-agnostic)
+ UNIVERSAL_PATTERNS = [
+ # UUID - truly universal format
+ (r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}",
+ DynamicCategory.UUID, "uuid"),
+
+ # ISO 8601 datetime (most universal date format)
+ (r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?",
+ DynamicCategory.DATETIME, "iso_datetime"),
+
+ # ISO 8601 date only
+ (r"\d{4}-\d{2}-\d{2}(?!\d)",
+ DynamicCategory.DATE, "iso_date"),
+
+ # Unix timestamps (10-13 digits, but NOT within longer numbers)
+ (r"(?<![0-9])\d{10,13}(?![0-9])",
+ DynamicCategory.TIMESTAMP, "unix_timestamp"),
+
+ # 24-hour time HH:MM:SS or HH:MM
+ (r"(?<![0-9])\d{1,2}:\d{2}(?::\d{2})?(?:\s*(?:AM|PM|am|pm))?(?![0-9])",
+ DynamicCategory.TIME, "time"),
+
+ # Version numbers with v prefix (unambiguous)
+ (r"\bv\d+\.\d+(?:\.\d+)?(?:-[a-zA-Z0-9.]+)?",
+ DynamicCategory.VERSION, "version"),
+
+ # API key/token patterns (prefix + random string)
+ (r"\b(?:sk|pk|api|key|token|bearer|auth)[-_][a-zA-Z0-9]{16,}",
+ DynamicCategory.REQUEST_ID, "api_key"),
+
+ # Common prefixed IDs (req_, sess_, txn_, etc.)
+ (r"\b[a-z]{2,6}_[a-zA-Z0-9]{8,}",
+ DynamicCategory.REQUEST_ID, "prefixed_id"),
+
+ # Hex strings of common ID lengths (32 = MD5, 40 = SHA1, 64 = SHA256)
+ (r"\b[a-fA-F0-9]{32}\b",
+ DynamicCategory.IDENTIFIER, "hex_32"),
+ (r"\b[a-fA-F0-9]{40}\b",
+ DynamicCategory.IDENTIFIER, "hex_40"),
+ (r"\b[a-fA-F0-9]{64}\b",
+ DynamicCategory.IDENTIFIER, "hex_64"),
+
+ # JWT tokens (three base64 sections separated by dots)
+ (r"eyJ[a-zA-Z0-9_-]+\.eyJ[a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+",
+ DynamicCategory.REQUEST_ID, "jwt"),
+ ]
+
+ def __init__(self, config: DetectorConfig):
+ """Initialize regex detector."""
+ self.config = config
+
+ # Compile universal patterns
+ self._universal_patterns: list[tuple[re.Pattern[str], DynamicCategory, str]] = [
+ (re.compile(pattern), category, name)
+ for pattern, category, name in self.UNIVERSAL_PATTERNS
+ ]
+
+ # Build structural pattern from dynamic labels
+ # Pattern: "label" followed by separator then value
+ labels_pattern = "|".join(re.escape(label) for label in config.dynamic_labels)
+ self._structural_pattern = re.compile(
+ rf"(?P<label>(?:{labels_pattern}))(?P<sep>\s*[:=]\s*|\s+)(?P<value>[^\n,;]+)",
+ re.IGNORECASE
+ )
+
+ # Compile custom patterns
+ self._custom_patterns: list[tuple[re.Pattern[str], DynamicCategory]] = [
+ (re.compile(pattern, re.IGNORECASE), category)
+ for pattern, category in config.custom_patterns
+ ]
+
+ def detect(self, content: str) -> list[DynamicSpan]:
+ """Detect dynamic content using structural, universal, and entropy detection."""
+ spans: list[DynamicSpan] = []
+ seen_ranges: set[tuple[int, int]] = set()
+
+ # 1. Universal patterns first (most specific)
+ for pattern, category, pattern_name in self._universal_patterns:
+ for match in pattern.finditer(content):
+ start, end = match.start(), match.end()
+ if self._is_overlapping(start, end, seen_ranges):
+ continue
+ if end - start < self.config.min_span_length:
+ continue
+
+ spans.append(DynamicSpan(
+ text=match.group(),
+ start=start,
+ end=end,
+ category=category,
+ tier="regex",
+ confidence=1.0,
+ metadata={"pattern": pattern_name, "method": "universal"},
+ ))
+ seen_ranges.add((start, end))
+
+ # 2. Structural detection: "Label: value" patterns
+ for match in self._structural_pattern.finditer(content):
+ # Get the full match range
+ start, end = match.start(), match.end()
+
+ # Skip if overlaps with universal patterns
+ if self._is_overlapping(start, end, seen_ranges):
+ continue
+
+ label = match.group("label").lower()
+ value = match.group("value").strip()
+
+ # Determine category from label
+ category = self._categorize_label(label)
+
+ # Only add the value portion (keep label as static)
+ value_start = match.start("value")
+ value_end = match.end("value")
+
+ # Skip if value is too short or empty
+ if value_end - value_start < self.config.min_span_length:
+ continue
+ if not value.strip():
+ continue
+
+ spans.append(DynamicSpan(
+ text=value,
+ start=value_start,
+ end=value_end,
+ category=category,
+ tier="regex",
+ confidence=0.9,
+ metadata={"pattern": "structural", "method": "structural", "label": label},
+ ))
+ seen_ranges.add((value_start, value_end))
+
+ # 3. Entropy-based detection for remaining potential IDs
+ spans.extend(self._detect_high_entropy(content, seen_ranges))
+
+ # 4. Custom patterns
+ for pattern, category in self._custom_patterns:
+ for match in pattern.finditer(content):
+ start, end = match.start(), match.end()
+ if self._is_overlapping(start, end, seen_ranges):
+ continue
+ if end - start < self.config.min_span_length:
+ continue
+
+ spans.append(DynamicSpan(
+ text=match.group(),
+ start=start,
+ end=end,
+ category=category,
+ tier="regex",
+ confidence=0.8,
+ metadata={"pattern": "custom", "method": "custom"},
+ ))
+ seen_ranges.add((start, end))
+
+ return sorted(spans, key=lambda s: s.start)
+
+ def _detect_high_entropy(
+ self,
+ content: str,
+ seen_ranges: set[tuple[int, int]],
+ ) -> list[DynamicSpan]:
+ """
+ Detect high-entropy strings that look like IDs/tokens.
+
+ Finds alphanumeric sequences and checks their entropy.
+ High entropy = likely random/generated = dynamic.
+ """
+ spans: list[DynamicSpan] = []
+
+ # Find alphanumeric sequences (potential IDs)
+ # Must be at least min_entropy_length chars, mix of letters/numbers
+ pattern = re.compile(r"\b[a-zA-Z0-9_-]{8,}\b")
+
+ for match in pattern.finditer(content):
+ start, end = match.start(), match.end()
+ text = match.group()
+
+ # Skip if already detected
+ if self._is_overlapping(start, end, seen_ranges):
+ continue
+
+ # Skip if too short
+ if len(text) < self.config.min_entropy_length:
+ continue
+
+ # Skip if all letters or all numbers (not random-looking)
+ if text.isalpha() or text.isdigit():
+ continue
+
+ # Skip common words that might look like IDs
+ if text.lower() in {"username", "password", "localhost", "undefined"}:
+ continue
+
+ # Calculate entropy
+ entropy = calculate_entropy(text)
+
+ if entropy >= self.config.entropy_threshold:
+ spans.append(DynamicSpan(
+ text=text,
+ start=start,
+ end=end,
+ category=DynamicCategory.IDENTIFIER,
+ tier="regex",
+ confidence=entropy, # Use entropy as confidence
+ metadata={"pattern": "entropy", "method": "entropy", "entropy": entropy},
+ ))
+ seen_ranges.add((start, end))
+
+ return spans
+
+ def _is_overlapping(
+ self,
+ start: int,
+ end: int,
+ seen_ranges: set[tuple[int, int]],
+ ) -> bool:
+ """Check if range overlaps with any existing range."""
+ return any(
+ not (end <= s or start >= e)
+ for s, e in seen_ranges
+ )
+
+ def _categorize_label(self, label: str) -> DynamicCategory:
+ """Categorize based on the label name."""
+ label = label.lower()
+
+ # Time-related
+ if label in {"date", "datetime", "created", "updated", "modified", "expires", "today"}:
+ return DynamicCategory.DATE
+ if label in {"time", "timestamp", "now"}:
+ return DynamicCategory.TIMESTAMP
+ if label == "current":
+ return DynamicCategory.DATETIME
+
+ # Identifiers
+ if label in {"id", "uuid", "guid"}:
+ return DynamicCategory.UUID
+ if label in {"session", "request", "trace", "span", "transaction", "correlation"}:
+ return DynamicCategory.SESSION
+ if label in {"token", "key", "secret"}:
+ return DynamicCategory.REQUEST_ID
+
+ # User-related
+ if label in {"user", "username", "email", "name", "phone", "address",
+ "customer", "client", "employee", "member"}:
+ return DynamicCategory.USER_DATA
+
+ # System state
+ if label in {"version", "build", "commit", "branch", "revision"}:
+ return DynamicCategory.VERSION
+ if label in {"status", "state", "count", "total", "balance", "remaining",
+ "load", "queue", "active", "pending"}:
+ return DynamicCategory.VOLATILE
+
+ # Order/ticket
+ if label in {"order", "ticket", "case", "invoice", "reference"}:
+ return DynamicCategory.REQUEST_ID
+
+ return DynamicCategory.UNKNOWN
+
+
+class NERDetector:
+ """Tier 2: spaCy-based Named Entity Recognition."""
+
+ # Map spaCy entity types to our categories
+ ENTITY_MAP = {
+ "DATE": DynamicCategory.DATE,
+ "TIME": DynamicCategory.TIME,
+ "MONEY": DynamicCategory.MONEY,
+ "PERSON": DynamicCategory.PERSON,
+ "ORG": DynamicCategory.ORG,
+ "GPE": DynamicCategory.LOCATION, # Geo-Political Entity
+ "LOC": DynamicCategory.LOCATION,
+ "FAC": DynamicCategory.LOCATION, # Facility
+ "CARDINAL": DynamicCategory.UNKNOWN, # Numbers
+ "ORDINAL": DynamicCategory.UNKNOWN,
+ }
+
+ def __init__(self, config: DetectorConfig):
+ """Initialize NER detector, loading spaCy model."""
+ self.config = config
+ self._nlp = None
+ self._load_error: str | None = None
+
+ if not _SPACY_AVAILABLE:
+ self._load_error = "spaCy not installed. Install with: pip install spacy && python -m spacy download en_core_web_sm"
+ return
+
+ try:
+ self._nlp = spacy.load(config.spacy_model)
+ except OSError:
+ self._load_error = f"spaCy model '{config.spacy_model}' not found. Install with: python -m spacy download {config.spacy_model}"
+
+ @property
+ def is_available(self) -> bool:
+ """Check if NER is available."""
+ return self._nlp is not None
+
+ def detect(
+ self,
+ content: str,
+ existing_spans: list[DynamicSpan] | None = None,
+ ) -> tuple[list[DynamicSpan], str | None]:
+ """
+ Detect dynamic content using NER.
+
+ Args:
+ content: Text to analyze.
+ existing_spans: Spans already detected (to avoid duplicates).
+
+ Returns:
+ Tuple of (new_spans, warning_message).
+ """
+ if not self.is_available:
+ return [], self._load_error
+
+ # Get existing ranges to avoid duplicates
+ existing_ranges = set()
+ if existing_spans:
+ existing_ranges = {(s.start, s.end) for s in existing_spans}
+
+ doc = self._nlp(content)
+ spans: list[DynamicSpan] = []
+
+ for ent in doc.ents:
+ # Skip entity types we don't care about
+ if ent.label_ not in self.config.ner_entity_types:
+ continue
+
+ # Skip if already detected by regex
+ if (ent.start_char, ent.end_char) in existing_ranges:
+ continue
+
+ # Check for overlap with existing spans
+ overlaps = any(
+ not (ent.end_char <= s or ent.start_char >= e)
+ for s, e in existing_ranges
+ )
+ if overlaps:
+ continue
+
+ # Map to our category
+ category = self.ENTITY_MAP.get(ent.label_, DynamicCategory.UNKNOWN)
+
+ # Skip unknown categories
+ if category == DynamicCategory.UNKNOWN:
+ continue
+
+ spans.append(DynamicSpan(
+ text=ent.text,
+ start=ent.start_char,
+ end=ent.end_char,
+ category=category,
+ tier="ner",
+ confidence=0.9,
+ metadata={"entity_type": ent.label_},
+ ))
+ existing_ranges.add((ent.start_char, ent.end_char))
+
+ return sorted(spans, key=lambda s: s.start), None
+
+
+class SemanticDetector:
+ """Tier 3: Embedding-based semantic detection."""
+
+ # Known phrases that indicate dynamic content
+ # These are SEMANTIC patterns, not literal strings to match
+ DYNAMIC_EXEMPLARS = [
+ # Time-sensitive
+ "The current date is",
+ "As of today",
+ "Updated on",
+ "Last refreshed",
+ "Real-time data",
+ "Live prices",
+ "Current stock price",
+
+ # Session-specific
+ "Your session ID",
+ "Your account balance",
+ "Your recent orders",
+ "Your conversation history",
+
+ # User-specific
+ "Hello [user]",
+ "Dear customer",
+ "Your name is",
+
+ # System state
+ "Server status",
+ "System load",
+ "Queue length",
+ "Active users",
+ ]
+
+ def __init__(self, config: DetectorConfig):
+ """Initialize semantic detector with embedding model."""
+ self.config = config
+ self._model = None
+ self._exemplar_embeddings = None
+ self._load_error: str | None = None
+
+ if not _SENTENCE_TRANSFORMERS_AVAILABLE:
+ self._load_error = "sentence-transformers not installed. Install with: pip install sentence-transformers"
+ return
+
+ try:
+ self._model = SentenceTransformer(config.embedding_model)
+ # Pre-compute exemplar embeddings
+ self._exemplar_embeddings = self._model.encode(
+ self.DYNAMIC_EXEMPLARS,
+ convert_to_numpy=True,
+ )
+ except Exception as e:
+ self._load_error = f"Failed to load embedding model: {e}"
+
+ @property
+ def is_available(self) -> bool:
+ """Check if semantic detection is available."""
+ return self._model is not None
+
+ def detect(
+ self,
+ content: str,
+ existing_spans: list[DynamicSpan] | None = None,
+ ) -> tuple[list[DynamicSpan], str | None]:
+ """
+ Detect dynamic content using semantic similarity.
+
+ Splits content into sentences and checks each against known
+ dynamic patterns using embedding similarity.
+
+ Args:
+ content: Text to analyze.
+ existing_spans: Spans already detected (to avoid duplicates).
+
+ Returns:
+ Tuple of (new_spans, warning_message).
+ """
+ if not self.is_available:
+ return [], self._load_error
+
+ # Simple sentence splitting (could use spaCy if available)
+ sentences = self._split_sentences(content)
+ spans: list[DynamicSpan] = []
+
+ # Get existing ranges
+ existing_ranges = set()
+ if existing_spans:
+ existing_ranges = {(s.start, s.end) for s in existing_spans}
+
+ # Encode all sentences
+ if not sentences:
+ return [], None
+
+ sentence_texts = [s[0] for s in sentences]
+ sentence_embeddings = self._model.encode(
+ sentence_texts,
+ convert_to_numpy=True,
+ )
+
+ # Compute similarities
+ similarities = np.dot(sentence_embeddings, self._exemplar_embeddings.T)
+
+ for i, (text, start, end) in enumerate(sentences):
+ # Get max similarity to any exemplar
+ max_sim = float(np.max(similarities[i]))
+
+ if max_sim < self.config.semantic_threshold:
+ continue
+
+ # Check overlap with existing spans
+ overlaps = any(
+ not (end <= s or start >= e)
+ for s, e in existing_ranges
+ )
+ if overlaps:
+ continue
+
+ # Find which exemplar matched best
+ best_exemplar_idx = int(np.argmax(similarities[i]))
+ best_exemplar = self.DYNAMIC_EXEMPLARS[best_exemplar_idx]
+
+ # Determine category based on exemplar
+ category = self._categorize_exemplar(best_exemplar)
+
+ spans.append(DynamicSpan(
+ text=text,
+ start=start,
+ end=end,
+ category=category,
+ tier="semantic",
+ confidence=max_sim,
+ metadata={
+ "matched_exemplar": best_exemplar,
+ "similarity": max_sim,
+ },
+ ))
+ existing_ranges.add((start, end))
+
+ return sorted(spans, key=lambda s: s.start), None
+
+ def _split_sentences(self, content: str) -> list[tuple[str, int, int]]:
+ """Split content into sentences with positions."""
+ sentences: list[tuple[str, int, int]] = []
+ pattern = r'[^.!?\n]+[.!?\n]?'
+ for match in re.finditer(pattern, content):
+ text = match.group().strip()
+ if len(text) > 10:
+ sentences.append((text, match.start(), match.end()))
+ return sentences
+
+ def _categorize_exemplar(self, exemplar: str) -> DynamicCategory:
+ """Categorize based on which exemplar matched."""
+ exemplar_lower = exemplar.lower()
+
+ if any(w in exemplar_lower for w in ["date", "today", "updated", "refreshed"]):
+ return DynamicCategory.DATE
+ elif any(w in exemplar_lower for w in ["price", "stock", "live", "real-time"]):
+ return DynamicCategory.REALTIME
+ elif any(w in exemplar_lower for w in ["session", "account", "your"]):
+ return DynamicCategory.SESSION
+ elif any(w in exemplar_lower for w in ["status", "load", "queue", "active"]):
+ return DynamicCategory.VOLATILE
+ else:
+ return DynamicCategory.VOLATILE
+
+
+class DynamicContentDetector:
+ """
+ Unified dynamic content detector with tiered detection.
+
+ Key Design Principles:
+ - NO hardcoded locale-specific patterns (no month names)
+ - Structural detection: Labels indicate what's dynamic
+ - Universal patterns: ISO 8601, UUIDs, Unix timestamps
+ - Entropy-based: High entropy = random/generated = dynamic
+
+ Usage:
+ # Fast mode (regex only - structural + universal + entropy)
+ detector = DynamicContentDetector(DetectorConfig(tiers=["regex"]))
+
+ # Balanced mode (regex + NER for names/money)
+ detector = DynamicContentDetector(DetectorConfig(tiers=["regex", "ner"]))
+
+ # Full mode (all tiers)
+ detector = DynamicContentDetector(DetectorConfig(
+ tiers=["regex", "ner", "semantic"]
+ ))
+
+ result = detector.detect("Session: abc123. User: John paid $500.")
+ """
+
+ def __init__(self, config: DetectorConfig | None = None):
+ """Initialize detector with configuration."""
+ self.config = config or DetectorConfig()
+
+ # Initialize detectors based on enabled tiers
+ self._regex_detector: RegexDetector | None = None
+ self._ner_detector: NERDetector | None = None
+ self._semantic_detector: SemanticDetector | None = None
+
+ if "regex" in self.config.tiers:
+ self._regex_detector = RegexDetector(self.config)
+
+ if "ner" in self.config.tiers:
+ self._ner_detector = NERDetector(self.config)
+
+ if "semantic" in self.config.tiers:
+ self._semantic_detector = SemanticDetector(self.config)
+
+ def detect(self, content: str) -> DetectionResult:
+ """
+ Detect dynamic content in text.
+
+ Runs enabled tiers in order, accumulating spans.
+ Each tier can see what previous tiers detected.
+
+ Args:
+ content: Text to analyze.
+
+ Returns:
+ DetectionResult with spans, static/dynamic content split, etc.
+ """
+ import time
+ start_time = time.perf_counter()
+
+ all_spans: list[DynamicSpan] = []
+ tiers_used: list[str] = []
+ warnings: list[str] = []
+
+ # Tier 1: Regex (structural + universal + entropy)
+ if self._regex_detector:
+ regex_spans = self._regex_detector.detect(content)
+ all_spans.extend(regex_spans)
+ tiers_used.append("regex")
+
+ # Tier 2: NER
+ if self._ner_detector:
+ ner_spans, ner_warning = self._ner_detector.detect(content, all_spans)
+ all_spans.extend(ner_spans)
+ if ner_warning:
+ warnings.append(ner_warning)
+ elif ner_spans or self._ner_detector.is_available:
+ tiers_used.append("ner")
+
+ # Tier 3: Semantic
+ if self._semantic_detector:
+ sem_spans, sem_warning = self._semantic_detector.detect(content, all_spans)
+ all_spans.extend(sem_spans)
+ if sem_warning:
+ warnings.append(sem_warning)
+ elif sem_spans or self._semantic_detector.is_available:
+ tiers_used.append("semantic")
+
+ # Sort by position
+ all_spans = sorted(all_spans, key=lambda s: s.start)
+
+ # Build static and dynamic content
+ static_content, dynamic_content = self._split_content(content, all_spans)
+
+ processing_time = (time.perf_counter() - start_time) * 1000
+
+ return DetectionResult(
+ spans=all_spans,
+ static_content=static_content,
+ dynamic_content=dynamic_content,
+ tiers_used=tiers_used,
+ processing_time_ms=processing_time,
+ warnings=warnings,
+ )
+
+ def _split_content(
+ self,
+ content: str,
+ spans: list[DynamicSpan],
+ ) -> tuple[str, str]:
+ """Split content into static and dynamic parts."""
+ if not spans:
+ return content, ""
+
+ static = content
+ dynamic_parts: list[str] = []
+
+ for span in reversed(spans):
+ dynamic_parts.append(span.text)
+ static = static[:span.start] + static[span.end:]
+
+ static = self._clean_static_content(static)
+ dynamic_parts.reverse()
+ dynamic = "\n".join(dynamic_parts)
+
+ return static, dynamic
+
+ def _clean_static_content(self, content: str) -> str:
+ """Clean up static content after span removal."""
+ lines = content.split("\n")
+ cleaned_lines: list[str] = []
+ prev_blank = False
+
+ for line in lines:
+ is_blank = not line.strip()
+ if is_blank and prev_blank:
+ continue
+ cleaned_lines.append(line.rstrip())
+ prev_blank = is_blank
+
+ return "\n".join(cleaned_lines).strip()
+
+ @property
+ def available_tiers(self) -> list[str]:
+ """Get list of actually available tiers (dependencies installed)."""
+ available = []
+
+ if self._regex_detector:
+ available.append("regex")
+
+ if self._ner_detector and self._ner_detector.is_available:
+ available.append("ner")
+
+ if self._semantic_detector and self._semantic_detector.is_available:
+ available.append("semantic")
+
+ return available
+
+
+# Convenience function
+def detect_dynamic_content(
+ content: str,
+ tiers: list[Literal["regex", "ner", "semantic"]] | None = None,
+) -> DetectionResult:
+ """
+ Detect dynamic content in text.
+
+ Convenience function that creates a detector with specified tiers.
+
+ Args:
+ content: Text to analyze.
+ tiers: Which tiers to use. Default: ["regex"] for speed.
+
+ Returns:
+ DetectionResult with detected spans and split content.
+
+ Example:
+ >>> result = detect_dynamic_content(
+ ... "Session: abc123xyz. User: John paid $500.",
+ ... tiers=["regex", "ner"]
+ ... )
+ >>> print(result.static_content)
+ >>> print(result.dynamic_content)
+ """
+ config = DetectorConfig(tiers=tiers or ["regex"])
+ detector = DynamicContentDetector(config)
+ return detector.detect(content)
diff --git a/headroom/cache/google.py b/headroom/cache/google.py
new file mode 100644
index 000000000..84e325c0f
--- /dev/null
+++ b/headroom/cache/google.py
@@ -0,0 +1,896 @@
+"""
+Google Cache Optimizer for CachedContent API.
+
+Google's Gemini API offers explicit cached content management through
+the `genai.caching.CachedContent` API. Key characteristics:
+
+- Minimum 32K tokens required for caching
+- 75% discount on cached input tokens
+- Storage costs (pay per hour for cached content)
+- User-defined TTL (default 1 hour)
+- Returns cache_id for subsequent requests
+
+This optimizer provides cache lifecycle management utilities without
+making actual API calls - users integrate with the google-generativeai
+package themselves.
+
+Usage:
+ optimizer = GoogleCacheOptimizer()
+
+ # Check if content is cacheable
+ analysis = optimizer.analyze_cacheability(messages, context)
+
+ # Optimize and get cache recommendation
+ result = optimizer.optimize(messages, context)
+
+ # After user creates cache via Google API, register it
+ optimizer.register_cache(
+ cache_id="cached-content-xyz",
+ content_hash=result.metrics.stable_prefix_hash,
+ token_count=50000,
+ expires_at=datetime.now() + timedelta(hours=1),
+ )
+
+ # Check if existing cache can be reused
+ cache_info = optimizer.get_reusable_cache(content_hash)
+
+ # Extend cache TTL
+ optimizer.extend_cache_ttl(cache_id, additional_seconds=3600)
+
+ # Clean up expired caches
+ optimizer.cleanup_expired_caches()
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta
+from typing import Any
+
+from .base import (
+ BaseCacheOptimizer,
+ CacheConfig,
+ CacheMetrics,
+ CacheResult,
+ CacheStrategy,
+ OptimizationContext,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# Google-specific constants
+GOOGLE_MIN_CACHE_TOKENS = 32_768 # 32K tokens minimum
+GOOGLE_CACHE_DISCOUNT = 0.75 # 75% discount on cached tokens
+GOOGLE_DEFAULT_TTL_SECONDS = 3600 # 1 hour default
+GOOGLE_MAX_TTL_SECONDS = 86400 * 7 # 7 days maximum
+
+
+@dataclass
+class CachedContentInfo:
+ """
+ Information about a cached content object.
+
+ Tracks the lifecycle of a Google CachedContent resource.
+ """
+
+ # Google's cache identifier
+ cache_id: str
+
+ # Hash of the content for matching
+ content_hash: str
+
+ # Timestamps
+ created_at: datetime
+ expires_at: datetime
+
+ # Token count in the cached content
+ token_count: int
+
+ # Optional model used (some caches are model-specific)
+ model: str | None = None
+
+ # Display name for the cached content
+ display_name: str | None = None
+
+ # Metadata for tracking
+ metadata: dict[str, Any] = field(default_factory=dict)
+
+ @property
+ def is_expired(self) -> bool:
+ """Check if cache has expired."""
+ return datetime.now() >= self.expires_at
+
+ @property
+ def ttl_remaining_seconds(self) -> int:
+ """Seconds remaining until expiry."""
+ remaining = (self.expires_at - datetime.now()).total_seconds()
+ return max(0, int(remaining))
+
+ @property
+ def age_seconds(self) -> int:
+ """Age of the cache in seconds."""
+ return int((datetime.now() - self.created_at).total_seconds())
+
+ def to_dict(self) -> dict[str, Any]:
+ """Serialize to dictionary."""
+ return {
+ "cache_id": self.cache_id,
+ "content_hash": self.content_hash,
+ "created_at": self.created_at.isoformat(),
+ "expires_at": self.expires_at.isoformat(),
+ "token_count": self.token_count,
+ "model": self.model,
+ "display_name": self.display_name,
+ "metadata": self.metadata,
+ }
+
+ @classmethod
+ def from_dict(cls, data: dict[str, Any]) -> CachedContentInfo:
+ """Deserialize from dictionary."""
+ return cls(
+ cache_id=data["cache_id"],
+ content_hash=data["content_hash"],
+ created_at=datetime.fromisoformat(data["created_at"]),
+ expires_at=datetime.fromisoformat(data["expires_at"]),
+ token_count=data["token_count"],
+ model=data.get("model"),
+ display_name=data.get("display_name"),
+ metadata=data.get("metadata", {}),
+ )
+
+
+@dataclass
+class CacheabilityAnalysis:
+ """
+ Analysis of whether content is suitable for Google caching.
+
+ Provides detailed information about caching viability and
+ potential savings.
+ """
+
+ # Whether content meets minimum threshold
+ is_cacheable: bool
+
+ # Token counts
+ total_tokens: int
+ cacheable_tokens: int
+
+ # Shortfall if not cacheable
+ tokens_below_minimum: int = 0
+
+ # Estimated savings
+ estimated_hourly_storage_cost_usd: float = 0.0
+ estimated_savings_per_request_percent: float = 0.0
+
+ # Recommendations
+ recommendations: list[str] = field(default_factory=list)
+
+ # Content hash for cache matching
+ content_hash: str = ""
+
+
+class GoogleCacheOptimizer(BaseCacheOptimizer):
+ """
+ Cache optimizer for Google's Gemini CachedContent API.
+
+ This optimizer provides:
+ 1. Analysis of whether content meets Google's caching requirements
+ 2. Cache lifecycle management (register, lookup, extend, delete)
+ 3. Optimization recommendations
+ 4. Integration utilities for the google-generativeai SDK
+
+ The optimizer does NOT make actual API calls - it provides the
+ infrastructure for users to manage caches themselves.
+
+ Example workflow:
+ optimizer = GoogleCacheOptimizer()
+
+ # Analyze content
+ result = optimizer.optimize(messages, context)
+
+ if result.metrics.cacheable_tokens >= GOOGLE_MIN_CACHE_TOKENS:
+ # User creates cache via Google SDK
+ cached_content = genai.caching.CachedContent.create(
+ model="gemini-1.5-pro",
+ contents=contents,
+ ttl=timedelta(hours=1),
+ )
+
+ # Register with optimizer for tracking
+ optimizer.register_cache(
+ cache_id=cached_content.name,
+ content_hash=result.metrics.stable_prefix_hash,
+ token_count=result.metrics.cacheable_tokens,
+ expires_at=datetime.now() + timedelta(hours=1),
+ )
+
+ # Later, check for reusable cache
+ cache = optimizer.get_reusable_cache(content_hash)
+ if cache:
+ # Use cache.cache_id in API call
+ pass
+ """
+
+ def __init__(self, config: CacheConfig | None = None):
+ """
+ Initialize Google cache optimizer.
+
+ Args:
+ config: Optional cache configuration
+ """
+ super().__init__(config)
+
+ # Override minimum tokens for Google's requirements
+ if self.config.min_cacheable_tokens < GOOGLE_MIN_CACHE_TOKENS:
+ self.config.min_cacheable_tokens = GOOGLE_MIN_CACHE_TOKENS
+
+ # Cache registry: content_hash -> CachedContentInfo
+ self._cache_registry: dict[str, CachedContentInfo] = {}
+
+ # Also index by cache_id for direct lookup
+ self._cache_by_id: dict[str, CachedContentInfo] = {}
+
+ # Statistics
+ self._caches_created: int = 0
+ self._caches_reused: int = 0
+ self._caches_expired: int = 0
+
+ @property
+ def name(self) -> str:
+ """Name of this optimizer."""
+ return "google-cached-content"
+
+ @property
+ def provider(self) -> str:
+ """Provider this optimizer is for."""
+ return "google"
+
+ @property
+ def strategy(self) -> CacheStrategy:
+ """The caching strategy this optimizer uses."""
+ return CacheStrategy.CACHED_CONTENT
+
+ def optimize(
+ self,
+ messages: list[dict[str, Any]],
+ context: OptimizationContext,
+ config: CacheConfig | None = None,
+ ) -> CacheResult:
+ """
+ Optimize messages for Google caching.
+
+ This method:
+ 1. Analyzes content for cacheability
+ 2. Checks for existing reusable caches
+ 3. Returns optimization metrics and recommendations
+
+ Args:
+ messages: The messages to optimize
+ context: Optimization context
+ config: Optional configuration override
+
+ Returns:
+ CacheResult with analysis and cache information
+ """
+ effective_config = config or self.config
+
+ # Extract cacheable content (system messages + static context)
+ cacheable_content = self._extract_cacheable_content(messages)
+ content_hash = self._compute_prefix_hash(cacheable_content)
+
+ # Estimate tokens
+ total_tokens = self._count_tokens_estimate(
+ self._messages_to_text(messages)
+ )
+ cacheable_tokens = self._count_tokens_estimate(cacheable_content)
+
+ # Check for existing cache
+ existing_cache = self.get_reusable_cache(content_hash)
+
+ # Build metrics
+ metrics = CacheMetrics(
+ stable_prefix_tokens=cacheable_tokens,
+ stable_prefix_hash=content_hash,
+ prefix_changed_from_previous=(
+ context.previous_prefix_hash != content_hash
+ if context.previous_prefix_hash
+ else False
+ ),
+ previous_prefix_hash=context.previous_prefix_hash,
+ cacheable_tokens=cacheable_tokens,
+ non_cacheable_tokens=total_tokens - cacheable_tokens,
+ )
+
+ # Calculate estimated savings
+ if cacheable_tokens >= GOOGLE_MIN_CACHE_TOKENS:
+ metrics.estimated_savings_percent = GOOGLE_CACHE_DISCOUNT * 100
+ metrics.estimated_cache_hit = existing_cache is not None
+
+ # Add cache info if available
+ if existing_cache:
+ metrics.provider_cache_id = existing_cache.cache_id
+ metrics.cache_ttl_remaining_seconds = existing_cache.ttl_remaining_seconds
+ self._caches_reused += 1
+
+ # Build warnings
+ warnings: list[str] = []
+ if cacheable_tokens < GOOGLE_MIN_CACHE_TOKENS:
+ shortfall = GOOGLE_MIN_CACHE_TOKENS - cacheable_tokens
+ warnings.append(
+ f"Content has {cacheable_tokens:,} tokens, needs {shortfall:,} more "
+ f"to meet Google's 32K minimum for caching"
+ )
+
+ if existing_cache and existing_cache.ttl_remaining_seconds < 300:
+ warnings.append(
+ f"Existing cache expires in {existing_cache.ttl_remaining_seconds}s - "
+ f"consider extending TTL"
+ )
+
+ # Record metrics
+ self._record_metrics(metrics)
+ self._previous_prefix_hash = content_hash
+
+ # Build transforms applied list
+ transforms: list[str] = ["content_analysis"]
+ if existing_cache:
+ transforms.append("cache_lookup")
+
+ return CacheResult(
+ messages=messages, # Messages unchanged - caching is separate
+ semantic_cache_hit=False,
+ metrics=metrics,
+ tokens_before=total_tokens,
+ tokens_after=total_tokens, # Token count doesn't change
+ transforms_applied=transforms,
+ warnings=warnings,
+ )
+
+ def analyze_cacheability(
+ self,
+ messages: list[dict[str, Any]],
+ context: OptimizationContext,
+ ) -> CacheabilityAnalysis:
+ """
+ Analyze content for Google cache suitability.
+
+ Provides detailed analysis including:
+ - Whether content meets minimum requirements
+ - Estimated costs and savings
+ - Recommendations for improving cacheability
+
+ Args:
+ messages: Messages to analyze
+ context: Optimization context
+
+ Returns:
+ CacheabilityAnalysis with detailed information
+ """
+ cacheable_content = self._extract_cacheable_content(messages)
+ content_hash = self._compute_prefix_hash(cacheable_content)
+
+ total_tokens = self._count_tokens_estimate(
+ self._messages_to_text(messages)
+ )
+ cacheable_tokens = self._count_tokens_estimate(cacheable_content)
+
+ is_cacheable = cacheable_tokens >= GOOGLE_MIN_CACHE_TOKENS
+ tokens_below_minimum = max(0, GOOGLE_MIN_CACHE_TOKENS - cacheable_tokens)
+
+ # Build recommendations
+ recommendations: list[str] = []
+
+ if not is_cacheable:
+ recommendations.append(
+ f"Add {tokens_below_minimum:,} more tokens to static content "
+ f"to enable caching"
+ )
+ recommendations.append(
+ "Consider adding detailed examples or documentation to system prompt"
+ )
+ else:
+ recommendations.append(
+ f"Content is cacheable. Create cache with google-generativeai SDK"
+ )
+
+ # Storage cost estimation (rough - actual pricing varies)
+ # Assuming ~$0.001 per 1000 tokens per hour (simplified)
+ hourly_cost = (cacheable_tokens / 1000) * 0.001
+ recommendations.append(
+ f"Estimated storage cost: ~${hourly_cost:.4f}/hour"
+ )
+
+ # Break-even analysis
+ if hourly_cost > 0:
+ # Assuming $0.01 per 1000 input tokens base price
+ base_cost_per_request = (cacheable_tokens / 1000) * 0.01
+ savings_per_request = base_cost_per_request * GOOGLE_CACHE_DISCOUNT
+ break_even_requests = hourly_cost / savings_per_request if savings_per_request > 0 else float('inf')
+ recommendations.append(
+ f"Break-even: ~{int(break_even_requests)} requests/hour"
+ )
+
+ return CacheabilityAnalysis(
+ is_cacheable=is_cacheable,
+ total_tokens=total_tokens,
+ cacheable_tokens=cacheable_tokens,
+ tokens_below_minimum=tokens_below_minimum,
+ estimated_savings_per_request_percent=(
+ GOOGLE_CACHE_DISCOUNT * 100 if is_cacheable else 0.0
+ ),
+ recommendations=recommendations,
+ content_hash=content_hash,
+ )
+
+ # -------------------------------------------------------------------------
+ # Cache Registry Management
+ # -------------------------------------------------------------------------
+
+ def register_cache(
+ self,
+ cache_id: str,
+ content_hash: str,
+ token_count: int,
+ expires_at: datetime,
+ *,
+ model: str | None = None,
+ display_name: str | None = None,
+ metadata: dict[str, Any] | None = None,
+ ) -> CachedContentInfo:
+ """
+ Register a cache after creating it via Google's API.
+
+ Call this after successfully creating a CachedContent resource
+ to enable cache reuse detection.
+
+ Args:
+ cache_id: Google's cache identifier (e.g., "cachedContents/xyz")
+ content_hash: Hash of cached content (from optimize() metrics)
+ token_count: Number of tokens in cached content
+ expires_at: When the cache expires
+ model: Optional model the cache was created for
+ display_name: Optional display name
+ metadata: Optional additional metadata
+
+ Returns:
+ CachedContentInfo for the registered cache
+
+ Example:
+ # After creating cache via Google SDK
+ cached_content = genai.caching.CachedContent.create(...)
+
+ info = optimizer.register_cache(
+ cache_id=cached_content.name,
+ content_hash=result.metrics.stable_prefix_hash,
+ token_count=result.metrics.cacheable_tokens,
+ expires_at=datetime.now() + timedelta(hours=1),
+ )
+ """
+ # Remove any existing cache with same content hash
+ old_cache = self._cache_registry.get(content_hash)
+ if old_cache:
+ self._cache_by_id.pop(old_cache.cache_id, None)
+ logger.debug(
+ f"Replacing existing cache for hash {content_hash}: "
+ f"{old_cache.cache_id} -> {cache_id}"
+ )
+
+ cache_info = CachedContentInfo(
+ cache_id=cache_id,
+ content_hash=content_hash,
+ created_at=datetime.now(),
+ expires_at=expires_at,
+ token_count=token_count,
+ model=model,
+ display_name=display_name,
+ metadata=metadata or {},
+ )
+
+ self._cache_registry[content_hash] = cache_info
+ self._cache_by_id[cache_id] = cache_info
+ self._caches_created += 1
+
+ logger.info(
+ f"Registered cache {cache_id} with {token_count:,} tokens, "
+ f"expires in {cache_info.ttl_remaining_seconds}s"
+ )
+
+ return cache_info
+
+ def get_reusable_cache(
+ self,
+ content_hash: str,
+ *,
+ min_ttl_seconds: int = 60,
+ ) -> CachedContentInfo | None:
+ """
+ Check if a reusable cache exists for the given content.
+
+ Args:
+ content_hash: Hash of the content to look up
+ min_ttl_seconds: Minimum remaining TTL to consider reusable
+
+ Returns:
+ CachedContentInfo if reusable cache exists, None otherwise
+ """
+ cache_info = self._cache_registry.get(content_hash)
+
+ if cache_info is None:
+ return None
+
+ if cache_info.is_expired:
+ self._remove_cache_internal(content_hash)
+ return None
+
+ if cache_info.ttl_remaining_seconds < min_ttl_seconds:
+ logger.debug(
+ f"Cache {cache_info.cache_id} has insufficient TTL "
+ f"({cache_info.ttl_remaining_seconds}s < {min_ttl_seconds}s)"
+ )
+ return None
+
+ return cache_info
+
+ def get_cache_by_id(self, cache_id: str) -> CachedContentInfo | None:
+ """
+ Look up cache information by cache ID.
+
+ Args:
+ cache_id: Google's cache identifier
+
+ Returns:
+ CachedContentInfo if found, None otherwise
+ """
+ return self._cache_by_id.get(cache_id)
+
+ def extend_cache_ttl(
+ self,
+ cache_id: str,
+ new_expires_at: datetime,
+ ) -> CachedContentInfo | None:
+ """
+ Update the expiry time for a cache after extending via Google API.
+
+ Call this after successfully calling update() on the CachedContent
+ to extend its TTL.
+
+ Args:
+ cache_id: Google's cache identifier
+ new_expires_at: New expiry time
+
+ Returns:
+ Updated CachedContentInfo or None if not found
+
+ Example:
+ # After extending via Google SDK
+ cached_content.update(ttl=timedelta(hours=2))
+
+ optimizer.extend_cache_ttl(
+ cache_id=cached_content.name,
+ new_expires_at=datetime.now() + timedelta(hours=2),
+ )
+ """
+ cache_info = self._cache_by_id.get(cache_id)
+ if cache_info is None:
+ logger.warning(f"Cannot extend unknown cache: {cache_id}")
+ return None
+
+ old_expires = cache_info.expires_at
+ cache_info.expires_at = new_expires_at
+
+ logger.info(
+ f"Extended cache {cache_id} TTL from {old_expires} to {new_expires_at}"
+ )
+
+ return cache_info
+
+ def remove_cache(self, cache_id: str) -> bool:
+ """
+ Remove a cache from the registry.
+
+ Call this after deleting the cache via Google API.
+
+ Args:
+ cache_id: Google's cache identifier
+
+ Returns:
+ True if cache was removed, False if not found
+ """
+ cache_info = self._cache_by_id.get(cache_id)
+ if cache_info is None:
+ return False
+
+ self._cache_by_id.pop(cache_id, None)
+ self._cache_registry.pop(cache_info.content_hash, None)
+
+ logger.info(f"Removed cache {cache_id} from registry")
+ return True
+
+ def _remove_cache_internal(self, content_hash: str) -> None:
+ """Remove cache by content hash (internal use)."""
+ cache_info = self._cache_registry.pop(content_hash, None)
+ if cache_info:
+ self._cache_by_id.pop(cache_info.cache_id, None)
+ self._caches_expired += 1
+
+ def cleanup_expired_caches(self) -> list[str]:
+ """
+ Remove all expired caches from the registry.
+
+ Returns:
+ List of removed cache IDs (for user to delete via Google API)
+
+ Example:
+ expired_ids = optimizer.cleanup_expired_caches()
+ for cache_id in expired_ids:
+ # User deletes via Google SDK
+ genai.caching.CachedContent.get(cache_id).delete()
+ """
+ expired_ids: list[str] = []
+
+ # Find expired caches
+ for content_hash, cache_info in list(self._cache_registry.items()):
+ if cache_info.is_expired:
+ expired_ids.append(cache_info.cache_id)
+ self._remove_cache_internal(content_hash)
+
+ if expired_ids:
+ logger.info(f"Cleaned up {len(expired_ids)} expired caches")
+
+ return expired_ids
+
+ def list_caches(
+ self,
+ *,
+ include_expired: bool = False,
+ ) -> list[CachedContentInfo]:
+ """
+ List all registered caches.
+
+ Args:
+ include_expired: Whether to include expired caches
+
+ Returns:
+ List of CachedContentInfo objects
+ """
+ caches = list(self._cache_registry.values())
+
+ if not include_expired:
+ caches = [c for c in caches if not c.is_expired]
+
+ # Sort by expiry time
+ caches.sort(key=lambda c: c.expires_at)
+
+ return caches
+
+ def get_statistics(self) -> dict[str, Any]:
+ """
+ Get cache usage statistics.
+
+ Returns:
+ Dictionary with cache statistics
+ """
+ active_caches = [c for c in self._cache_registry.values() if not c.is_expired]
+ total_cached_tokens = sum(c.token_count for c in active_caches)
+
+ return {
+ "active_caches": len(active_caches),
+ "total_cached_tokens": total_cached_tokens,
+ "caches_created": self._caches_created,
+ "caches_reused": self._caches_reused,
+ "caches_expired": self._caches_expired,
+ "cache_hit_rate": (
+ self._caches_reused / (self._caches_reused + self._caches_created)
+ if (self._caches_reused + self._caches_created) > 0
+ else 0.0
+ ),
+ }
+
+ # -------------------------------------------------------------------------
+ # Cache Creation Helpers
+ # -------------------------------------------------------------------------
+
+ def prepare_cache_creation(
+ self,
+ messages: list[dict[str, Any]],
+ context: OptimizationContext,
+ ttl_seconds: int = GOOGLE_DEFAULT_TTL_SECONDS,
+ ) -> dict[str, Any] | None:
+ """
+ Prepare parameters for creating a Google cache.
+
+ Returns a dictionary with suggested parameters for
+ genai.caching.CachedContent.create().
+
+ Args:
+ messages: Messages to cache
+ context: Optimization context
+ ttl_seconds: Desired TTL in seconds
+
+ Returns:
+ Dictionary with cache creation parameters, or None if not cacheable
+
+ Example:
+ params = optimizer.prepare_cache_creation(messages, context)
+ if params:
+ cached_content = genai.caching.CachedContent.create(**params)
+ """
+ analysis = self.analyze_cacheability(messages, context)
+
+ if not analysis.is_cacheable:
+ logger.debug(
+ f"Content not cacheable: {analysis.tokens_below_minimum} "
+ f"tokens below minimum"
+ )
+ return None
+
+ cacheable_content = self._extract_cacheable_content(messages)
+
+ return {
+ "contents": cacheable_content,
+ "ttl": timedelta(seconds=min(ttl_seconds, GOOGLE_MAX_TTL_SECONDS)),
+ "display_name": f"headroom-cache-{analysis.content_hash[:8]}",
+ "_headroom_metadata": {
+ "content_hash": analysis.content_hash,
+ "token_count": analysis.cacheable_tokens,
+ "created_by": "headroom",
+ },
+ }
+
+ def build_request_with_cache(
+ self,
+ messages: list[dict[str, Any]],
+ cache_id: str,
+ ) -> dict[str, Any]:
+ """
+ Build request parameters using an existing cache.
+
+ Returns a dictionary suggesting how to structure the API call
+ when using cached content.
+
+ Args:
+ messages: Full message list
+ cache_id: Cache ID to use
+
+ Returns:
+ Dictionary with suggested request structure
+ """
+ # Extract only the non-cached (dynamic) content
+ dynamic_messages = self._extract_dynamic_messages(messages)
+
+ return {
+ "cached_content": cache_id,
+ "contents": dynamic_messages,
+ "_headroom_note": (
+ "Use cached_content parameter with GenerativeModel "
+ "to leverage the cache"
+ ),
+ }
+
+ # -------------------------------------------------------------------------
+ # Content Extraction Helpers
+ # -------------------------------------------------------------------------
+
+ def _extract_cacheable_content(self, messages: list[dict[str, Any]]) -> str:
+ """
+ Extract content suitable for caching.
+
+ Includes:
+ - System messages
+ - Static context (tools, examples)
+
+ Excludes:
+ - Recent conversation turns
+ - Dynamic content (dates, user-specific data)
+ """
+ cacheable_parts: list[str] = []
+
+ for msg in messages:
+ role = msg.get("role", "")
+
+ # System messages are always cacheable
+ if role == "system":
+ content = self._extract_message_content(msg)
+ if content:
+ cacheable_parts.append(content)
+
+ # First few user/assistant turns with examples might be cacheable
+ # but we're conservative - only include system by default
+
+ return "\n\n".join(cacheable_parts)
+
+ def _extract_dynamic_messages(
+ self,
+ messages: list[dict[str, Any]],
+ ) -> list[dict[str, Any]]:
+ """
+ Extract messages that should NOT be cached.
+
+ These are the conversation turns after the cached prefix.
+ """
+ dynamic: list[dict[str, Any]] = []
+
+ for msg in messages:
+ if msg.get("role") != "system":
+ dynamic.append(msg)
+
+ return dynamic
+
+ def _extract_message_content(self, message: dict[str, Any]) -> str:
+ """Extract text content from a message."""
+ content = message.get("content", "")
+
+ if isinstance(content, str):
+ return content
+
+ if isinstance(content, list):
+ parts = []
+ for block in content:
+ if isinstance(block, dict):
+ if block.get("type") == "text":
+ parts.append(block.get("text", ""))
+ elif isinstance(block, str):
+ parts.append(block)
+ return "\n".join(parts)
+
+ return ""
+
+ def _messages_to_text(self, messages: list[dict[str, Any]]) -> str:
+ """Convert all messages to text for token counting."""
+ parts = []
+ for msg in messages:
+ content = self._extract_message_content(msg)
+ if content:
+ parts.append(f"{msg.get('role', 'unknown')}: {content}")
+ return "\n\n".join(parts)
+
+ # -------------------------------------------------------------------------
+ # Serialization for Persistence
+ # -------------------------------------------------------------------------
+
+ def export_cache_registry(self) -> list[dict[str, Any]]:
+ """
+ Export cache registry for persistence.
+
+ Returns:
+ List of cache info dictionaries
+ """
+ return [info.to_dict() for info in self._cache_registry.values()]
+
+ def import_cache_registry(
+ self,
+ cache_data: list[dict[str, Any]],
+ *,
+ skip_expired: bool = True,
+ ) -> int:
+ """
+ Import caches from persisted data.
+
+ Args:
+ cache_data: List of cache info dictionaries
+ skip_expired: Whether to skip already-expired caches
+
+ Returns:
+ Number of caches imported
+ """
+ imported = 0
+
+ for data in cache_data:
+ try:
+ cache_info = CachedContentInfo.from_dict(data)
+
+ if skip_expired and cache_info.is_expired:
+ continue
+
+ self._cache_registry[cache_info.content_hash] = cache_info
+ self._cache_by_id[cache_info.cache_id] = cache_info
+ imported += 1
+
+ except (KeyError, ValueError) as e:
+ logger.warning(f"Failed to import cache entry: {e}")
+ continue
+
+ logger.info(f"Imported {imported} caches from persisted data")
+ return imported
diff --git a/headroom/cache/openai.py b/headroom/cache/openai.py
new file mode 100644
index 000000000..6f23806b2
--- /dev/null
+++ b/headroom/cache/openai.py
@@ -0,0 +1,591 @@
+"""
+OpenAI Cache Optimizer.
+
+This module implements cache optimization for OpenAI's automatic prefix caching.
+Unlike Anthropic, OpenAI's caching is fully automatic - users cannot control what
+gets cached. The only optimization strategy is to stabilize prefixes to maximize
+cache hit rates.
+
+OpenAI Caching Details:
+ - Fully automatic - no explicit cache control available
+ - 50% discount on cached input tokens
+ - Requires prompts > 1024 tokens to activate
+ - 5-60 minute TTL (varies based on usage patterns)
+ - Cache is prefix-based - changes invalidate downstream cache
+
+Optimization Strategy:
+ Since we can't control caching explicitly, we focus on PREFIX_STABILIZATION:
+ - Extract dynamic content (dates, timestamps) and move to end
+ - Normalize whitespace for consistent hashing
+ - Remove random IDs from system prompts
+ - Track prefix stability to estimate cache hit probability
+
+Dynamic Content Detection Tiers:
+ - Tier 1 (regex): Always on, ~0ms - dates, UUIDs, timestamps
+ - Tier 2 (ner): Optional, ~5-10ms - names, money, organizations
+ - Tier 3 (semantic): Optional, ~20-50ms - volatile patterns via embeddings
+
+Usage:
+ # Default: regex only (fastest)
+ optimizer = OpenAICacheOptimizer()
+
+ # With NER (requires spacy)
+ optimizer = OpenAICacheOptimizer(
+ config=CacheConfig(dynamic_detection_tiers=["regex", "ner"])
+ )
+
+ # Full detection (requires spacy + sentence-transformers)
+ optimizer = OpenAICacheOptimizer(
+ config=CacheConfig(dynamic_detection_tiers=["regex", "ner", "semantic"])
+ )
+"""
+
+from __future__ import annotations
+
+import re
+from copy import deepcopy
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+from .base import (
+ BaseCacheOptimizer,
+ CacheConfig,
+ CacheMetrics,
+ CacheResult,
+ CacheStrategy,
+ OptimizationContext,
+)
+from .dynamic_detector import (
+ DetectorConfig,
+ DynamicContentDetector,
+ DynamicSpan,
+)
+
+
+@dataclass
+class PrefixAnalysis:
+ """
+ Analysis of prefix stability.
+
+ Used to determine likelihood of cache hits and track changes
+ between requests.
+ """
+
+ # Hash of the stabilized prefix
+ prefix_hash: str
+
+ # Estimated token count of stable prefix
+ stable_tokens: int
+
+ # Dynamic content that was extracted
+ dynamic_spans: list[DynamicSpan] = field(default_factory=list)
+
+ # Whether prefix changed from previous request
+ changed_from_previous: bool = False
+
+ # Previous hash for comparison
+ previous_hash: str | None = None
+
+ # Detection processing time
+ detection_time_ms: float = 0.0
+
+
+class OpenAICacheOptimizer(BaseCacheOptimizer):
+ """
+ Cache optimizer for OpenAI's automatic prefix caching.
+
+ OpenAI automatically caches prompt prefixes for requests > 1024 tokens.
+ Since caching is automatic, this optimizer focuses on maximizing cache
+ hit rates by stabilizing prefixes.
+
+ Key Optimizations:
+ 1. Extract dynamic content (dates, times) and move to end of messages
+ 2. Normalize whitespace for consistent formatting
+ 3. Remove random IDs and timestamps from system prompts
+ 4. Track prefix changes to estimate cache hit probability
+
+ Usage:
+ optimizer = OpenAICacheOptimizer()
+ result = optimizer.optimize(messages, context)
+
+ # Check if prefix was stable (likely cache hit)
+ if not result.metrics.prefix_changed_from_previous:
+ print("Likely cache hit - prefix unchanged")
+
+ # Estimate savings
+ savings = result.metrics.estimated_savings_percent
+ print(f"Estimated savings: {savings:.1f}%")
+
+ Attributes:
+ name: Identifier for this optimizer
+ provider: The provider this optimizer targets ("openai")
+ strategy: Always CacheStrategy.PREFIX_STABILIZATION
+ """
+
+ # OpenAI-specific constants
+ MIN_TOKENS_FOR_CACHING = 1024
+ CACHE_DISCOUNT_PERCENT = 50.0
+
+ def __init__(self, config: CacheConfig | None = None):
+ """
+ Initialize the OpenAI cache optimizer.
+
+ Args:
+ config: Optional cache configuration. If not provided,
+ sensible defaults are used.
+
+ The optimizer uses the DynamicContentDetector with configurable tiers:
+ - "regex": Fast pattern matching (~0ms) - always on
+ - "ner": Named Entity Recognition (~5-10ms) - requires spacy
+ - "semantic": Embedding similarity (~20-50ms) - requires sentence-transformers
+
+ Configure tiers via config.dynamic_detection_tiers.
+ """
+ super().__init__(config)
+
+ # Initialize the tiered dynamic content detector
+ detector_config = DetectorConfig(
+ tiers=self.config.dynamic_detection_tiers, # type: ignore
+ )
+ self._detector = DynamicContentDetector(detector_config)
+
+ @property
+ def name(self) -> str:
+ """Name of this optimizer."""
+ return "openai-prefix-stabilizer"
+
+ @property
+ def provider(self) -> str:
+ """Provider this optimizer is for."""
+ return "openai"
+
+ @property
+ def strategy(self) -> CacheStrategy:
+ """The caching strategy this optimizer uses."""
+ return CacheStrategy.PREFIX_STABILIZATION
+
+ def optimize(
+ self,
+ messages: list[dict[str, Any]],
+ context: OptimizationContext,
+ config: CacheConfig | None = None,
+ ) -> CacheResult:
+ """
+ Optimize messages for OpenAI's prefix caching.
+
+ This method stabilizes the message prefix to maximize cache hit rates.
+ Since OpenAI caching is automatic, we focus on ensuring the prefix
+ remains consistent across requests.
+
+ Args:
+ messages: List of message dictionaries in OpenAI format.
+ context: Optimization context with request metadata.
+ config: Optional configuration override.
+
+ Returns:
+ CacheResult containing:
+ - Optimized messages with stabilized prefixes
+ - Metrics about prefix stability and estimated savings
+ - List of transforms applied
+ - Any warnings encountered
+
+ Example:
+ >>> optimizer = OpenAICacheOptimizer()
+ >>> messages = [
+ ... {"role": "system", "content": "Today is Jan 1, 2024. You are helpful."},
+ ... {"role": "user", "content": "Hello!"}
+ ... ]
+ >>> context = OptimizationContext(provider="openai", model="gpt-4")
+ >>> result = optimizer.optimize(messages, context)
+ >>> # Date moved to end, prefix stabilized
+ """
+ effective_config = config or self.config
+
+ # Handle disabled optimization
+ if not effective_config.enabled:
+ return CacheResult(
+ messages=messages,
+ metrics=CacheMetrics(),
+ transforms_applied=[],
+ )
+
+ # Deep copy to avoid mutating input
+ optimized_messages = deepcopy(messages)
+ transforms_applied: list[str] = []
+ warnings: list[str] = []
+
+ # Track all extracted spans across messages
+ all_spans: list[DynamicSpan] = []
+ total_detection_time = 0.0
+
+ # Process system messages for prefix stabilization
+ for i, msg in enumerate(optimized_messages):
+ if msg.get("role") == "system":
+ content = msg.get("content", "")
+
+ if isinstance(content, str):
+ # Use tiered dynamic content detector
+ result = self._detector.detect(content)
+ all_spans.extend(result.spans)
+ total_detection_time += result.processing_time_ms
+
+ # Add any detector warnings
+ warnings.extend(result.warnings)
+
+ if result.spans:
+ transforms_applied.append(
+ f"extracted_{len(result.spans)}_dynamic_elements"
+ )
+ transforms_applied.extend(
+ f"tier_{tier}" for tier in result.tiers_used
+ )
+
+ # Get static content with dynamic parts removed
+ stabilized = result.static_content
+
+ # Normalize whitespace
+ if effective_config.normalize_whitespace:
+ stabilized = self._normalize_whitespace(
+ stabilized,
+ collapse_blank_lines=effective_config.collapse_blank_lines,
+ )
+ transforms_applied.append("normalized_whitespace")
+
+ # If we extracted dynamic content, append it at the end
+ if result.dynamic_content:
+ dynamic_section = self._format_dynamic_section(
+ result.dynamic_content,
+ separator=effective_config.dynamic_separator,
+ )
+ stabilized = stabilized.rstrip() + dynamic_section
+
+ optimized_messages[i]["content"] = stabilized
+
+ elif isinstance(content, list):
+ # Handle content blocks (less common for OpenAI)
+ new_content = []
+ for block in content:
+ if isinstance(block, dict) and block.get("type") == "text":
+ text = block.get("text", "")
+ result = self._detector.detect(text)
+ all_spans.extend(result.spans)
+ total_detection_time += result.processing_time_ms
+ warnings.extend(result.warnings)
+
+ stabilized = result.static_content
+
+ if effective_config.normalize_whitespace:
+ stabilized = self._normalize_whitespace(stabilized)
+
+ if result.dynamic_content:
+ dynamic_section = self._format_dynamic_section(
+ result.dynamic_content,
+ separator=effective_config.dynamic_separator,
+ )
+ stabilized = stabilized.rstrip() + dynamic_section
+
+ new_content.append({**block, "text": stabilized})
+ else:
+ new_content.append(block)
+
+ optimized_messages[i]["content"] = new_content
+ if all_spans:
+ transforms_applied.append("processed_content_blocks")
+
+ # Analyze prefix stability
+ analysis = self._analyze_prefix(optimized_messages, context)
+
+ # Calculate token estimates
+ tokens_before = self._estimate_total_tokens(messages)
+ tokens_after = self._estimate_total_tokens(optimized_messages)
+
+ # Build metrics
+ metrics = CacheMetrics(
+ stable_prefix_tokens=analysis.stable_tokens,
+ stable_prefix_hash=analysis.prefix_hash,
+ prefix_changed_from_previous=analysis.changed_from_previous,
+ previous_prefix_hash=analysis.previous_hash,
+ estimated_cache_hit=not analysis.changed_from_previous,
+ cacheable_tokens=self._calculate_cacheable_tokens(analysis.stable_tokens),
+ non_cacheable_tokens=max(0, tokens_after - analysis.stable_tokens),
+ estimated_savings_percent=self._calculate_savings_percent(
+ analysis.stable_tokens,
+ tokens_after,
+ likely_cache_hit=not analysis.changed_from_previous,
+ ),
+ )
+
+ # Add warnings for suboptimal cases
+ if tokens_after < self.MIN_TOKENS_FOR_CACHING:
+ warnings.append(
+ f"Prompt has ~{tokens_after} tokens, below OpenAI's {self.MIN_TOKENS_FOR_CACHING} "
+ f"token minimum for caching. Consider adding more static context."
+ )
+
+ if analysis.changed_from_previous:
+ warnings.append(
+ "Prefix changed from previous request - cache miss likely. "
+ "Consider reviewing what content is changing between requests."
+ )
+
+ # Record metrics and update state
+ self._record_metrics(metrics)
+ self._previous_prefix_hash = analysis.prefix_hash
+
+ return CacheResult(
+ messages=optimized_messages,
+ metrics=metrics,
+ tokens_before=tokens_before,
+ tokens_after=tokens_after,
+ transforms_applied=list(set(transforms_applied)), # Dedupe
+ warnings=warnings,
+ )
+
+ def estimate_savings(
+ self,
+ messages: list[dict[str, Any]],
+ context: OptimizationContext,
+ ) -> float:
+ """
+ Estimate potential cost savings from caching.
+
+ OpenAI provides 50% discount on cached tokens. This method estimates
+ what portion of tokens are likely to be cached based on prefix
+ stability and token count.
+
+ Args:
+ messages: Messages to analyze.
+ context: Optimization context.
+
+ Returns:
+ Estimated savings as a percentage (0-100).
+ Returns 0 if prompt is below caching threshold.
+
+ Example:
+ >>> savings = optimizer.estimate_savings(messages, context)
+ >>> print(f"Potential savings: {savings:.1f}%")
+ """
+ total_tokens = self._estimate_total_tokens(messages)
+
+ # No savings if below threshold
+ if total_tokens < self.MIN_TOKENS_FOR_CACHING:
+ return 0.0
+
+ # Extract system content for prefix analysis
+ system_content = self._extract_system_content(messages)
+ system_tokens = self._count_tokens_estimate(system_content)
+
+ # Estimate cacheable portion (system + early messages)
+ # OpenAI caches the longest matching prefix
+ cacheable_ratio = min(1.0, system_tokens / total_tokens)
+
+ # Check if prefix is stable
+ current_hash = self._compute_prefix_hash(system_content)
+ likely_hit = (
+ self._previous_prefix_hash is not None
+ and current_hash == self._previous_prefix_hash
+ )
+
+ if likely_hit:
+ # 50% savings on cacheable portion
+ return cacheable_ratio * self.CACHE_DISCOUNT_PERCENT
+ else:
+ # First request or prefix changed - no immediate savings
+ # but return expected savings for future requests
+ return cacheable_ratio * self.CACHE_DISCOUNT_PERCENT * 0.5
+
+ def _normalize_whitespace(
+ self,
+ content: str,
+ collapse_blank_lines: bool = True,
+ ) -> str:
+ """
+ Normalize whitespace in content.
+
+ Ensures consistent whitespace formatting to improve prefix matching.
+ This helps when the same logical content has minor formatting differences.
+
+ Args:
+ content: Text to normalize.
+ collapse_blank_lines: If True, multiple blank lines become one.
+
+ Returns:
+ Content with normalized whitespace.
+ """
+ # Normalize line endings
+ result = content.replace("\r\n", "\n").replace("\r", "\n")
+
+ # Collapse multiple spaces (but preserve indentation)
+ lines = result.split("\n")
+ normalized_lines = []
+
+ for line in lines:
+ # Preserve leading whitespace, normalize trailing
+ stripped = line.rstrip()
+ if stripped:
+ # Find leading whitespace
+ leading = len(line) - len(line.lstrip())
+ # Collapse multiple spaces in content (not indentation)
+ content_part = " ".join(stripped.split())
+ normalized_lines.append(" " * leading + content_part[leading:] if leading else content_part)
+ else:
+ normalized_lines.append("")
+
+ result = "\n".join(normalized_lines)
+
+ # Collapse multiple blank lines
+ if collapse_blank_lines:
+ while "\n\n\n" in result:
+ result = result.replace("\n\n\n", "\n\n")
+
+ return result.strip()
+
+ def _format_dynamic_section(
+ self,
+ dynamic_content: str,
+ separator: str = "\n\n---\n\n",
+ ) -> str:
+ """
+ Format extracted dynamic content as a section to append.
+
+ Creates a clearly marked section containing dynamic values,
+ appended to the end of the message to preserve prefix stability.
+
+ Args:
+ dynamic_content: The dynamic content string to append.
+ separator: Separator to use before the dynamic section.
+
+ Returns:
+ Formatted dynamic section string.
+ """
+ if not dynamic_content or not dynamic_content.strip():
+ return ""
+
+ # Format as a context section
+ return f"{separator}[Current Context]\n{dynamic_content.strip()}\n"
+
+ def _analyze_prefix(
+ self,
+ messages: list[dict[str, Any]],
+ context: OptimizationContext,
+ ) -> PrefixAnalysis:
+ """
+ Analyze the prefix for stability metrics.
+
+ Computes hash of the stable prefix portion and compares with
+ previous requests to estimate cache hit likelihood.
+
+ Args:
+ messages: Messages to analyze.
+ context: Optimization context with previous hash.
+
+ Returns:
+ PrefixAnalysis with stability metrics.
+ """
+ # Extract prefix content (system messages + structure)
+ prefix_parts = []
+
+ for msg in messages:
+ if msg.get("role") == "system":
+ content = msg.get("content", "")
+ if isinstance(content, str):
+ prefix_parts.append(content)
+ elif isinstance(content, list):
+ for block in content:
+ if isinstance(block, dict) and block.get("type") == "text":
+ prefix_parts.append(block.get("text", ""))
+
+ prefix_content = "\n".join(prefix_parts)
+ prefix_hash = self._compute_prefix_hash(prefix_content)
+ stable_tokens = self._count_tokens_estimate(prefix_content)
+
+ # Check for changes from previous request
+ previous_hash = context.previous_prefix_hash or self._previous_prefix_hash
+ changed = previous_hash is not None and prefix_hash != previous_hash
+
+ return PrefixAnalysis(
+ prefix_hash=prefix_hash,
+ stable_tokens=stable_tokens,
+ changed_from_previous=changed,
+ previous_hash=previous_hash,
+ )
+
+ def _calculate_cacheable_tokens(self, stable_prefix_tokens: int) -> int:
+ """
+ Calculate how many tokens are likely cacheable.
+
+ OpenAI only caches prompts > 1024 tokens, and caches in chunks.
+
+ Args:
+ stable_prefix_tokens: Number of tokens in stable prefix.
+
+ Returns:
+ Estimated cacheable token count.
+ """
+ if stable_prefix_tokens < self.MIN_TOKENS_FOR_CACHING:
+ return 0
+
+ # OpenAI caches in 128-token chunks (aligned)
+ # Return the aligned cacheable amount
+ return (stable_prefix_tokens // 128) * 128
+
+ def _calculate_savings_percent(
+ self,
+ stable_tokens: int,
+ total_tokens: int,
+ likely_cache_hit: bool,
+ ) -> float:
+ """
+ Calculate estimated savings percentage.
+
+ Args:
+ stable_tokens: Tokens in stable prefix.
+ total_tokens: Total tokens in request.
+ likely_cache_hit: Whether a cache hit is likely.
+
+ Returns:
+ Estimated savings as percentage (0-100).
+ """
+ if total_tokens == 0:
+ return 0.0
+
+ cacheable = self._calculate_cacheable_tokens(stable_tokens)
+ if cacheable == 0:
+ return 0.0
+
+ cacheable_ratio = cacheable / total_tokens
+
+ if likely_cache_hit:
+ # Full 50% savings on cacheable portion
+ return cacheable_ratio * self.CACHE_DISCOUNT_PERCENT
+ else:
+ # No savings on first request, but show potential
+ return 0.0
+
+ def _estimate_total_tokens(self, messages: list[dict[str, Any]]) -> int:
+ """
+ Estimate total tokens in messages.
+
+ Args:
+ messages: Messages to count.
+
+ Returns:
+ Estimated token count.
+ """
+ total = 0
+ for msg in messages:
+ content = msg.get("content", "")
+ if isinstance(content, str):
+ total += self._count_tokens_estimate(content)
+ elif isinstance(content, list):
+ for block in content:
+ if isinstance(block, dict):
+ if block.get("type") == "text":
+ total += self._count_tokens_estimate(
+ block.get("text", "")
+ )
+ elif block.get("type") == "image_url":
+ # Rough estimate for images
+ total += 85 # Base cost
+ return total
+
diff --git a/headroom/cache/registry.py b/headroom/cache/registry.py
new file mode 100644
index 000000000..26856052e
--- /dev/null
+++ b/headroom/cache/registry.py
@@ -0,0 +1,181 @@
+"""
+Cache Optimizer Registry.
+
+Provides a plugin system for registering and retrieving cache optimizers.
+This allows users to swap implementations and register custom optimizers.
+"""
+
+from __future__ import annotations
+
+from typing import Type
+
+from .base import CacheOptimizer, BaseCacheOptimizer, CacheConfig
+
+
+class CacheOptimizerRegistry:
+ """
+ Registry for cache optimizer plugins.
+
+ This registry allows:
+ - Registration of custom optimizers
+ - Retrieval by provider name
+ - Tier-based selection (oss vs enterprise)
+
+ Usage:
+ # Get default optimizer for provider
+ optimizer = CacheOptimizerRegistry.get("anthropic")
+
+ # Get enterprise version if available
+ optimizer = CacheOptimizerRegistry.get("anthropic", tier="enterprise")
+
+ # Register custom optimizer
+ CacheOptimizerRegistry.register("my-provider", MyOptimizer)
+ """
+
+ _optimizers: dict[str, Type[BaseCacheOptimizer]] = {}
+ _instances: dict[str, BaseCacheOptimizer] = {}
+
+ @classmethod
+ def register(
+ cls,
+ name: str,
+ optimizer_class: Type[BaseCacheOptimizer],
+ *,
+ override: bool = False,
+ ) -> None:
+ """
+ Register a cache optimizer.
+
+ Args:
+ name: Name to register under (e.g., "anthropic", "anthropic-enterprise")
+ optimizer_class: The optimizer class to register
+ override: Whether to override existing registration
+
+ Raises:
+ ValueError: If name already registered and override=False
+ """
+ if name in cls._optimizers and not override:
+ raise ValueError(
+ f"Optimizer '{name}' already registered. "
+ f"Use override=True to replace."
+ )
+ cls._optimizers[name] = optimizer_class
+ # Clear cached instance if exists
+ cls._instances.pop(name, None)
+
+ @classmethod
+ def unregister(cls, name: str) -> None:
+ """
+ Unregister a cache optimizer.
+
+ Args:
+ name: Name to unregister
+ """
+ cls._optimizers.pop(name, None)
+ cls._instances.pop(name, None)
+
+ @classmethod
+ def get(
+ cls,
+ provider: str,
+ tier: str = "oss",
+ config: CacheConfig | None = None,
+ *,
+ cached: bool = True,
+ ) -> BaseCacheOptimizer:
+ """
+ Get a cache optimizer for a provider.
+
+ Args:
+ provider: Provider name (e.g., "anthropic", "openai", "google")
+ tier: Tier to get ("oss" or "enterprise")
+ config: Optional configuration
+ cached: Whether to return cached instance
+
+ Returns:
+ Cache optimizer instance
+
+ Raises:
+ KeyError: If no optimizer registered for provider/tier
+ """
+ # Build the lookup key
+ if tier != "oss":
+ key = f"{provider}-{tier}"
+ # Fall back to OSS if enterprise not available
+ if key not in cls._optimizers:
+ key = provider
+ else:
+ key = provider
+
+ if key not in cls._optimizers:
+ available = list(cls._optimizers.keys())
+ raise KeyError(
+ f"No optimizer registered for '{key}'. "
+ f"Available: {available}"
+ )
+
+ # Return cached instance if requested
+ cache_key = f"{key}:{id(config)}" if config else key
+ if cached and cache_key in cls._instances:
+ return cls._instances[cache_key]
+
+ # Create new instance
+ optimizer_class = cls._optimizers[key]
+ instance = optimizer_class(config)
+
+ if cached:
+ cls._instances[cache_key] = instance
+
+ return instance
+
+ @classmethod
+ def list_providers(cls) -> list[str]:
+ """List all registered provider names (excluding tier suffixes)."""
+ providers = set()
+ for name in cls._optimizers:
+ # Remove tier suffix if present
+ base_name = name.split("-")[0]
+ providers.add(base_name)
+ return sorted(providers)
+
+ @classmethod
+ def list_all(cls) -> list[str]:
+ """List all registered optimizer names."""
+ return sorted(cls._optimizers.keys())
+
+ @classmethod
+ def is_registered(cls, name: str) -> bool:
+ """Check if an optimizer is registered."""
+ return name in cls._optimizers
+
+ @classmethod
+ def clear(cls) -> None:
+ """Clear all registrations. Mainly for testing."""
+ cls._optimizers.clear()
+ cls._instances.clear()
+
+ @classmethod
+ def reset_to_defaults(cls) -> None:
+ """Reset to default registrations."""
+ cls.clear()
+ _register_defaults()
+
+
+def _register_defaults() -> None:
+ """Register default optimizers."""
+ # Import here to avoid circular imports
+ from .anthropic import AnthropicCacheOptimizer
+ from .openai import OpenAICacheOptimizer
+ from .google import GoogleCacheOptimizer
+
+ CacheOptimizerRegistry.register("anthropic", AnthropicCacheOptimizer)
+ CacheOptimizerRegistry.register("openai", OpenAICacheOptimizer)
+ CacheOptimizerRegistry.register("google", GoogleCacheOptimizer)
+
+
+# Auto-register defaults on module import
+# Wrapped in try/except to allow partial imports during development
+try:
+ _register_defaults()
+except ImportError:
+ pass
diff --git a/headroom/cache/semantic.py b/headroom/cache/semantic.py
new file mode 100644
index 000000000..17ecafec1
--- /dev/null
+++ b/headroom/cache/semantic.py
@@ -0,0 +1,447 @@
+"""
+Semantic Cache Layer.
+
+Provides query-level semantic caching using embedding similarity.
+This is COMPLEMENTARY to provider prompt caching - it caches complete
+responses for semantically similar queries.
+
+How it works:
+1. When a query comes in, compute its embedding
+2. Search for similar queries in the cache (cosine similarity)
+3. If similarity > threshold, return cached response
+4. Otherwise, proceed with normal optimization
+
+Key difference from Prompt Caching:
+- Prompt Caching: Provider caches KV-cache for prefix (same prompt = faster)
+- Semantic Caching: We cache responses for similar queries (similar query = cached answer)
+
+Usage:
+ from headroom.cache import SemanticCacheLayer, CacheOptimizerRegistry
+
+ # Get provider optimizer
+ provider_optimizer = CacheOptimizerRegistry.get("anthropic")
+
+ # Wrap with semantic layer
+ semantic = SemanticCacheLayer(
+ provider_optimizer,
+ similarity_threshold=0.95,
+ )
+
+ result = semantic.process(messages, context)
+ if result.semantic_cache_hit:
+ # Use result.cached_response directly
+ pass
+"""
+
+from __future__ import annotations
+
+import hashlib
+import time
+from collections import OrderedDict
+from dataclasses import dataclass, field
+from typing import Any, Callable
+
+from .base import (
+ BaseCacheOptimizer,
+ CacheConfig,
+ CacheMetrics,
+ CacheResult,
+ OptimizationContext,
+)
+
+
+@dataclass
+class CacheEntry:
+ """Entry in the semantic cache."""
+
+ # Query embedding
+ embedding: list[float]
+
+ # Original query text
+ query: str
+
+ # Cached response
+ response: Any
+
+ # Metadata
+ created_at: float
+ last_accessed: float
+ access_count: int = 1
+
+ # Hash of the full messages for exact matching
+ messages_hash: str = ""
+
+
+@dataclass
+class SemanticCacheConfig:
+ """Configuration for semantic caching."""
+
+ # Similarity threshold for cache hit (0.0 - 1.0)
+ similarity_threshold: float = 0.95
+
+ # Maximum entries in cache
+ max_entries: int = 1000
+
+ # TTL in seconds (0 = no expiry)
+ ttl_seconds: int = 300
+
+ # Whether to use exact hash matching as fallback
+ use_exact_matching: bool = True
+
+ # Embedding model (if using embeddings)
+ embedding_model: str = "all-MiniLM-L6-v2"
+
+
+class SemanticCache:
+ """
+ In-memory semantic cache with LRU eviction.
+
+ Stores query embeddings and responses, supporting both
+ semantic similarity search and exact hash matching.
+ """
+
+ def __init__(
+ self,
+ config: SemanticCacheConfig | None = None,
+ embedding_fn: Callable[[str], list[float]] | None = None,
+ ):
+ """
+ Initialize the semantic cache.
+
+ Args:
+ config: Cache configuration
+ embedding_fn: Optional custom embedding function.
+ If not provided, uses simple hash-based matching.
+ """
+ self.config = config or SemanticCacheConfig()
+ self._embedding_fn = embedding_fn
+
+ # LRU cache: key -> CacheEntry
+ self._cache: OrderedDict[str, CacheEntry] = OrderedDict()
+
+ # Exact hash index: messages_hash -> key
+ self._hash_index: dict[str, str] = {}
+
+ # Statistics
+ self._hits = 0
+ self._misses = 0
+ self._evictions = 0
+
+ def get(
+ self,
+ query: str,
+ messages_hash: str | None = None,
+ ) -> CacheEntry | None:
+ """
+ Look up a cached entry.
+
+ Args:
+ query: Query text to search for
+ messages_hash: Optional exact hash for fast lookup
+
+ Returns:
+ CacheEntry if found, None otherwise
+ """
+ self._cleanup_expired()
+
+ # Try exact hash match first
+ if messages_hash and self.config.use_exact_matching:
+ key = self._hash_index.get(messages_hash)
+ if key and key in self._cache:
+ entry = self._cache[key]
+ self._touch(key)
+ self._hits += 1
+ return entry
+
+ # Try semantic similarity if we have embedding function
+ if self._embedding_fn:
+ query_embedding = self._embedding_fn(query)
+ best_match, best_similarity = self._find_similar(query_embedding)
+
+ if best_similarity >= self.config.similarity_threshold:
+ self._touch(best_match)
+ self._hits += 1
+ return self._cache[best_match]
+
+ self._misses += 1
+ return None
+
+ def put(
+ self,
+ query: str,
+ response: Any,
+ messages_hash: str | None = None,
+ ) -> str:
+ """
+ Store a response in the cache.
+
+ Args:
+ query: Query text
+ response: Response to cache
+ messages_hash: Optional exact hash for fast lookup
+
+ Returns:
+ Cache key for the entry
+ """
+ self._cleanup_expired()
+
+ # Evict if at capacity
+ while len(self._cache) >= self.config.max_entries:
+ self._evict_oldest()
+
+ # Generate embedding if available
+ embedding: list[float] = []
+ if self._embedding_fn:
+ embedding = self._embedding_fn(query)
+
+ # Create cache key
+ key = self._generate_key(query)
+
+ now = time.time()
+ entry = CacheEntry(
+ embedding=embedding,
+ query=query,
+ response=response,
+ created_at=now,
+ last_accessed=now,
+ messages_hash=messages_hash or "",
+ )
+
+ self._cache[key] = entry
+
+ # Index by hash for fast exact matching
+ if messages_hash:
+ self._hash_index[messages_hash] = key
+
+ return key
+
+ def invalidate(self, key: str) -> bool:
+ """Invalidate a cache entry by key."""
+ if key in self._cache:
+ entry = self._cache.pop(key)
+ if entry.messages_hash:
+ self._hash_index.pop(entry.messages_hash, None)
+ return True
+ return False
+
+ def clear(self) -> None:
+ """Clear all cache entries."""
+ self._cache.clear()
+ self._hash_index.clear()
+
+ def get_stats(self) -> dict[str, Any]:
+ """Get cache statistics."""
+ total = self._hits + self._misses
+ hit_rate = self._hits / total if total > 0 else 0.0
+
+ return {
+ "entries": len(self._cache),
+ "max_entries": self.config.max_entries,
+ "hits": self._hits,
+ "misses": self._misses,
+ "hit_rate": hit_rate,
+ "evictions": self._evictions,
+ }
+
+ def _find_similar(
+ self,
+ query_embedding: list[float],
+ ) -> tuple[str, float]:
+ """Find the most similar cached entry."""
+ best_key = ""
+ best_similarity = -1.0
+
+ for key, entry in self._cache.items():
+ if not entry.embedding:
+ continue
+
+ similarity = self._cosine_similarity(query_embedding, entry.embedding)
+ if similarity > best_similarity:
+ best_similarity = similarity
+ best_key = key
+
+ return best_key, best_similarity
+
+ def _cosine_similarity(
+ self,
+ a: list[float],
+ b: list[float],
+ ) -> float:
+ """Compute cosine similarity between two vectors."""
+ if len(a) != len(b) or not a:
+ return 0.0
+
+ dot_product = sum(x * y for x, y in zip(a, b))
+ norm_a = sum(x * x for x in a) ** 0.5
+ norm_b = sum(x * x for x in b) ** 0.5
+
+ if norm_a == 0 or norm_b == 0:
+ return 0.0
+
+ return dot_product / (norm_a * norm_b)
+
+ def _touch(self, key: str) -> None:
+ """Update access time and move to end of LRU."""
+ if key in self._cache:
+ entry = self._cache.pop(key)
+ entry.last_accessed = time.time()
+ entry.access_count += 1
+ self._cache[key] = entry
+
+ def _evict_oldest(self) -> None:
+ """Evict the oldest (least recently used) entry."""
+ if self._cache:
+ key, entry = self._cache.popitem(last=False)
+ if entry.messages_hash:
+ self._hash_index.pop(entry.messages_hash, None)
+ self._evictions += 1
+
+ def _cleanup_expired(self) -> None:
+ """Remove expired entries."""
+ if self.config.ttl_seconds <= 0:
+ return
+
+ now = time.time()
+ expired = [
+ key for key, entry in self._cache.items()
+ if now - entry.created_at > self.config.ttl_seconds
+ ]
+
+ for key in expired:
+ entry = self._cache.pop(key)
+ if entry.messages_hash:
+ self._hash_index.pop(entry.messages_hash, None)
+
+ def _generate_key(self, query: str) -> str:
+ """Generate a cache key for a query."""
+ return hashlib.sha256(query.encode()).hexdigest()[:16]
+
+
+class SemanticCacheLayer:
+ """
+ Layer that adds semantic caching on top of provider optimizers.
+
+ This layer checks for semantically similar queries before
+ delegating to the underlying provider optimizer.
+ """
+
+ def __init__(
+ self,
+ provider_optimizer: BaseCacheOptimizer,
+ similarity_threshold: float = 0.95,
+ max_entries: int = 1000,
+ ttl_seconds: int = 300,
+ embedding_fn: Callable[[str], list[float]] | None = None,
+ ):
+ """
+ Initialize the semantic cache layer.
+
+ Args:
+ provider_optimizer: Underlying provider optimizer
+ similarity_threshold: Similarity threshold for cache hits
+ max_entries: Maximum cache entries
+ ttl_seconds: Cache TTL in seconds
+ embedding_fn: Optional embedding function
+ """
+ self.provider_optimizer = provider_optimizer
+
+ cache_config = SemanticCacheConfig(
+ similarity_threshold=similarity_threshold,
+ max_entries=max_entries,
+ ttl_seconds=ttl_seconds,
+ )
+ self.cache = SemanticCache(cache_config, embedding_fn)
+
+ def process(
+ self,
+ messages: list[dict[str, Any]],
+ context: OptimizationContext,
+ config: CacheConfig | None = None,
+ ) -> CacheResult:
+ """
+ Process messages through semantic cache and provider optimizer.
+
+ Args:
+ messages: Messages to process
+ context: Optimization context
+ config: Optional configuration override
+
+ Returns:
+ CacheResult with semantic_cache_hit=True if cache hit
+ """
+ # Extract query for semantic matching
+ query = context.query or self._extract_query(messages)
+ messages_hash = self._compute_messages_hash(messages)
+
+ # Check semantic cache
+ cached = self.cache.get(query, messages_hash)
+ if cached:
+ return CacheResult(
+ messages=messages,
+ semantic_cache_hit=True,
+ cached_response=cached.response,
+ metrics=CacheMetrics(
+ estimated_cache_hit=True,
+ estimated_savings_percent=100.0,
+ ),
+ transforms_applied=["semantic_cache_hit"],
+ )
+
+ # Delegate to provider optimizer
+ result = self.provider_optimizer.optimize(messages, context, config)
+
+ return result
+
+ def store_response(
+ self,
+ messages: list[dict[str, Any]],
+ response: Any,
+ context: OptimizationContext | None = None,
+ ) -> str:
+ """
+ Store a response in the semantic cache.
+
+ Call this after receiving a response from the LLM to enable
+ future cache hits.
+
+ Args:
+ messages: Original messages
+ response: Response from LLM
+ context: Optional context with query
+
+ Returns:
+ Cache key
+ """
+ query = (context.query if context else None) or self._extract_query(messages)
+ messages_hash = self._compute_messages_hash(messages)
+
+ return self.cache.put(query, response, messages_hash)
+
+ def get_stats(self) -> dict[str, Any]:
+ """Get combined statistics."""
+ return {
+ "semantic_cache": self.cache.get_stats(),
+ "provider_optimizer": self.provider_optimizer.name,
+ }
+
+ def _extract_query(self, messages: list[dict[str, Any]]) -> str:
+ """Extract the last user query from messages."""
+ for msg in reversed(messages):
+ if msg.get("role") == "user":
+ content = msg.get("content", "")
+ if isinstance(content, str):
+ return content
+ elif isinstance(content, list):
+ for block in content:
+ if isinstance(block, dict) and block.get("type") == "text":
+ return block.get("text", "")
+ return ""
+
+ def _compute_messages_hash(self, messages: list[dict[str, Any]]) -> str:
+ """Compute a hash of all messages."""
+ import json
+ try:
+ content = json.dumps(messages, sort_keys=True)
+ return hashlib.sha256(content.encode()).hexdigest()[:24]
+ except (TypeError, ValueError):
+ return ""
diff --git a/headroom/client.py b/headroom/client.py
index a77cda7b2..900675e82 100644
--- a/headroom/client.py
+++ b/headroom/client.py
@@ -6,6 +6,13 @@ from collections.abc import Iterator
from datetime import datetime
from typing import Any
+from .cache import (
+ BaseCacheOptimizer,
+ CacheOptimizerRegistry,
+ CacheConfig,
+ OptimizationContext,
+ SemanticCacheLayer,
+)
from .config import (
HeadroomConfig,
HeadroomMode,
@@ -256,6 +263,9 @@ class HeadroomClient:
store_url: str = "sqlite:///headroom.db",
default_mode: str = "audit",
model_context_limits: dict[str, int] | None = None,
+ cache_optimizer: BaseCacheOptimizer | None = None,
+ enable_cache_optimizer: bool = True,
+ enable_semantic_cache: bool = False,
):
"""
Initialize HeadroomClient.
@@ -266,6 +276,10 @@ class HeadroomClient:
store_url: Storage URL (sqlite:// or jsonl://).
default_mode: Default mode ("audit" | "optimize").
model_context_limits: Override context limits for models.
+ cache_optimizer: Optional custom cache optimizer. If None and
+ enable_cache_optimizer=True, auto-detects from provider.
+ enable_cache_optimizer: Enable provider-specific cache optimization.
+ enable_semantic_cache: Enable query-level semantic caching.
"""
self._original = original_client
self._provider = provider
@@ -276,6 +290,8 @@ class HeadroomClient:
self._config = HeadroomConfig()
self._config.store_url = store_url
self._config.default_mode = self._default_mode
+ self._config.cache_optimizer.enabled = enable_cache_optimizer
+ self._config.cache_optimizer.enable_semantic_cache = enable_semantic_cache
if model_context_limits:
self._config.model_context_limits.update(model_context_limits)
@@ -286,6 +302,34 @@ class HeadroomClient:
# Initialize transform pipeline
self._pipeline = TransformPipeline(self._config, provider=self._provider)
+ # Initialize cache optimizer
+ self._cache_optimizer: BaseCacheOptimizer | None = None
+ self._semantic_cache_layer: SemanticCacheLayer | None = None
+
+ if enable_cache_optimizer:
+ if cache_optimizer is not None:
+ self._cache_optimizer = cache_optimizer
+ else:
+ # Auto-detect from provider
+ provider_name = self._provider.name.lower()
+ if CacheOptimizerRegistry.is_registered(provider_name):
+ cache_config = CacheConfig(
+ min_cacheable_tokens=self._config.cache_optimizer.min_cacheable_tokens,
+ )
+ self._cache_optimizer = CacheOptimizerRegistry.get(
+ provider_name,
+ config=cache_config,
+ )
+
+ # Wrap with semantic cache if enabled
+ if enable_semantic_cache and self._cache_optimizer is not None:
+ self._semantic_cache_layer = SemanticCacheLayer(
+ self._cache_optimizer,
+ similarity_threshold=self._config.cache_optimizer.semantic_cache_similarity,
+ max_entries=self._config.cache_optimizer.semantic_cache_max_entries,
+ ttl_seconds=self._config.cache_optimizer.semantic_cache_ttl_seconds,
+ )
+
# Public API - OpenAI style
self.chat = type("Chat", (), {"completions": ChatCompletions(self)})()
# Public API - Anthropic style
@@ -339,6 +383,16 @@ class HeadroomClient:
# Compute stable prefix hash
stable_prefix_hash = compute_prefix_hash(messages)
+ # Cache optimizer metrics (populated later if optimizer is used)
+ cache_optimizer_used = None
+ cache_optimizer_strategy = None
+ cacheable_tokens = 0
+ breakpoints_inserted = 0
+ estimated_cache_hit = False
+ estimated_savings_percent = 0.0
+ semantic_cache_hit = False
+ cached_response = None
+
# Apply transforms if in optimize mode
if mode == HeadroomMode.OPTIMIZE:
output_buffer = headroom_output_buffer_tokens or self._config.rolling_window.output_buffer_tokens
@@ -356,6 +410,54 @@ class HeadroomClient:
tokens_after = result.tokens_after
transforms_applied = result.transforms_applied
+ # Apply provider-specific cache optimization
+ if self._cache_optimizer is not None or self._semantic_cache_layer is not None:
+ cache_context = OptimizationContext(
+ provider=self._provider.name.lower(),
+ model=model,
+ query=self._extract_query(optimized_messages),
+ )
+
+ # Check semantic cache first (if enabled)
+ if self._semantic_cache_layer is not None:
+ cache_result = self._semantic_cache_layer.process(
+ optimized_messages, cache_context
+ )
+ semantic_cache_hit = cache_result.semantic_cache_hit
+ if semantic_cache_hit:
+ cached_response = cache_result.cached_response
+
+ # Update metrics from cache result
+ cache_optimizer_used = cache_result.metrics.optimizer_name or self._cache_optimizer.name
+ cache_optimizer_strategy = cache_result.metrics.strategy
+ cacheable_tokens = cache_result.metrics.cacheable_tokens
+ breakpoints_inserted = cache_result.metrics.breakpoints_inserted
+ estimated_cache_hit = cache_result.metrics.estimated_cache_hit
+ estimated_savings_percent = cache_result.metrics.estimated_savings_percent
+
+ # Apply optimized messages (with cache_control blocks for Anthropic)
+ if cache_result.messages:
+ optimized_messages = cache_result.messages
+
+ elif self._cache_optimizer is not None:
+ # Direct cache optimizer (no semantic layer)
+ cache_result = self._cache_optimizer.optimize(
+ optimized_messages, cache_context
+ )
+ cache_optimizer_used = self._cache_optimizer.name
+ cache_optimizer_strategy = self._cache_optimizer.strategy.value
+ cacheable_tokens = cache_result.metrics.cacheable_tokens
+ breakpoints_inserted = cache_result.metrics.breakpoints_inserted
+ estimated_cache_hit = cache_result.metrics.estimated_cache_hit
+ estimated_savings_percent = cache_result.metrics.estimated_savings_percent
+
+ if cache_result.messages:
+ optimized_messages = cache_result.messages
+
+ transforms_applied.extend(
+ f"cache_optimizer:{t}" for t in (cache_result.transforms_applied or [])
+ )
+
# Recalculate prefix hash after optimization
stable_prefix_hash = compute_prefix_hash(optimized_messages)
else:
@@ -379,8 +481,21 @@ class HeadroomClient:
cache_alignment_score=cache_alignment_score,
transforms_applied=transforms_applied,
messages_hash=compute_messages_hash(messages),
+ # Cache optimizer metrics
+ cache_optimizer_used=cache_optimizer_used,
+ cache_optimizer_strategy=cache_optimizer_strategy,
+ cacheable_tokens=cacheable_tokens,
+ breakpoints_inserted=breakpoints_inserted,
+ estimated_cache_hit=estimated_cache_hit,
+ estimated_savings_percent=estimated_savings_percent,
+ semantic_cache_hit=semantic_cache_hit,
)
+ # Return cached response if semantic cache hit
+ if semantic_cache_hit and cached_response is not None:
+ self._storage.save(metrics)
+ return cached_response
+
# Call underlying client based on API style
try:
if api_style == "anthropic":
@@ -494,6 +609,65 @@ class HeadroomClient:
# Note: output tokens unknown for streams
self._storage.save(metrics)
+ def _extract_query(self, messages: list[dict[str, Any]]) -> str:
+ """Extract query from messages for semantic caching.
+
+ Returns the last user message content as the query.
+ """
+ for msg in reversed(messages):
+ if msg.get("role") == "user":
+ content = msg.get("content", "")
+ if isinstance(content, str):
+ return content
+ elif isinstance(content, list):
+ # Content block format
+ for block in content:
+ if isinstance(block, dict) and block.get("type") == "text":
+ return block.get("text", "")
+ return ""
+ return ""
+
+ def _store_response_in_semantic_cache(
+ self,
+ messages: list[dict[str, Any]],
+ response: Any,
+ model: str,
+ ) -> None:
+ """Store response in semantic cache for future hits."""
+ if self._semantic_cache_layer is not None:
+ cache_context = OptimizationContext(
+ provider=self._provider.name.lower(),
+ model=model,
+ query=self._extract_query(messages),
+ )
+ # Extract response content for caching
+ response_data = self._extract_response_content(response)
+ if response_data:
+ self._semantic_cache_layer.store_response(
+ messages, response_data, cache_context
+ )
+
+ def _extract_response_content(self, response: Any) -> dict[str, Any] | None:
+ """Extract cacheable content from API response."""
+ try:
+ # OpenAI format
+ if hasattr(response, "choices") and response.choices:
+ choice = response.choices[0]
+ if hasattr(choice, "message"):
+ return {
+ "role": "assistant",
+ "content": choice.message.content,
+ }
+ # Anthropic format
+ elif hasattr(response, "content"):
+ return {
+ "role": "assistant",
+ "content": response.content,
+ }
+ except Exception:
+ pass
+ return None
+
def _simulate(
self,
*,
diff --git a/headroom/config.py b/headroom/config.py
index 770d544fa..9b6d6e8b9 100644
--- a/headroom/config.py
+++ b/headroom/config.py
@@ -193,6 +193,31 @@ class SmartCrusherConfig:
relevance: RelevanceScorerConfig = field(default_factory=RelevanceScorerConfig)
+@dataclass
+class CacheOptimizerConfig:
+ """Configuration for provider-specific cache optimization.
+
+ The CacheOptimizer system provides provider-specific caching strategies:
+ - Anthropic: Explicit cache_control breakpoints for prompt caching
+ - OpenAI: Prefix stabilization for automatic prefix caching
+ - Google: CachedContent API lifecycle management
+
+ This is COMPLEMENTARY to the CacheAligner transform - CacheAligner does
+ basic prefix stabilization (date extraction, whitespace normalization),
+ while CacheOptimizer applies provider-specific optimizations.
+
+ Enable this for maximum cache hit rates when you know your provider.
+ """
+
+ enabled: bool = True # Enable provider-specific cache optimization
+ auto_detect_provider: bool = True # Auto-detect from HeadroomClient provider
+ min_cacheable_tokens: int = 1024 # Minimum tokens for caching (provider may override)
+ enable_semantic_cache: bool = False # Enable query-level semantic caching
+ semantic_cache_similarity: float = 0.95 # Similarity threshold for semantic cache
+ semantic_cache_max_entries: int = 1000 # Max semantic cache entries
+ semantic_cache_ttl_seconds: int = 300 # Semantic cache TTL
+
+
@dataclass
class HeadroomConfig:
"""Main configuration for HeadroomClient."""
@@ -206,6 +231,7 @@ class HeadroomConfig:
smart_crusher: SmartCrusherConfig = field(default_factory=SmartCrusherConfig)
cache_aligner: CacheAlignerConfig = field(default_factory=CacheAlignerConfig)
rolling_window: RollingWindowConfig = field(default_factory=RollingWindowConfig)
+ cache_optimizer: CacheOptimizerConfig = field(default_factory=CacheOptimizerConfig)
# Debugging - opt-in diff artifact generation
generate_diff_artifact: bool = False # Enable to get detailed transform diffs
@@ -370,11 +396,20 @@ class RequestMetrics:
# Waste signals
waste_signals: dict[str, int] = field(default_factory=dict)
- # Cache metrics
+ # Cache metrics (basic)
stable_prefix_hash: str = ""
cache_alignment_score: float = 0.0
cached_tokens: int | None = None # From API response if available
+ # Cache optimizer metrics (provider-specific)
+ cache_optimizer_used: str | None = None # e.g., "anthropic-cache-optimizer"
+ cache_optimizer_strategy: str | None = None # e.g., "explicit_breakpoints"
+ cacheable_tokens: int = 0 # Tokens eligible for caching
+ breakpoints_inserted: int = 0 # Cache breakpoints added (Anthropic)
+ estimated_cache_hit: bool = False # Whether prefix matches previous
+ estimated_savings_percent: float = 0.0 # Estimated savings if cached
+ semantic_cache_hit: bool = False # Whether semantic cache was hit
+
# Transform details
transforms_applied: list[str] = field(default_factory=list)
tool_units_dropped: int = 0
diff --git a/tests/test_cache/__init__.py b/tests/test_cache/__init__.py
new file mode 100644
index 000000000..875b77f8a
--- /dev/null
+++ b/tests/test_cache/__init__.py
@@ -0,0 +1 @@
+"""Tests for the cache optimization module."""
diff --git a/tests/test_cache/test_anthropic.py b/tests/test_cache/test_anthropic.py
new file mode 100644
index 000000000..81cdcb126
--- /dev/null
+++ b/tests/test_cache/test_anthropic.py
@@ -0,0 +1,184 @@
+"""Tests for AnthropicCacheOptimizer."""
+
+import pytest
+from headroom.cache import (
+ AnthropicCacheOptimizer,
+ CacheConfig,
+ OptimizationContext,
+)
+from headroom.cache.base import CacheStrategy, BreakpointLocation
+
+
+class TestAnthropicCacheOptimizer:
+ """Test AnthropicCacheOptimizer functionality."""
+
+ @pytest.fixture
+ def optimizer(self):
+ """Create optimizer instance."""
+ return AnthropicCacheOptimizer()
+
+ @pytest.fixture
+ def context(self):
+ """Create optimization context."""
+ return OptimizationContext(
+ provider="anthropic",
+ model="claude-3-opus",
+ )
+
+ def test_optimizer_properties(self, optimizer):
+ """Test optimizer properties."""
+ assert optimizer.name == "anthropic-cache-optimizer"
+ assert optimizer.provider == "anthropic"
+ assert optimizer.strategy == CacheStrategy.EXPLICIT_BREAKPOINTS
+
+ def test_enforces_minimum_tokens(self):
+ """Test that Anthropic minimum is enforced."""
+ config = CacheConfig(min_cacheable_tokens=100)
+ optimizer = AnthropicCacheOptimizer(config)
+ assert optimizer.config.min_cacheable_tokens >= 1024
+
+ def test_enforces_maximum_breakpoints(self):
+ """Test that Anthropic maximum breakpoints is enforced."""
+ config = CacheConfig(max_breakpoints=10)
+ optimizer = AnthropicCacheOptimizer(config)
+ assert optimizer.config.max_breakpoints <= 4
+
+ def test_optimize_simple_messages(self, optimizer, context):
+ """Test optimizing simple messages."""
+ messages = [
+ {"role": "system", "content": "You are a helpful assistant. " * 500},
+ {"role": "user", "content": "Hello!"},
+ ]
+
+ result = optimizer.optimize(messages, context)
+
+ assert result.messages is not None
+ assert len(result.messages) == 2
+ assert result.metrics.stable_prefix_hash != ""
+
+ def test_optimize_inserts_cache_control(self, optimizer, context):
+ """Test that optimization inserts cache_control blocks."""
+ # Large system prompt to trigger caching
+ messages = [
+ {"role": "system", "content": "You are a helpful assistant. " * 500},
+ {"role": "user", "content": "Hello!"},
+ ]
+
+ result = optimizer.optimize(messages, context)
+
+ # Check if cache_control was inserted
+ system_content = result.messages[0]["content"]
+ if isinstance(system_content, list):
+ has_cache_control = any(
+ "cache_control" in block
+ for block in system_content
+ if isinstance(block, dict)
+ )
+ assert has_cache_control
+
+ def test_optimize_with_dates(self, optimizer, context):
+ """Test optimization extracts dates."""
+ messages = [
+ {
+ "role": "system",
+ "content": "Today is January 7, 2026. You are a helpful assistant. " * 300,
+ },
+ {"role": "user", "content": "Hello!"},
+ ]
+
+ result = optimizer.optimize(messages, context)
+
+ # Dates should be moved to end
+ assert "extracted_dates" in result.transforms_applied or result.metrics.breakpoints_inserted >= 0
+
+ def test_optimize_disabled(self, context):
+ """Test optimization when disabled."""
+ config = CacheConfig(enabled=False)
+ optimizer = AnthropicCacheOptimizer(config)
+
+ messages = [
+ {"role": "system", "content": "Test"},
+ {"role": "user", "content": "Hello!"},
+ ]
+
+ result = optimizer.optimize(messages, context)
+
+ assert result.transforms_applied == []
+
+ def test_prefix_hash_tracking(self, optimizer, context):
+ """Test that prefix hash is tracked between calls."""
+ messages = [
+ {"role": "system", "content": "You are a helpful assistant. " * 500},
+ {"role": "user", "content": "Hello!"},
+ ]
+
+ result1 = optimizer.optimize(messages, context)
+ result2 = optimizer.optimize(messages, context)
+
+ # Second call should detect stable prefix
+ assert result2.metrics.previous_prefix_hash == result1.metrics.stable_prefix_hash
+
+ def test_estimate_savings(self, optimizer, context):
+ """Test savings estimation."""
+ messages = [
+ {"role": "system", "content": "You are a helpful assistant. " * 500},
+ {"role": "user", "content": "Hello!"},
+ ]
+
+ savings = optimizer.estimate_savings(messages, context)
+ assert savings >= 0.0
+ assert savings <= 100.0
+
+ def test_content_block_format(self, optimizer, context):
+ """Test handling of content block format."""
+ messages = [
+ {
+ "role": "system",
+ "content": [
+ {"type": "text", "text": "You are a helpful assistant. " * 500}
+ ],
+ },
+ {"role": "user", "content": "Hello!"},
+ ]
+
+ result = optimizer.optimize(messages, context)
+ assert result.messages is not None
+
+ def test_tools_are_cacheable(self, optimizer, context):
+ """Test that tools are identified as cacheable."""
+ messages = [
+ {"role": "system", "content": "You are helpful. " * 300},
+ {
+ "role": "user",
+ "content": "Use tools",
+ "tools": [
+ {
+ "name": "search",
+ "description": "Search the web " * 200,
+ "input_schema": {"type": "object"},
+ }
+ ],
+ },
+ ]
+
+ result = optimizer.optimize(messages, context)
+ assert result.metrics.cacheable_tokens > 0
+
+ def test_metrics_history(self, optimizer, context):
+ """Test that metrics are recorded."""
+ messages = [
+ {"role": "system", "content": "You are helpful. " * 500},
+ {"role": "user", "content": "Hello!"},
+ ]
+
+ optimizer.optimize(messages, context)
+ metrics = optimizer.get_metrics()
+
+ assert metrics is not None
+ assert metrics.stable_prefix_hash != ""
+
+ def test_cache_constants(self, optimizer):
+ """Test cache-related constants."""
+ assert optimizer.get_cache_write_cost_multiplier() == 1.25
+ assert optimizer.get_cache_read_cost_multiplier() == 0.10
+ assert optimizer.get_cache_ttl_seconds() == 300
diff --git a/tests/test_cache/test_base.py b/tests/test_cache/test_base.py
new file mode 100644
index 000000000..b2d04d49c
--- /dev/null
+++ b/tests/test_cache/test_base.py
@@ -0,0 +1,141 @@
+"""Tests for cache base types and interfaces."""
+
+import pytest
+from headroom.cache.base import (
+ CacheStrategy,
+ BreakpointLocation,
+ CacheBreakpoint,
+ CacheConfig,
+ CacheMetrics,
+ CacheResult,
+ OptimizationContext,
+)
+
+
+class TestCacheStrategy:
+ """Test CacheStrategy enum."""
+
+ def test_strategies_exist(self):
+ """Test all expected strategies exist."""
+ assert CacheStrategy.PREFIX_STABILIZATION.value == "prefix_stabilization"
+ assert CacheStrategy.EXPLICIT_BREAKPOINTS.value == "explicit_breakpoints"
+ assert CacheStrategy.CACHED_CONTENT.value == "cached_content"
+ assert CacheStrategy.NONE.value == "none"
+
+
+class TestCacheConfig:
+ """Test CacheConfig dataclass."""
+
+ def test_default_values(self):
+ """Test default configuration values."""
+ config = CacheConfig()
+ assert config.enabled is True
+ assert config.min_cacheable_tokens == 1024
+ assert config.max_breakpoints == 4
+ assert config.normalize_whitespace is True
+ assert config.collapse_blank_lines is True
+
+ def test_custom_values(self):
+ """Test custom configuration."""
+ config = CacheConfig(
+ enabled=False,
+ min_cacheable_tokens=2048,
+ max_breakpoints=2,
+ )
+ assert config.enabled is False
+ assert config.min_cacheable_tokens == 2048
+ assert config.max_breakpoints == 2
+
+ def test_date_patterns(self):
+ """Test date patterns are set."""
+ config = CacheConfig()
+ assert len(config.date_patterns) > 0
+ assert any("Today" in p for p in config.date_patterns)
+
+
+class TestCacheMetrics:
+ """Test CacheMetrics dataclass."""
+
+ def test_default_values(self):
+ """Test default metrics values."""
+ metrics = CacheMetrics()
+ assert metrics.stable_prefix_tokens == 0
+ assert metrics.breakpoints_inserted == 0
+ assert metrics.estimated_cache_hit is False
+ assert metrics.estimated_savings_percent == 0.0
+
+ def test_custom_values(self):
+ """Test custom metrics."""
+ metrics = CacheMetrics(
+ stable_prefix_tokens=5000,
+ breakpoints_inserted=2,
+ estimated_cache_hit=True,
+ estimated_savings_percent=90.0,
+ )
+ assert metrics.stable_prefix_tokens == 5000
+ assert metrics.breakpoints_inserted == 2
+ assert metrics.estimated_cache_hit is True
+ assert metrics.estimated_savings_percent == 90.0
+
+
+class TestCacheBreakpoint:
+ """Test CacheBreakpoint dataclass."""
+
+ def test_breakpoint_creation(self):
+ """Test creating a breakpoint."""
+ bp = CacheBreakpoint(
+ message_index=0,
+ location=BreakpointLocation.AFTER_SYSTEM,
+ tokens_at_breakpoint=2000,
+ reason="System prompt is cacheable",
+ )
+ assert bp.message_index == 0
+ assert bp.location == BreakpointLocation.AFTER_SYSTEM
+ assert bp.tokens_at_breakpoint == 2000
+ assert bp.content_index is None
+
+
+class TestCacheResult:
+ """Test CacheResult dataclass."""
+
+ def test_result_creation(self):
+ """Test creating a cache result."""
+ messages = [{"role": "system", "content": "Hello"}]
+ result = CacheResult(
+ messages=messages,
+ metrics=CacheMetrics(cacheable_tokens=1000),
+ transforms_applied=["normalized_whitespace"],
+ )
+ assert result.messages == messages
+ assert result.metrics.cacheable_tokens == 1000
+ assert "normalized_whitespace" in result.transforms_applied
+
+ def test_semantic_cache_hit(self):
+ """Test semantic cache hit result."""
+ result = CacheResult(
+ messages=[],
+ semantic_cache_hit=True,
+ cached_response={"text": "cached response"},
+ )
+ assert result.semantic_cache_hit is True
+ assert result.cached_response["text"] == "cached response"
+
+
+class TestOptimizationContext:
+ """Test OptimizationContext dataclass."""
+
+ def test_context_creation(self):
+ """Test creating optimization context."""
+ context = OptimizationContext(
+ provider="anthropic",
+ model="claude-3-opus",
+ request_id="req-123",
+ )
+ assert context.provider == "anthropic"
+ assert context.model == "claude-3-opus"
+ assert context.request_id == "req-123"
+
+ def test_default_timestamp(self):
+ """Test default timestamp is set."""
+ context = OptimizationContext()
+ assert context.timestamp is not None
diff --git a/tests/test_cache/test_client_integration.py b/tests/test_cache/test_client_integration.py
new file mode 100644
index 000000000..d5fa7606a
--- /dev/null
+++ b/tests/test_cache/test_client_integration.py
@@ -0,0 +1,221 @@
+"""Tests for HeadroomClient cache optimizer integration."""
+
+import pytest
+import tempfile
+import os
+from unittest.mock import MagicMock, patch
+from headroom import (
+ HeadroomClient,
+ AnthropicCacheOptimizer,
+ CacheOptimizerRegistry,
+)
+from headroom.providers import AnthropicProvider, OpenAIProvider
+
+
+def temp_db():
+ """Create a temporary database file."""
+ fd, path = tempfile.mkstemp(suffix=".db")
+ os.close(fd)
+ yield f"sqlite:///{path}"
+ if os.path.exists(path):
+ os.unlink(path)
+
+
+class MockTokenCounter:
+ """Mock token counter for testing."""
+
+ def count_tokens(self, text: str) -> int:
+ return len(text) // 4
+
+ def count_messages(self, messages: list) -> int:
+ total = 0
+ for msg in messages:
+ content = msg.get("content", "")
+ if isinstance(content, str):
+ total += len(content) // 4
+ elif isinstance(content, list):
+ for block in content:
+ if isinstance(block, dict):
+ total += len(block.get("text", "")) // 4
+ return total
+
+
+class MockAnthropicProvider:
+ """Mock Anthropic provider for testing."""
+
+ name = "anthropic"
+
+ def get_token_counter(self, model: str):
+ return MockTokenCounter()
+
+ def get_context_limit(self, model: str) -> int:
+ return 200000
+
+
+class MockOpenAIProvider:
+ """Mock OpenAI provider for testing."""
+
+ name = "openai"
+
+ def get_token_counter(self, model: str):
+ return MockTokenCounter()
+
+ def get_context_limit(self, model: str) -> int:
+ return 128000
+
+
+class TestHeadroomClientCacheIntegration:
+ """Test HeadroomClient cache optimizer integration."""
+
+ def test_auto_detect_anthropic_optimizer(self, temp_db):
+ """Test that Anthropic optimizer is auto-detected."""
+ mock_client = MagicMock()
+ provider = MockAnthropicProvider()
+
+ client = HeadroomClient(
+ original_client=mock_client,
+ provider=provider,
+ store_url=temp_db,
+ enable_cache_optimizer=True,
+ )
+
+ assert client._cache_optimizer is not None
+ assert client._cache_optimizer.name == "anthropic-cache-optimizer"
+
+ def test_auto_detect_openai_optimizer(self, temp_db):
+ """Test that OpenAI optimizer is auto-detected."""
+ mock_client = MagicMock()
+ provider = MockOpenAIProvider()
+
+ client = HeadroomClient(
+ original_client=mock_client,
+ provider=provider,
+ store_url=temp_db,
+ enable_cache_optimizer=True,
+ )
+
+ assert client._cache_optimizer is not None
+ assert client._cache_optimizer.name == "openai-prefix-stabilizer"
+
+ def test_custom_optimizer(self, temp_db):
+ """Test using a custom optimizer."""
+ mock_client = MagicMock()
+ provider = MockAnthropicProvider()
+ custom_optimizer = AnthropicCacheOptimizer()
+
+ client = HeadroomClient(
+ original_client=mock_client,
+ provider=provider,
+ store_url=temp_db,
+ cache_optimizer=custom_optimizer,
+ )
+
+ assert client._cache_optimizer is custom_optimizer
+
+ def test_disable_cache_optimizer(self, temp_db):
+ """Test disabling cache optimizer."""
+ mock_client = MagicMock()
+ provider = MockAnthropicProvider()
+
+ client = HeadroomClient(
+ original_client=mock_client,
+ provider=provider,
+ store_url=temp_db,
+ enable_cache_optimizer=False,
+ )
+
+ assert client._cache_optimizer is None
+
+ def test_semantic_cache_layer_creation(self, temp_db):
+ """Test semantic cache layer is created when enabled."""
+ mock_client = MagicMock()
+ provider = MockAnthropicProvider()
+
+ client = HeadroomClient(
+ original_client=mock_client,
+ provider=provider,
+ store_url=temp_db,
+ enable_cache_optimizer=True,
+ enable_semantic_cache=True,
+ )
+
+ assert client._semantic_cache_layer is not None
+ assert client._cache_optimizer is not None
+
+ def test_extract_query_from_string_content(self, temp_db):
+ """Test query extraction from string content."""
+ mock_client = MagicMock()
+ provider = MockAnthropicProvider()
+
+ client = HeadroomClient(
+ original_client=mock_client,
+ provider=provider,
+ store_url=temp_db,
+ )
+
+ messages = [
+ {"role": "system", "content": "You are helpful."},
+ {"role": "user", "content": "What is 2+2?"},
+ ]
+
+ query = client._extract_query(messages)
+ assert query == "What is 2+2?"
+
+ def test_extract_query_from_content_blocks(self, temp_db):
+ """Test query extraction from content block format."""
+ mock_client = MagicMock()
+ provider = MockAnthropicProvider()
+
+ client = HeadroomClient(
+ original_client=mock_client,
+ provider=provider,
+ store_url=temp_db,
+ )
+
+ messages = [
+ {"role": "system", "content": "You are helpful."},
+ {
+ "role": "user",
+ "content": [{"type": "text", "text": "What is 2+2?"}],
+ },
+ ]
+
+ query = client._extract_query(messages)
+ assert query == "What is 2+2?"
+
+ def test_extract_query_last_user_message(self, temp_db):
+ """Test that query extraction uses last user message."""
+ mock_client = MagicMock()
+ provider = MockAnthropicProvider()
+
+ client = HeadroomClient(
+ original_client=mock_client,
+ provider=provider,
+ store_url=temp_db,
+ )
+
+ messages = [
+ {"role": "user", "content": "First question"},
+ {"role": "assistant", "content": "First answer"},
+ {"role": "user", "content": "Second question"},
+ ]
+
+ query = client._extract_query(messages)
+ assert query == "Second question"
+
+ def test_config_propagation(self, temp_db):
+ """Test that config is properly propagated."""
+ mock_client = MagicMock()
+ provider = MockAnthropicProvider()
+
+ client = HeadroomClient(
+ original_client=mock_client,
+ provider=provider,
+ store_url=temp_db,
+ enable_cache_optimizer=True,
+ enable_semantic_cache=True,
+ )
+
+ assert client._config.cache_optimizer.enabled is True
+ assert client._config.cache_optimizer.enable_semantic_cache is True
diff --git a/tests/test_cache/test_dynamic_detector.py b/tests/test_cache/test_dynamic_detector.py
new file mode 100644
index 000000000..2617f3d8e
--- /dev/null
+++ b/tests/test_cache/test_dynamic_detector.py
@@ -0,0 +1,505 @@
+"""Tests for the dynamic content detector."""
+
+import pytest
+from headroom.cache.dynamic_detector import (
+ DetectionResult,
+ DetectorConfig,
+ DynamicCategory,
+ DynamicContentDetector,
+ DynamicSpan,
+ RegexDetector,
+ detect_dynamic_content,
+)
+
+
+class TestRegexDetector:
+ """Test the Tier 1 regex detector."""
+
+ @pytest.fixture
+ def detector(self):
+ """Create a regex detector."""
+ config = DetectorConfig(tiers=["regex"])
+ return RegexDetector(config)
+
+ def test_iso_date(self, detector):
+ """Test ISO date detection."""
+ spans = detector.detect("The date is 2024-01-15.")
+ assert len(spans) == 1
+ assert spans[0].text == "2024-01-15"
+ assert spans[0].category == DynamicCategory.DATE
+ assert spans[0].tier == "regex"
+
+ def test_structural_detection(self, detector):
+ """Test structural detection via 'Label: value' patterns."""
+ # New scalable approach: detect via structural "Today: value" pattern
+ spans = detector.detect("Date: 2024-01-15")
+ assert len(spans) == 1
+ assert spans[0].text == "2024-01-15"
+ assert spans[0].category == DynamicCategory.DATE
+
+ # Test user label detection
+ spans = detector.detect("User: [email protected]")
+ user_spans = [s for s in spans if s.category == DynamicCategory.USER_DATA]
+ assert len(user_spans) == 1
+
+ def test_datetime_iso(self, detector):
+ """Test ISO datetime detection."""
+ spans = detector.detect("Timestamp: 2024-01-15T10:30:00Z")
+ assert len(spans) == 1
+ assert spans[0].text == "2024-01-15T10:30:00Z"
+ assert spans[0].category == DynamicCategory.DATETIME
+
+ def test_uuid(self, detector):
+ """Test UUID detection."""
+ spans = detector.detect("ID: 550e8400-e29b-41d4-a716-446655440000")
+ assert len(spans) == 1
+ assert spans[0].text == "550e8400-e29b-41d4-a716-446655440000"
+ assert spans[0].category == DynamicCategory.UUID
+
+ def test_request_id(self, detector):
+ """Test request ID detection."""
+ spans = detector.detect("Request: req_abc123def456ghi789")
+ assert len(spans) == 1
+ assert "req_" in spans[0].text
+ assert spans[0].category == DynamicCategory.REQUEST_ID
+
+ def test_unix_timestamp(self, detector):
+ """Test Unix timestamp detection."""
+ spans = detector.detect("Time: 1705312200")
+ assert len(spans) == 1
+ assert spans[0].text == "1705312200"
+ assert spans[0].category == DynamicCategory.TIMESTAMP
+
+ def test_time(self, detector):
+ """Test time detection."""
+ spans = detector.detect("Meeting at 10:30 AM")
+ assert len(spans) == 1
+ assert spans[0].text == "10:30 AM"
+ assert spans[0].category == DynamicCategory.TIME
+
+ def test_version(self, detector):
+ """Test version number detection."""
+ spans = detector.detect("Running v2.3.1-beta")
+ assert len(spans) == 1
+ assert spans[0].text == "v2.3.1-beta"
+ assert spans[0].category == DynamicCategory.VERSION
+
+ def test_date_prefix_pattern(self, detector):
+ """Test full date prefix phrase detection."""
+ spans = detector.detect("Today is Monday, January 15, 2024. You are an assistant.")
+ assert len(spans) >= 1
+ # Should detect the full phrase
+ date_spans = [s for s in spans if s.category == DynamicCategory.DATE]
+ assert len(date_spans) >= 1
+
+ def test_multiple_dynamic_elements(self, detector):
+ """Test detecting multiple dynamic elements."""
+ content = """
+ Date: 2024-01-15
+ Time: 10:30:00
+ Request ID: req_abc123def456ghi789xyz
+ UUID: 550e8400-e29b-41d4-a716-446655440000
+ """
+ spans = detector.detect(content)
+ assert len(spans) == 4
+ categories = {s.category for s in spans}
+ assert DynamicCategory.DATE in categories
+ assert DynamicCategory.TIME in categories
+ assert DynamicCategory.REQUEST_ID in categories
+ assert DynamicCategory.UUID in categories
+
+ def test_no_false_positives_on_static(self, detector):
+ """Test that static content doesn't trigger false positives."""
+ spans = detector.detect("You are a helpful assistant. Answer questions clearly.")
+ assert len(spans) == 0
+
+ def test_positions_are_correct(self, detector):
+ """Test that span positions are correct."""
+ content = "Date: 2024-01-15"
+ spans = detector.detect(content)
+ assert len(spans) == 1
+ assert content[spans[0].start:spans[0].end] == spans[0].text
+
+
+class TestDynamicContentDetector:
+ """Test the unified dynamic content detector."""
+
+ def test_regex_only(self):
+ """Test detector with regex tier only."""
+ config = DetectorConfig(tiers=["regex"])
+ detector = DynamicContentDetector(config)
+
+ result = detector.detect("Today is 2024-01-15. You are helpful.")
+
+ assert len(result.spans) == 1
+ assert result.spans[0].text == "2024-01-15"
+ assert "regex" in result.tiers_used
+ assert result.processing_time_ms < 10 # Should be very fast
+
+ def test_static_dynamic_split(self):
+ """Test that content is properly split."""
+ config = DetectorConfig(tiers=["regex"])
+ detector = DynamicContentDetector(config)
+
+ result = detector.detect("Today is 2024-01-15. You are helpful.")
+
+ assert "2024-01-15" not in result.static_content
+ assert "2024-01-15" in result.dynamic_content
+ assert "You are helpful" in result.static_content
+
+ def test_complex_content(self):
+ """Test with realistic system prompt."""
+ config = DetectorConfig(tiers=["regex"])
+ detector = DynamicContentDetector(config)
+
+ content = """You are a helpful AI assistant.
+Today is January 15, 2024.
+Current session: sess_abc123def456ghi789xyz
+
+Instructions:
+1. Be concise
+2. Be accurate
+3. Be helpful
+
+Request ID: req_xyz789abc123def456ghi"""
+
+ result = detector.detect(content)
+
+ # Should find date, session ID, request ID
+ assert len(result.spans) >= 2
+ categories = {s.category for s in result.spans}
+ assert DynamicCategory.DATE in categories or DynamicCategory.REQUEST_ID in categories
+
+ def test_empty_content(self):
+ """Test with empty content."""
+ detector = DynamicContentDetector()
+ result = detector.detect("")
+
+ assert len(result.spans) == 0
+ assert result.static_content == ""
+ assert result.dynamic_content == ""
+
+ def test_no_dynamic_content(self):
+ """Test with fully static content."""
+ detector = DynamicContentDetector()
+ content = "You are a helpful assistant. Answer questions clearly and concisely."
+
+ result = detector.detect(content)
+
+ assert len(result.spans) == 0
+ assert result.static_content == content
+ assert result.dynamic_content == ""
+
+ def test_custom_patterns(self):
+ """Test adding custom regex patterns."""
+ config = DetectorConfig(
+ tiers=["regex"],
+ custom_patterns=[
+ (r"CUSTOM_\d{4}", DynamicCategory.REQUEST_ID),
+ ],
+ )
+ detector = DynamicContentDetector(config)
+
+ result = detector.detect("Code: CUSTOM_1234")
+
+ custom_spans = [s for s in result.spans if s.text == "CUSTOM_1234"]
+ assert len(custom_spans) == 1
+
+ def test_available_tiers(self):
+ """Test that available_tiers reflects actual availability."""
+ config = DetectorConfig(tiers=["regex", "ner", "semantic"])
+ detector = DynamicContentDetector(config)
+
+ # Regex should always be available
+ assert "regex" in detector.available_tiers
+
+ # NER and semantic depend on optional dependencies
+ # They may or may not be available
+
+ def test_warnings_for_missing_dependencies(self):
+ """Test that warnings are generated for missing dependencies."""
+ config = DetectorConfig(tiers=["regex", "ner", "semantic"])
+ detector = DynamicContentDetector(config)
+
+ result = detector.detect("Test content")
+
+ # If NER/semantic not installed, should have warnings
+ # (This test passes either way - it's informational)
+ # If deps ARE installed, no warnings. If not, warnings present.
+
+
+class TestConvenienceFunction:
+ """Test the detect_dynamic_content convenience function."""
+
+ def test_basic_usage(self):
+ """Test basic convenience function usage."""
+ result = detect_dynamic_content("Date: 2024-01-15")
+
+ assert isinstance(result, DetectionResult)
+ assert len(result.spans) == 1
+ assert result.spans[0].text == "2024-01-15"
+
+ def test_with_tiers(self):
+ """Test specifying tiers."""
+ result = detect_dynamic_content(
+ "Date: 2024-01-15",
+ tiers=["regex"],
+ )
+
+ assert "regex" in result.tiers_used
+
+
+class TestEntropyDetection:
+ """Test entropy-based detection for random IDs/tokens."""
+
+ def test_high_entropy_string(self):
+ """Test that high-entropy strings are detected."""
+ from headroom.cache.dynamic_detector import calculate_entropy
+
+ # High entropy strings (random-looking)
+ assert calculate_entropy("abc123xyz789def") > 0.7
+ assert calculate_entropy("550e8400e29b41d4") > 0.7
+
+ # Low entropy strings (repetitive)
+ assert calculate_entropy("aaaaaaaaaa") < 0.3
+ assert calculate_entropy("abababab") < 0.6
+
+ def test_entropy_detection_finds_ids(self):
+ """Test that entropy detection finds random IDs."""
+ detector = DynamicContentDetector()
+
+ # Random-looking ID that isn't covered by universal patterns
+ result = detector.detect("Auth: xK7mN2pQr9sT4vW")
+
+ # Should find the ID via entropy or structural detection
+ assert len(result.spans) >= 1
+
+ def test_entropy_skips_common_words(self):
+ """Test that common words aren't flagged as high-entropy."""
+ detector = DynamicContentDetector()
+
+ # These words have mixed case/numbers but aren't IDs
+ result = detector.detect("Use username and password correctly.")
+
+ # "username" and "password" shouldn't be detected
+ flagged_words = [s.text for s in result.spans]
+ assert "username" not in flagged_words
+ assert "password" not in flagged_words
+
+
+class TestEdgeCases:
+ """Test edge cases and tricky inputs."""
+
+ def test_overlapping_patterns(self):
+ """Test that overlapping patterns don't cause duplicates."""
+ detector = DynamicContentDetector()
+
+ # ISO datetime contains ISO date - shouldn't match both
+ result = detector.detect("Time: 2024-01-15T10:30:00Z")
+
+ # Should match datetime, not date separately
+ assert len(result.spans) == 1
+ assert result.spans[0].category == DynamicCategory.DATETIME
+
+ def test_adjacent_dynamic_content(self):
+ """Test adjacent dynamic elements."""
+ detector = DynamicContentDetector()
+
+ result = detector.detect("2024-01-15 10:30:00")
+
+ # Should find both date and time
+ assert len(result.spans) == 2
+
+ def test_very_long_content(self):
+ """Test with long content."""
+ detector = DynamicContentDetector()
+
+ # Create long content with some dynamic parts
+ static_parts = ["This is static text. "] * 100
+ content = "".join(static_parts) + "Date: 2024-01-15. " + "".join(static_parts)
+
+ result = detector.detect(content)
+
+ assert len(result.spans) == 1
+ assert result.processing_time_ms < 100 # Should still be fast
+
+ def test_special_characters(self):
+ """Test content with special characters."""
+ detector = DynamicContentDetector()
+
+ content = "Date: 2024-01-15\nUUID: 550e8400-e29b-41d4-a716-446655440000\n\n---\n"
+ result = detector.detect(content)
+
+ assert len(result.spans) == 2
+
+ def test_unicode_content(self):
+ """Test with Unicode content."""
+ detector = DynamicContentDetector()
+
+ content = "日期: 2024-01-15. Héllo wörld!"
+ result = detector.detect(content)
+
+ # Should still find the date
+ assert len(result.spans) == 1
+ assert result.spans[0].text == "2024-01-15"
+
+
+class TestCacheAlignmentScenarios:
+ """Test scenarios relevant to cache alignment."""
+
+ def test_system_prompt_dates(self):
+ """Test extracting dates from system prompts."""
+ detector = DynamicContentDetector()
+
+ content = """You are Claude, an AI assistant by Anthropic.
+Today is Monday, January 15, 2024.
+Current time: 10:30 AM PST.
+
+Your task is to help users with coding questions."""
+
+ result = detector.detect(content)
+
+ # Should extract date and time
+ assert len(result.spans) >= 1
+
+ # Static content should not have dates
+ assert "2024" not in result.static_content or "January" in result.static_content
+
+ # Dynamic content should have the dates
+ assert "January" in result.dynamic_content or "2024-01-15" in result.dynamic_content or "10:30" in result.dynamic_content
+
+ def test_request_metadata(self):
+ """Test extracting request metadata."""
+ detector = DynamicContentDetector()
+
+ content = """Request ID: req_abc123xyz789
+Trace ID: 550e8400-e29b-41d4-a716-446655440000
+Timestamp: 1705312200
+
+Process the following query:"""
+
+ result = detector.detect(content)
+
+ # Should find request ID, UUID, timestamp
+ categories = {s.category for s in result.spans}
+ assert len(result.spans) >= 2
+
+ def test_mixed_static_dynamic(self):
+ """Test content with interspersed static and dynamic parts."""
+ detector = DynamicContentDetector()
+
+ content = """You are helpful (static).
+Today is 2024-01-15 (dynamic).
+Always be accurate (static).
+Session: sess_abc123xyz789 (dynamic).
+Never lie (static)."""
+
+ result = detector.detect(content)
+
+ # Should find date and session ID
+ assert len(result.spans) >= 1
+
+ # Static content should preserve the static parts
+ assert "helpful" in result.static_content
+ assert "accurate" in result.static_content
+
+
+class TestNERDetector:
+ """Test Tier 2 NER detector (if spaCy available)."""
+
+ @pytest.fixture
+ def ner_detector(self):
+ """Create detector with NER enabled."""
+ from headroom.cache.dynamic_detector import NERDetector, _SPACY_AVAILABLE
+
+ if not _SPACY_AVAILABLE:
+ pytest.skip("spaCy not installed")
+
+ config = DetectorConfig(tiers=["ner"])
+ detector = NERDetector(config)
+
+ if not detector.is_available:
+ pytest.skip("spaCy model not available")
+
+ return detector
+
+ def test_person_detection(self, ner_detector):
+ """Test detecting person names."""
+ spans, _ = ner_detector.detect("John Smith sent the message.")
+
+ person_spans = [s for s in spans if s.category == DynamicCategory.PERSON]
+ # NER might or might not detect "John Smith" depending on model
+ # This is more of an integration test
+
+ def test_money_detection(self, ner_detector):
+ """Test detecting money amounts."""
+ spans, _ = ner_detector.detect("The total is $500.00")
+
+ money_spans = [s for s in spans if s.category == DynamicCategory.MONEY]
+ # May or may not detect depending on spaCy model
+
+
+class TestSemanticDetector:
+ """Test Tier 3 semantic detector (if sentence-transformers available)."""
+
+ @pytest.fixture
+ def semantic_detector(self):
+ """Create detector with semantic enabled."""
+ from headroom.cache.dynamic_detector import SemanticDetector, _SENTENCE_TRANSFORMERS_AVAILABLE
+
+ if not _SENTENCE_TRANSFORMERS_AVAILABLE:
+ pytest.skip("sentence-transformers not installed")
+
+ config = DetectorConfig(tiers=["semantic"])
+ detector = SemanticDetector(config)
+
+ if not detector.is_available:
+ pytest.skip("Embedding model not available")
+
+ return detector
+
+ def test_realtime_detection(self, semantic_detector):
+ """Test detecting real-time/volatile content."""
+ content = "The current stock price is updated every minute."
+ spans, _ = semantic_detector.detect(content)
+
+ # Should detect this as volatile/realtime
+ # Depends on similarity threshold
+
+
+class TestIntegrationWithAllTiers:
+ """Integration tests using all available tiers."""
+
+ def test_all_tiers_together(self):
+ """Test running all tiers on complex content."""
+ config = DetectorConfig(tiers=["regex", "ner", "semantic"])
+ detector = DynamicContentDetector(config)
+
+ content = """Today is January 15, 2024.
+John paid $500 for the service.
+Request ID: req_abc123xyz789.
+The stock price updates in real-time.
+Be helpful and accurate."""
+
+ result = detector.detect(content)
+
+ # Should find at least the regex matches
+ assert len(result.spans) >= 1
+
+ # Check processing time is reasonable
+ # NER + semantic might add 50-100ms
+ assert result.processing_time_ms < 5000 # Very generous timeout
+
+ # Should have used at least regex
+ assert "regex" in result.tiers_used
+
+ def test_tier_precedence(self):
+ """Test that earlier tiers take precedence."""
+ config = DetectorConfig(tiers=["regex", "ner"])
+ detector = DynamicContentDetector(config)
+
+ # Date should be caught by regex, not NER
+ result = detector.detect("Date: 2024-01-15")
+
+ assert len(result.spans) == 1
+ assert result.spans[0].tier == "regex"
diff --git a/tests/test_cache/test_google.py b/tests/test_cache/test_google.py
new file mode 100644
index 000000000..0e9e5489b
--- /dev/null
+++ b/tests/test_cache/test_google.py
@@ -0,0 +1,270 @@
+"""Tests for GoogleCacheOptimizer."""
+
+import pytest
+from datetime import datetime, timedelta
+from headroom.cache import GoogleCacheOptimizer, CacheConfig, OptimizationContext
+from headroom.cache.base import CacheStrategy
+from headroom.cache.google import (
+ GOOGLE_MIN_CACHE_TOKENS,
+ GOOGLE_CACHE_DISCOUNT,
+ CachedContentInfo,
+ CacheabilityAnalysis,
+)
+
+
+class TestGoogleCacheOptimizer:
+ """Test GoogleCacheOptimizer functionality."""
+
+ @pytest.fixture
+ def optimizer(self):
+ """Create optimizer instance."""
+ return GoogleCacheOptimizer()
+
+ @pytest.fixture
+ def context(self):
+ """Create optimization context."""
+ return OptimizationContext(
+ provider="google",
+ model="gemini-1.5-pro",
+ )
+
+ def test_optimizer_properties(self, optimizer):
+ """Test optimizer properties."""
+ assert optimizer.name == "google-cached-content"
+ assert optimizer.provider == "google"
+ assert optimizer.strategy == CacheStrategy.CACHED_CONTENT
+
+ def test_enforces_minimum_tokens(self):
+ """Test that Google's minimum is enforced."""
+ config = CacheConfig(min_cacheable_tokens=100)
+ optimizer = GoogleCacheOptimizer(config)
+ assert optimizer.config.min_cacheable_tokens >= GOOGLE_MIN_CACHE_TOKENS
+
+ def test_optimize_below_threshold(self, optimizer, context):
+ """Test optimization with content below threshold."""
+ messages = [
+ {"role": "system", "content": "Short system prompt."},
+ {"role": "user", "content": "Hello!"},
+ ]
+
+ result = optimizer.optimize(messages, context)
+
+ assert result.metrics.cacheable_tokens < GOOGLE_MIN_CACHE_TOKENS
+ assert any("32K" in w for w in result.warnings)
+
+ def test_optimize_above_threshold(self, optimizer, context):
+ """Test optimization with content above threshold."""
+ # Create content above 32K tokens
+ messages = [
+ {"role": "system", "content": "You are helpful. " * 15000},
+ {"role": "user", "content": "Hello!"},
+ ]
+
+ result = optimizer.optimize(messages, context)
+
+ assert result.metrics.cacheable_tokens > 0
+ if result.metrics.cacheable_tokens >= GOOGLE_MIN_CACHE_TOKENS:
+ assert result.metrics.estimated_savings_percent == GOOGLE_CACHE_DISCOUNT * 100
+
+ def test_analyze_cacheability(self, optimizer, context):
+ """Test cacheability analysis."""
+ messages = [
+ {"role": "system", "content": "Short"},
+ {"role": "user", "content": "Hello!"},
+ ]
+
+ analysis = optimizer.analyze_cacheability(messages, context)
+
+ assert isinstance(analysis, CacheabilityAnalysis)
+ assert analysis.total_tokens > 0
+ assert analysis.tokens_below_minimum > 0
+ assert analysis.is_cacheable is False
+ assert len(analysis.recommendations) > 0
+
+ def test_cache_registration(self, optimizer):
+ """Test registering a cache."""
+ cache_info = optimizer.register_cache(
+ cache_id="test-cache-123",
+ content_hash="abc123",
+ token_count=50000,
+ expires_at=datetime.now() + timedelta(hours=1),
+ )
+
+ assert cache_info.cache_id == "test-cache-123"
+ assert cache_info.content_hash == "abc123"
+ assert cache_info.token_count == 50000
+ assert not cache_info.is_expired
+ assert cache_info.ttl_remaining_seconds > 0
+
+ def test_cache_lookup(self, optimizer):
+ """Test looking up a registered cache."""
+ optimizer.register_cache(
+ cache_id="test-cache-456",
+ content_hash="def456",
+ token_count=50000,
+ expires_at=datetime.now() + timedelta(hours=1),
+ )
+
+ found = optimizer.get_reusable_cache("def456")
+ assert found is not None
+ assert found.cache_id == "test-cache-456"
+
+ def test_cache_lookup_expired(self, optimizer):
+ """Test that expired caches are not returned."""
+ optimizer.register_cache(
+ cache_id="expired-cache",
+ content_hash="expired123",
+ token_count=50000,
+ expires_at=datetime.now() - timedelta(hours=1),
+ )
+
+ found = optimizer.get_reusable_cache("expired123")
+ assert found is None
+
+ def test_cache_lookup_insufficient_ttl(self, optimizer):
+ """Test that caches with insufficient TTL are not returned."""
+ optimizer.register_cache(
+ cache_id="short-ttl-cache",
+ content_hash="shortttl123",
+ token_count=50000,
+ expires_at=datetime.now() + timedelta(seconds=30),
+ )
+
+ # Default min_ttl is 60 seconds
+ found = optimizer.get_reusable_cache("shortttl123")
+ assert found is None
+
+ def test_extend_cache_ttl(self, optimizer):
+ """Test extending cache TTL."""
+ optimizer.register_cache(
+ cache_id="extend-cache",
+ content_hash="extend123",
+ token_count=50000,
+ expires_at=datetime.now() + timedelta(hours=1),
+ )
+
+ new_expires = datetime.now() + timedelta(hours=2)
+ updated = optimizer.extend_cache_ttl("extend-cache", new_expires)
+
+ assert updated is not None
+ assert updated.expires_at == new_expires
+
+ def test_remove_cache(self, optimizer):
+ """Test removing a cache."""
+ optimizer.register_cache(
+ cache_id="remove-cache",
+ content_hash="remove123",
+ token_count=50000,
+ expires_at=datetime.now() + timedelta(hours=1),
+ )
+
+ removed = optimizer.remove_cache("remove-cache")
+ assert removed is True
+
+ found = optimizer.get_reusable_cache("remove123")
+ assert found is None
+
+ def test_cleanup_expired_caches(self, optimizer):
+ """Test cleaning up expired caches."""
+ # Register expired cache
+ optimizer.register_cache(
+ cache_id="cleanup-expired",
+ content_hash="cleanup123",
+ token_count=50000,
+ expires_at=datetime.now() - timedelta(hours=1),
+ )
+
+ expired_ids = optimizer.cleanup_expired_caches()
+ assert "cleanup-expired" in expired_ids
+
+ def test_list_caches(self, optimizer):
+ """Test listing caches."""
+ optimizer.register_cache(
+ cache_id="list-cache-1",
+ content_hash="list1",
+ token_count=50000,
+ expires_at=datetime.now() + timedelta(hours=1),
+ )
+ optimizer.register_cache(
+ cache_id="list-cache-2",
+ content_hash="list2",
+ token_count=50000,
+ expires_at=datetime.now() + timedelta(hours=2),
+ )
+
+ caches = optimizer.list_caches()
+ assert len(caches) >= 2
+
+ def test_get_statistics(self, optimizer):
+ """Test getting statistics."""
+ optimizer.register_cache(
+ cache_id="stats-cache",
+ content_hash="stats123",
+ token_count=50000,
+ expires_at=datetime.now() + timedelta(hours=1),
+ )
+
+ stats = optimizer.get_statistics()
+ assert "active_caches" in stats
+ assert "caches_created" in stats
+ assert stats["caches_created"] >= 1
+
+ def test_prepare_cache_creation(self, optimizer, context):
+ """Test preparing cache creation parameters."""
+ messages = [
+ {"role": "system", "content": "You are helpful. " * 15000},
+ {"role": "user", "content": "Hello!"},
+ ]
+
+ params = optimizer.prepare_cache_creation(messages, context)
+
+ if params is not None:
+ assert "contents" in params
+ assert "ttl" in params
+ assert "display_name" in params
+
+ def test_build_request_with_cache(self, optimizer):
+ """Test building request with cache."""
+ messages = [
+ {"role": "system", "content": "System prompt"},
+ {"role": "user", "content": "User message"},
+ ]
+
+ request = optimizer.build_request_with_cache(messages, "cache-123")
+
+ assert "cached_content" in request
+ assert request["cached_content"] == "cache-123"
+ assert "contents" in request
+
+ def test_export_import_registry(self, optimizer):
+ """Test exporting and importing cache registry."""
+ optimizer.register_cache(
+ cache_id="export-cache",
+ content_hash="export123",
+ token_count=50000,
+ expires_at=datetime.now() + timedelta(hours=1),
+ )
+
+ exported = optimizer.export_cache_registry()
+ assert len(exported) >= 1
+
+ new_optimizer = GoogleCacheOptimizer()
+ imported = new_optimizer.import_cache_registry(exported)
+ assert imported >= 1
+
+ def test_cached_content_info_serialization(self):
+ """Test CachedContentInfo serialization."""
+ info = CachedContentInfo(
+ cache_id="test",
+ content_hash="hash",
+ created_at=datetime.now(),
+ expires_at=datetime.now() + timedelta(hours=1),
+ token_count=50000,
+ )
+
+ data = info.to_dict()
+ restored = CachedContentInfo.from_dict(data)
+
+ assert restored.cache_id == info.cache_id
+ assert restored.content_hash == info.content_hash
+ assert restored.token_count == info.token_count
diff --git a/tests/test_cache/test_openai.py b/tests/test_cache/test_openai.py
new file mode 100644
index 000000000..7e26b646e
--- /dev/null
+++ b/tests/test_cache/test_openai.py
@@ -0,0 +1,196 @@
+"""Tests for OpenAICacheOptimizer."""
+
+import pytest
+from headroom.cache import OpenAICacheOptimizer, CacheConfig, OptimizationContext
+from headroom.cache.base import CacheStrategy
+
+
+class TestOpenAICacheOptimizer:
+ """Test OpenAICacheOptimizer functionality."""
+
+ @pytest.fixture
+ def optimizer(self):
+ """Create optimizer instance."""
+ return OpenAICacheOptimizer()
+
+ @pytest.fixture
+ def context(self):
+ """Create optimization context."""
+ return OptimizationContext(
+ provider="openai",
+ model="gpt-4",
+ )
+
+ def test_optimizer_properties(self, optimizer):
+ """Test optimizer properties."""
+ assert optimizer.name == "openai-prefix-stabilizer"
+ assert optimizer.provider == "openai"
+ assert optimizer.strategy == CacheStrategy.PREFIX_STABILIZATION
+
+ def test_optimize_simple_messages(self, optimizer, context):
+ """Test optimizing simple messages."""
+ messages = [
+ {"role": "system", "content": "You are a helpful assistant."},
+ {"role": "user", "content": "Hello!"},
+ ]
+
+ result = optimizer.optimize(messages, context)
+
+ assert result.messages is not None
+ assert len(result.messages) == 2
+ assert result.metrics.stable_prefix_hash != ""
+
+ def test_date_extraction(self, optimizer, context):
+ """Test that dates are extracted from system prompt."""
+ messages = [
+ {
+ "role": "system",
+ "content": "Today is January 7, 2026. You are a helpful assistant.",
+ },
+ {"role": "user", "content": "Hello!"},
+ ]
+
+ result = optimizer.optimize(messages, context)
+
+ # Check that date was extracted and moved
+ system_content = result.messages[0]["content"]
+ # The date should be moved to a dynamic section at the end
+ assert "You are a helpful assistant" in system_content
+
+ def test_whitespace_normalization(self, optimizer, context):
+ """Test whitespace normalization."""
+ messages = [
+ {
+ "role": "system",
+ "content": "You are a helpful assistant.",
+ },
+ {"role": "user", "content": "Hello!"},
+ ]
+
+ result = optimizer.optimize(messages, context)
+
+ # Whitespace should be normalized
+ system_content = result.messages[0]["content"]
+ assert " " not in system_content # Multiple spaces collapsed
+
+ def test_optimize_disabled(self, context):
+ """Test optimization when disabled."""
+ config = CacheConfig(enabled=False)
+ optimizer = OpenAICacheOptimizer(config)
+
+ messages = [
+ {"role": "system", "content": "Test"},
+ {"role": "user", "content": "Hello!"},
+ ]
+
+ result = optimizer.optimize(messages, context)
+ assert result.transforms_applied == []
+
+ def test_prefix_stability_tracking(self, optimizer, context):
+ """Test that prefix stability is tracked."""
+ messages = [
+ {"role": "system", "content": "You are a helpful assistant."},
+ {"role": "user", "content": "Hello!"},
+ ]
+
+ # First call
+ result1 = optimizer.optimize(messages, context)
+
+ # Second call with same messages
+ result2 = optimizer.optimize(messages, context)
+
+ # Second call should detect stable prefix
+ assert result2.metrics.estimated_cache_hit is True
+ assert result2.metrics.prefix_changed_from_previous is False
+
+ def test_prefix_change_detection(self, optimizer, context):
+ """Test detection of prefix changes."""
+ messages1 = [
+ {"role": "system", "content": "You are a helpful assistant."},
+ {"role": "user", "content": "Hello!"},
+ ]
+ messages2 = [
+ {"role": "system", "content": "You are a different assistant."},
+ {"role": "user", "content": "Hello!"},
+ ]
+
+ result1 = optimizer.optimize(messages1, context)
+ result2 = optimizer.optimize(messages2, context)
+
+ # Second call should detect prefix change
+ assert result2.metrics.prefix_changed_from_previous is True
+
+ def test_token_threshold_warning(self, optimizer, context):
+ """Test warning when below token threshold."""
+ messages = [
+ {"role": "system", "content": "Short."},
+ {"role": "user", "content": "Hi"},
+ ]
+
+ result = optimizer.optimize(messages, context)
+
+ # Should have warning about being below threshold
+ assert any("1024" in w for w in result.warnings)
+
+ def test_estimate_savings_below_threshold(self, optimizer, context):
+ """Test savings estimation below threshold."""
+ messages = [
+ {"role": "system", "content": "Short system prompt."},
+ {"role": "user", "content": "Hello!"},
+ ]
+
+ savings = optimizer.estimate_savings(messages, context)
+ assert savings == 0.0 # Below threshold
+
+ def test_estimate_savings_above_threshold(self, optimizer, context):
+ """Test savings estimation above threshold."""
+ messages = [
+ {"role": "system", "content": "You are helpful. " * 500},
+ {"role": "user", "content": "Hello!"},
+ ]
+
+ # First call to establish baseline
+ optimizer.optimize(messages, context)
+
+ # Second call should show savings
+ savings = optimizer.estimate_savings(messages, context)
+ assert savings > 0.0
+
+ def test_uuid_pattern_detection(self, optimizer, context):
+ """Test detection of UUIDs in content."""
+ messages = [
+ {
+ "role": "system",
+ "content": "Request ID: 12345678-1234-1234-1234-123456789012. Be helpful.",
+ },
+ {"role": "user", "content": "Hello!"},
+ ]
+
+ result = optimizer.optimize(messages, context)
+ # UUID should be detected as dynamic content
+ assert result.metrics.stable_prefix_hash != ""
+
+ def test_content_block_format(self, optimizer, context):
+ """Test handling of content block format."""
+ messages = [
+ {
+ "role": "system",
+ "content": [{"type": "text", "text": "You are helpful."}],
+ },
+ {"role": "user", "content": "Hello!"},
+ ]
+
+ result = optimizer.optimize(messages, context)
+ assert result.messages is not None
+
+ def test_metrics_recording(self, optimizer, context):
+ """Test that metrics are recorded."""
+ messages = [
+ {"role": "system", "content": "You are helpful."},
+ {"role": "user", "content": "Hello!"},
+ ]
+
+ optimizer.optimize(messages, context)
+ metrics = optimizer.get_metrics()
+
+ assert metrics.stable_prefix_hash != ""
diff --git a/tests/test_cache/test_registry.py b/tests/test_cache/test_registry.py
new file mode 100644
index 000000000..c383e325a
--- /dev/null
+++ b/tests/test_cache/test_registry.py
@@ -0,0 +1,135 @@
+"""Tests for CacheOptimizerRegistry."""
+
+import pytest
+from headroom.cache import (
+ CacheOptimizerRegistry,
+ AnthropicCacheOptimizer,
+ OpenAICacheOptimizer,
+ GoogleCacheOptimizer,
+ CacheConfig,
+)
+from headroom.cache.base import BaseCacheOptimizer, CacheStrategy, CacheResult, OptimizationContext
+
+
+class MockOptimizer(BaseCacheOptimizer):
+ """Mock optimizer for testing."""
+
+ @property
+ def name(self) -> str:
+ return "mock-optimizer"
+
+ @property
+ def provider(self) -> str:
+ return "mock"
+
+ @property
+ def strategy(self) -> CacheStrategy:
+ return CacheStrategy.NONE
+
+ def optimize(self, messages, context, config=None):
+ return CacheResult(messages=messages)
+
+
+class TestCacheOptimizerRegistry:
+ """Test CacheOptimizerRegistry functionality."""
+
+ def test_default_providers_registered(self):
+ """Test that default providers are registered on import."""
+ providers = CacheOptimizerRegistry.list_all()
+ assert "anthropic" in providers
+ assert "openai" in providers
+ assert "google" in providers
+
+ def test_get_anthropic(self):
+ """Test getting Anthropic optimizer."""
+ optimizer = CacheOptimizerRegistry.get("anthropic")
+ assert isinstance(optimizer, AnthropicCacheOptimizer)
+ assert optimizer.provider == "anthropic"
+ assert optimizer.strategy == CacheStrategy.EXPLICIT_BREAKPOINTS
+
+ def test_get_openai(self):
+ """Test getting OpenAI optimizer."""
+ optimizer = CacheOptimizerRegistry.get("openai")
+ assert isinstance(optimizer, OpenAICacheOptimizer)
+ assert optimizer.provider == "openai"
+ assert optimizer.strategy == CacheStrategy.PREFIX_STABILIZATION
+
+ def test_get_google(self):
+ """Test getting Google optimizer."""
+ optimizer = CacheOptimizerRegistry.get("google")
+ assert isinstance(optimizer, GoogleCacheOptimizer)
+ assert optimizer.provider == "google"
+ assert optimizer.strategy == CacheStrategy.CACHED_CONTENT
+
+ def test_get_with_config(self):
+ """Test getting optimizer with custom config."""
+ config = CacheConfig(min_cacheable_tokens=2048)
+ optimizer = CacheOptimizerRegistry.get("anthropic", config=config, cached=False)
+ assert optimizer.config.min_cacheable_tokens >= 1024 # Anthropic enforces minimum
+
+ def test_register_custom_optimizer(self):
+ """Test registering a custom optimizer."""
+ CacheOptimizerRegistry.register("mock", MockOptimizer)
+ try:
+ optimizer = CacheOptimizerRegistry.get("mock")
+ assert isinstance(optimizer, MockOptimizer)
+ finally:
+ CacheOptimizerRegistry.unregister("mock")
+
+ def test_register_duplicate_raises(self):
+ """Test that registering duplicate without override raises."""
+ CacheOptimizerRegistry.register("test-dup", MockOptimizer)
+ try:
+ with pytest.raises(ValueError):
+ CacheOptimizerRegistry.register("test-dup", MockOptimizer)
+ finally:
+ CacheOptimizerRegistry.unregister("test-dup")
+
+ def test_register_with_override(self):
+ """Test registering with override."""
+ CacheOptimizerRegistry.register("test-override", MockOptimizer)
+ try:
+ CacheOptimizerRegistry.register("test-override", MockOptimizer, override=True)
+ optimizer = CacheOptimizerRegistry.get("test-override")
+ assert isinstance(optimizer, MockOptimizer)
+ finally:
+ CacheOptimizerRegistry.unregister("test-override")
+
+ def test_get_unknown_provider_raises(self):
+ """Test getting unknown provider raises KeyError."""
+ with pytest.raises(KeyError):
+ CacheOptimizerRegistry.get("unknown-provider")
+
+ def test_list_providers(self):
+ """Test listing providers."""
+ providers = CacheOptimizerRegistry.list_providers()
+ assert "anthropic" in providers
+ assert "openai" in providers
+ assert "google" in providers
+
+ def test_is_registered(self):
+ """Test is_registered check."""
+ assert CacheOptimizerRegistry.is_registered("anthropic")
+ assert not CacheOptimizerRegistry.is_registered("nonexistent")
+
+ def test_cached_instances(self):
+ """Test that cached instances are reused."""
+ opt1 = CacheOptimizerRegistry.get("anthropic", cached=True)
+ opt2 = CacheOptimizerRegistry.get("anthropic", cached=True)
+ assert opt1 is opt2
+
+ def test_uncached_instances(self):
+ """Test that uncached instances are not reused."""
+ opt1 = CacheOptimizerRegistry.get("anthropic", cached=False)
+ opt2 = CacheOptimizerRegistry.get("anthropic", cached=False)
+ assert opt1 is not opt2
+
+ def test_tier_based_selection(self):
+ """Test tier-based optimizer selection."""
+ # OSS tier should work
+ oss_opt = CacheOptimizerRegistry.get("anthropic", tier="oss")
+ assert oss_opt is not None
+
+ # Enterprise tier falls back to OSS if not registered
+ ent_opt = CacheOptimizerRegistry.get("anthropic", tier="enterprise")
+ assert ent_opt is not None
diff --git a/tests/test_cache/test_semantic.py b/tests/test_cache/test_semantic.py
new file mode 100644
index 000000000..3cda666bd
--- /dev/null
+++ b/tests/test_cache/test_semantic.py
@@ -0,0 +1,297 @@
+"""Tests for SemanticCache and SemanticCacheLayer."""
+
+import pytest
+import time
+from headroom.cache import (
+ SemanticCacheLayer,
+ SemanticCache,
+ AnthropicCacheOptimizer,
+ OptimizationContext,
+)
+from headroom.cache.semantic import SemanticCacheConfig, CacheEntry
+
+
+class TestSemanticCacheConfig:
+ """Test SemanticCacheConfig."""
+
+ def test_default_values(self):
+ """Test default configuration values."""
+ config = SemanticCacheConfig()
+ assert config.similarity_threshold == 0.95
+ assert config.max_entries == 1000
+ assert config.ttl_seconds == 300
+ assert config.use_exact_matching is True
+
+
+class TestSemanticCache:
+ """Test SemanticCache functionality."""
+
+ @pytest.fixture
+ def cache(self):
+ """Create cache instance."""
+ config = SemanticCacheConfig(
+ max_entries=10,
+ ttl_seconds=60,
+ )
+ return SemanticCache(config)
+
+ def test_put_and_get_exact_match(self, cache):
+ """Test storing and retrieving with exact hash matching."""
+ response = {"text": "Hello, how can I help?"}
+ cache.put("What is the weather?", response, messages_hash="hash123")
+
+ entry = cache.get("What is the weather?", messages_hash="hash123")
+ assert entry is not None
+ assert entry.response == response
+
+ def test_get_miss(self, cache):
+ """Test cache miss."""
+ entry = cache.get("Unknown query", messages_hash="unknown")
+ assert entry is None
+
+ def test_lru_eviction(self):
+ """Test LRU eviction when at capacity."""
+ config = SemanticCacheConfig(max_entries=3)
+ cache = SemanticCache(config)
+
+ # Fill cache
+ cache.put("query1", "response1", messages_hash="h1")
+ cache.put("query2", "response2", messages_hash="h2")
+ cache.put("query3", "response3", messages_hash="h3")
+
+ # Access query1 to make it recently used
+ cache.get("query1", messages_hash="h1")
+
+ # Add new entry, should evict query2 (oldest unused)
+ cache.put("query4", "response4", messages_hash="h4")
+
+ # query1 should still be there (recently accessed)
+ assert cache.get("query1", messages_hash="h1") is not None
+ # query2 should be evicted
+ assert cache.get("query2", messages_hash="h2") is None
+ # query3 and query4 should be there
+ assert cache.get("query3", messages_hash="h3") is not None
+ assert cache.get("query4", messages_hash="h4") is not None
+
+ def test_ttl_expiration(self):
+ """Test TTL expiration."""
+ config = SemanticCacheConfig(ttl_seconds=1)
+ cache = SemanticCache(config)
+
+ cache.put("expiring query", "response", messages_hash="exp1")
+
+ # Should be available immediately
+ assert cache.get("expiring query", messages_hash="exp1") is not None
+
+ # Wait for TTL
+ time.sleep(1.1)
+
+ # Should be expired
+ assert cache.get("expiring query", messages_hash="exp1") is None
+
+ def test_invalidate(self, cache):
+ """Test invalidating an entry."""
+ key = cache.put("query", "response", messages_hash="inv1")
+
+ assert cache.get("query", messages_hash="inv1") is not None
+
+ cache.invalidate(key)
+
+ assert cache.get("query", messages_hash="inv1") is None
+
+ def test_clear(self, cache):
+ """Test clearing cache."""
+ cache.put("query1", "response1", messages_hash="c1")
+ cache.put("query2", "response2", messages_hash="c2")
+
+ cache.clear()
+
+ stats = cache.get_stats()
+ assert stats["entries"] == 0
+
+ def test_stats(self, cache):
+ """Test statistics."""
+ cache.put("query", "response", messages_hash="s1")
+ cache.get("query", messages_hash="s1") # hit
+ cache.get("unknown", messages_hash="unknown") # miss
+
+ stats = cache.get_stats()
+
+ assert stats["entries"] == 1
+ assert stats["hits"] == 1
+ assert stats["misses"] == 1
+ assert stats["hit_rate"] == 0.5
+
+ def test_access_count(self, cache):
+ """Test that access count is tracked."""
+ cache.put("query", "response", messages_hash="ac1")
+
+ # Access multiple times
+ for _ in range(5):
+ entry = cache.get("query", messages_hash="ac1")
+
+ # Initial count is 1, plus 5 accesses = 6
+ assert entry.access_count == 6
+
+ def test_semantic_similarity_with_embedding_fn(self):
+ """Test semantic similarity with custom embedding function."""
+
+ def mock_embedding(text: str) -> list[float]:
+ # Simple mock: return consistent embedding for similar queries
+ if "weather" in text.lower():
+ return [1.0, 0.0, 0.0]
+ elif "time" in text.lower():
+ return [0.0, 1.0, 0.0]
+ else:
+ return [0.0, 0.0, 1.0]
+
+ config = SemanticCacheConfig(similarity_threshold=0.9)
+ cache = SemanticCache(config, embedding_fn=mock_embedding)
+
+ # Store a weather query
+ cache.put("What is the weather today?", "It's sunny", messages_hash="w1")
+
+ # Similar weather query should hit
+ entry = cache.get("How is the weather?")
+ assert entry is not None
+ assert entry.response == "It's sunny"
+
+ # Different query should miss
+ entry = cache.get("What time is it?")
+ assert entry is None
+
+
+class TestSemanticCacheLayer:
+ """Test SemanticCacheLayer functionality."""
+
+ @pytest.fixture
+ def layer(self):
+ """Create cache layer with Anthropic optimizer."""
+ optimizer = AnthropicCacheOptimizer()
+ return SemanticCacheLayer(
+ optimizer,
+ similarity_threshold=0.95,
+ max_entries=100,
+ ttl_seconds=60,
+ )
+
+ @pytest.fixture
+ def context(self):
+ """Create optimization context."""
+ return OptimizationContext(
+ provider="anthropic",
+ model="claude-3-opus",
+ )
+
+ def test_process_no_cache_hit(self, layer, context):
+ """Test processing with no cache hit."""
+ messages = [
+ {"role": "system", "content": "You are helpful."},
+ {"role": "user", "content": "Hello!"},
+ ]
+
+ result = layer.process(messages, context)
+
+ assert result.semantic_cache_hit is False
+ assert result.cached_response is None
+
+ def test_process_with_cache_hit(self, layer, context):
+ """Test processing with cache hit."""
+ messages = [
+ {"role": "system", "content": "You are helpful."},
+ {"role": "user", "content": "What is 2+2?"},
+ ]
+
+ # First, store a response
+ layer.store_response(messages, {"text": "4"}, context)
+
+ # Now process same messages
+ result = layer.process(messages, context)
+
+ assert result.semantic_cache_hit is True
+ assert result.cached_response == {"text": "4"}
+
+ def test_store_response(self, layer, context):
+ """Test storing a response."""
+ messages = [
+ {"role": "system", "content": "You are helpful."},
+ {"role": "user", "content": "Tell me a joke"},
+ ]
+
+ key = layer.store_response(messages, {"text": "Why did..."}, context)
+
+ assert key is not None
+ assert len(key) > 0
+
+ def test_get_stats(self, layer, context):
+ """Test getting statistics."""
+ messages = [
+ {"role": "system", "content": "You are helpful."},
+ {"role": "user", "content": "Hello"},
+ ]
+
+ layer.process(messages, context)
+ stats = layer.get_stats()
+
+ assert "semantic_cache" in stats
+ assert "provider_optimizer" in stats
+ assert stats["provider_optimizer"] == "anthropic-cache-optimizer"
+
+ def test_query_extraction(self, layer, context):
+ """Test query extraction from messages."""
+ messages = [
+ {"role": "system", "content": "System"},
+ {"role": "user", "content": "First question"},
+ {"role": "assistant", "content": "Answer"},
+ {"role": "user", "content": "Second question"},
+ ]
+
+ # Store response
+ layer.store_response(messages, {"text": "Response"}, context)
+
+ # The query should be the last user message
+ result = layer.process(messages, context)
+ assert result.semantic_cache_hit is True
+
+ def test_query_from_context(self, layer):
+ """Test using query from context."""
+ messages = [
+ {"role": "user", "content": "Some message"},
+ ]
+ context = OptimizationContext(
+ query="Specific query for caching",
+ )
+
+ layer.store_response(messages, {"text": "Response"}, context)
+ result = layer.process(messages, context)
+
+ assert result.semantic_cache_hit is True
+
+ def test_provider_optimizer_fallback(self, layer, context):
+ """Test that provider optimizer is used on cache miss."""
+ messages = [
+ {"role": "system", "content": "You are helpful. " * 500},
+ {"role": "user", "content": "New uncached question"},
+ ]
+
+ result = layer.process(messages, context)
+
+ # Should have used provider optimizer
+ assert result.semantic_cache_hit is False
+ # Provider optimizer should have processed
+ assert result.metrics.stable_prefix_hash != ""
+
+ def test_content_block_query_extraction(self, layer, context):
+ """Test query extraction from content block format."""
+ messages = [
+ {"role": "system", "content": "System"},
+ {
+ "role": "user",
+ "content": [{"type": "text", "text": "Block format question"}],
+ },
+ ]
+
+ layer.store_response(messages, {"text": "Response"}, context)
+ result = layer.process(messages, context)
+
+ assert result.semantic_cache_hit is True