Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
Add cache optimization module with scalable dynamic content detection
chopratejas7a05808e0f92
diff --git a/benchmarks/dynamic_detector_benchmark.py b/benchmarks/dynamic_detector_benchmark.pynew file mode 100644index 000000000..4a8d307e9--- /dev/null+++ b/benchmarks/dynamic_detector_benchmark.py@@ -0,0 +1,375 @@+#!/usr/bin/env python3+"""+Real-world benchmark for DynamicContentDetector.++Tests the detector against realistic system prompts from AI coding agents,+chatbots, and enterprise applications.+"""++import time+import statistics+from dataclasses import dataclass+from typing import Any++from headroom.cache.dynamic_detector import (+ DetectorConfig,+ DynamicContentDetector,+ DynamicCategory,+)+++@dataclass+class BenchmarkResult:+ """Result of a single benchmark run."""+ name: str+ content_length: int+ spans_found: int+ categories: list[str]+ static_length: int+ dynamic_length: int+ latency_ms: float+ tiers_used: list[str]+ warnings: list[str]+++# Real-world system prompts+REAL_WORLD_PROMPTS = {+ "claude_code_style": """You are Claude, an AI assistant created by Anthropic to be helpful, harmless, and honest.++Today is Tuesday, January 7, 2026.+Current time: 10:30:45 AM PST.++You are operating in a software development environment with access to:+- File system operations+- Terminal commands+- Web search++Session ID: sess_abc123def456ghi789jkl012+Request ID: req_xyz789abc123def456ghi789+User: tchopra+Workspace: /Users/tchopra/claude-projects/headroom++Be concise, accurate, and helpful. Follow the user's instructions carefully.""",++ "enterprise_assistant": """You are an enterprise AI assistant for Acme Corporation.++Current Date: 2026-01-07T10:30:00Z+Last Updated: 2026-01-07T09:00:00Z++User Profile:+- Name: John Smith+- Employee ID: EMP-2024-00542+- Department: Engineering+- Manager: Sarah Johnson+- Location: San Francisco, CA+- Hire Date: March 15, 2023++System Status:+- API Version: v2.3.1-beta+- Server Load: 45%+- Active Users: 1,247+- Queue Length: 23++Budget Information:+- Monthly Allowance: $5,000.00+- Used This Month: $2,341.67+- Remaining: $2,658.33++Help the user with their work tasks while following company policies.""",++ "coding_agent": """You are an autonomous coding agent with access to tools.++Environment:+- OS: macOS Darwin 25.1.0+- Working Directory: /Users/developer/projects/myapp+- Git Branch: feature/JIRA-1234-add-auth+- Last Commit: a1b2c3d4e5f6 (2 hours ago)+- Node Version: v20.10.0+- Python Version: 3.11.7++Current Task Context:+- Task ID: 550e8400-e29b-41d4-a716-446655440000+- Created: 2026-01-07T08:15:30Z+- Priority: High+- Estimated Time: 2 hours++API Keys Available:+- OPENAI_API_KEY: sk-proj-xxxxxxxxxxxxxxxxxxxxxxxxxxxx+- ANTHROPIC_API_KEY: sk-ant-xxxxxxxxxxxxxxxxxxxxxxxxxxxx+- DATABASE_URL: postgresql://user:pass@localhost:5432/mydb++Execute tasks step by step, verify each action, and report progress.""",++ "customer_support": """You are a customer support agent for TechStore Inc.++Current Time: January 7, 2026, 3:45 PM EST+Support Ticket: #TKT-2026-0107-4521++Customer Information:+- Name: Alice Chen+- Email: [email protected]+- Phone: (555) 123-4567+- Customer Since: August 2021+- Loyalty Tier: Gold+- Total Purchases: $12,456.78++Recent Orders:+- Order #ORD-2026-0105-7823 - iPhone 15 Pro - $1,199.00 - Delivered+- Order #ORD-2025-1220-3456 - AirPods Pro - $249.00 - Delivered+- Order #ORD-2025-1115-9012 - MacBook Air - $1,299.00 - Returned++Active Issues:+- Case #CS-2026-0107-001 - Battery drain issue - Open since today++Provide helpful, empathetic support while following company guidelines.""",++ "data_analysis": """You are a data analysis assistant.++Report Generated: 2026-01-07 10:30:00 UTC+Report ID: RPT-550e8400-e29b-41d4-a716-446655440000+Data Range: 2025-12-01 to 2025-12-31++Summary Statistics:+- Total Revenue: $1,234,567.89+- Total Orders: 45,678+- Average Order Value: $27.03+- Top Product: Widget Pro ($234,567.00)+- Top Region: California (23.4%)++Key Metrics:+- DAU: 125,000+- MAU: 890,000+- Churn Rate: 2.3%+- NPS Score: 67++Anomalies Detected:+- Spike on Dec 15: 3.2x normal traffic+- Drop on Dec 25: 0.4x normal (expected - holiday)++Help analyze the data and provide insights.""",++ "minimal_static": """You are a helpful AI assistant.++Your role is to:+1. Answer questions accurately+2. Be concise and clear+3. Follow instructions carefully+4. Admit when you don't know something++Always be helpful, harmless, and honest.""",++ "heavy_dynamic": """Session started at 2026-01-07T10:30:45.123Z+Request ID: req_abc123def456ghi789jkl012mno345pqr678+Trace ID: 550e8400-e29b-41d4-a716-446655440000+Parent Span: span_xyz789abc123+User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)+IP Address: 192.168.1.100+Geo: San Francisco, CA, USA (37.7749, -122.4194)++Auth Token: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...+Token Expires: 2026-01-07T11:30:45Z+Refresh Token: rt_abc123def456++Last Login: 2026-01-06T18:45:30Z+Login Count: 1,247+Account Balance: $5,432.10+Credit Limit: $10,000.00++Real-time Stock Prices (as of 10:30 AM):+- AAPL: $185.42 (+1.2%)+- GOOGL: $142.89 (-0.5%)+- MSFT: $378.23 (+0.8%)+- AMZN: $156.78 (+2.1%)++Process this request.""",+}+++def run_benchmark(+ prompts: dict[str, str],+ tiers: list[str],+ iterations: int = 10,+) -> dict[str, Any]:+ """Run benchmark on prompts with specified tiers."""++ config = DetectorConfig(tiers=tiers) # type: ignore+ detector = DynamicContentDetector(config)++ results: dict[str, list[BenchmarkResult]] = {}++ for name, content in prompts.items():+ results[name] = []++ for _ in range(iterations):+ start = time.perf_counter()+ result = detector.detect(content)+ elapsed = (time.perf_counter() - start) * 1000++ categories = list(set(s.category.value for s in result.spans))++ results[name].append(BenchmarkResult(+ name=name,+ content_length=len(content),+ spans_found=len(result.spans),+ categories=categories,+ static_length=len(result.static_content),+ dynamic_length=len(result.dynamic_content),+ latency_ms=elapsed,+ tiers_used=result.tiers_used,+ warnings=result.warnings,+ ))++ return results+++def print_results(+ results: dict[str, list[BenchmarkResult]],+ tier_name: str,+):+ """Print benchmark results."""++ print(f"\n{'='*80}")+ print(f"BENCHMARK RESULTS: {tier_name}")+ print(f"{'='*80}")++ for name, runs in results.items():+ latencies = [r.latency_ms for r in runs]+ avg_latency = statistics.mean(latencies)+ std_latency = statistics.stdev(latencies) if len(latencies) > 1 else 0++ # Use first run for span info (consistent across runs)+ first = runs[0]++ compression = (1 - first.static_length / first.content_length) * 100 if first.content_length > 0 else 0++ print(f"\n📄 {name}")+ print(f" Content: {first.content_length:,} chars")+ print(f" Spans found: {first.spans_found}")+ print(f" Categories: {', '.join(first.categories) if first.categories else 'none'}")+ print(f" Static: {first.static_length:,} chars | Dynamic: {first.dynamic_length:,} chars")+ print(f" Compression: {compression:.1f}% removed")+ print(f" Latency: {avg_latency:.2f}ms ± {std_latency:.2f}ms")+ print(f" Tiers used: {', '.join(first.tiers_used)}")+ if first.warnings:+ print(f" ⚠️ Warnings: {len(first.warnings)}")+++def print_comparison(all_results: dict[str, dict[str, list[BenchmarkResult]]]):+ """Print comparison across tiers."""++ print(f"\n{'='*80}")+ print("TIER COMPARISON")+ print(f"{'='*80}")++ prompts = list(REAL_WORLD_PROMPTS.keys())+ tiers = list(all_results.keys())++ # Header+ header = f"{'Prompt':<25}"+ for tier in tiers:+ header += f" | {tier:>12} spans | {'latency':>8}"+ print(header)+ print("-" * len(header))++ for prompt in prompts:+ row = f"{prompt:<25}"+ for tier in tiers:+ if prompt in all_results[tier]:+ runs = all_results[tier][prompt]+ spans = runs[0].spans_found+ latency = statistics.mean([r.latency_ms for r in runs])+ row += f" | {spans:>12} | {latency:>7.2f}ms"+ else:+ row += f" | {'N/A':>12} | {'N/A':>8}"+ print(row)++ # Summary+ print(f"\n{'='*80}")+ print("SUMMARY")+ print(f"{'='*80}")++ for tier in tiers:+ all_latencies = []+ total_spans = 0+ for runs in all_results[tier].values():+ all_latencies.extend([r.latency_ms for r in runs])+ total_spans += runs[0].spans_found++ avg = statistics.mean(all_latencies)+ p50 = statistics.median(all_latencies)+ p99 = sorted(all_latencies)[int(len(all_latencies) * 0.99)] if len(all_latencies) > 1 else avg++ print(f"\n{tier}:")+ print(f" Total spans detected: {total_spans}")+ print(f" Avg latency: {avg:.2f}ms")+ print(f" P50 latency: {p50:.2f}ms")+ print(f" P99 latency: {p99:.2f}ms")+++def show_detection_details(prompt_name: str, content: str):+ """Show detailed detection for a specific prompt."""++ print(f"\n{'='*80}")+ print(f"DETECTION DETAILS: {prompt_name}")+ print(f"{'='*80}")++ config = DetectorConfig(tiers=["regex"])+ detector = DynamicContentDetector(config)+ result = detector.detect(content)++ print(f"\nOriginal content ({len(content)} chars):")+ print("-" * 40)+ print(content[:500] + "..." if len(content) > 500 else content)++ print(f"\n\nDetected spans ({len(result.spans)}):")+ print("-" * 40)+ for span in result.spans:+ print(f" [{span.category.value:12}] '{span.text[:50]}{'...' if len(span.text) > 50 else ''}'")++ print(f"\n\nStatic content ({len(result.static_content)} chars):")+ print("-" * 40)+ print(result.static_content[:500] + "..." if len(result.static_content) > 500 else result.static_content)++ print(f"\n\nDynamic content ({len(result.dynamic_content)} chars):")+ print("-" * 40)+ print(result.dynamic_content if result.dynamic_content else "(none)")+++def main():+ """Run the benchmark."""++ print("🚀 Dynamic Content Detector - Real World Benchmark")+ print("=" * 80)++ iterations = 20++ # Test each tier configuration+ tier_configs = {+ "regex_only": ["regex"],+ # "regex+ner": ["regex", "ner"], # Uncomment if spacy installed+ # "all_tiers": ["regex", "ner", "semantic"], # Uncomment if all deps installed+ }++ all_results: dict[str, dict[str, list[BenchmarkResult]]] = {}++ for tier_name, tiers in tier_configs.items():+ print(f"\n⏱️ Running {tier_name} ({iterations} iterations per prompt)...")+ results = run_benchmark(REAL_WORLD_PROMPTS, tiers, iterations)+ all_results[tier_name] = results+ print_results(results, tier_name)++ # Print comparison if multiple tiers tested+ if len(all_results) > 1:+ print_comparison(all_results)++ # Show detailed detection for a few prompts+ print("\n" + "=" * 80)+ print("DETAILED DETECTION EXAMPLES")+ print("=" * 80)++ for name in ["claude_code_style", "enterprise_assistant", "heavy_dynamic"]:+ show_detection_details(name, REAL_WORLD_PROMPTS[name])+++if __name__ == "__main__":+ main()diff --git a/headroom/__init__.py b/headroom/__init__.pyindex fe02d8e5b..4868c09bf 100644--- a/headroom/__init__.py+++ b/headroom/__init__.py@@ -38,10 +38,25 @@ Example usage:print(f"Would save {plan.tokens_saved} tokens")"""+from .cache import (+ AnthropicCacheOptimizer,+ BaseCacheOptimizer,+ CacheConfig,+ CacheMetrics,+ CacheOptimizerRegistry,+ CacheResult,+ CacheStrategy,+ GoogleCacheOptimizer,+ OpenAICacheOptimizer,+ OptimizationContext,+ SemanticCache,+ SemanticCacheLayer,+)from .client import HeadroomClientfrom .config import (Block,CacheAlignerConfig,+ CacheOptimizerConfig,CachePrefixMetrics,DiffArtifact,HeadroomConfig,@@ -92,6 +107,7 @@ __all__ = ["ToolCrusherConfig","SmartCrusherConfig","CacheAlignerConfig",+ "CacheOptimizerConfig","RollingWindowConfig","RelevanceScorerConfig",# Data models@@ -109,6 +125,19 @@ __all__ = ["CacheAligner","RollingWindow","TransformPipeline",+ # Cache optimizers+ "BaseCacheOptimizer",+ "CacheConfig",+ "CacheMetrics",+ "CacheResult",+ "CacheStrategy",+ "OptimizationContext",+ "CacheOptimizerRegistry",+ "AnthropicCacheOptimizer",+ "OpenAICacheOptimizer",+ "GoogleCacheOptimizer",+ "SemanticCache",+ "SemanticCacheLayer",# Relevance scoring"RelevanceScore","RelevanceScorer",diff --git a/headroom/cache/__init__.py b/headroom/cache/__init__.pynew file mode 100644index 000000000..88f5998c9--- /dev/null+++ b/headroom/cache/__init__.py@@ -0,0 +1,76 @@+"""+Headroom Cache Optimization Module.++This module provides a plugin-based architecture for cache optimization+across different LLM providers. Each provider has different caching+mechanisms and this module abstracts those differences.++Provider Caching Differences:+- Anthropic: Explicit cache_control blocks, 90% savings, 5-min TTL+- OpenAI: Automatic prefix caching, 50% savings, no user control+- Google: Separate CachedContent API, 75% savings + storage costs++Usage:+ from headroom.cache import CacheOptimizerRegistry, SemanticCacheLayer++ # Get provider-specific optimizer+ optimizer = CacheOptimizerRegistry.get("anthropic")+ result = optimizer.optimize(messages, context)++ # With semantic caching layer+ semantic = SemanticCacheLayer(optimizer, similarity_threshold=0.95)+ result = semantic.process(messages, context)++ # Register custom optimizer+ CacheOptimizerRegistry.register("my-provider", MyOptimizer)+"""++from .base import (+ BaseCacheOptimizer,+ CacheBreakpoint,+ CacheConfig,+ CacheMetrics,+ CacheOptimizer,+ CacheResult,+ CacheStrategy,+ OptimizationContext,+)+from .dynamic_detector import (+ DetectorConfig,+ DynamicCategory,+ DynamicContentDetector,+ DynamicSpan,+ detect_dynamic_content,+)+from .registry import CacheOptimizerRegistry+from .anthropic import AnthropicCacheOptimizer+from .openai import OpenAICacheOptimizer+from .google import GoogleCacheOptimizer+from .semantic import SemanticCacheLayer, SemanticCache++__all__ = [+ # Base types+ "BaseCacheOptimizer",+ "CacheBreakpoint",+ "CacheConfig",+ "CacheMetrics",+ "CacheOptimizer",+ "CacheResult",+ "CacheStrategy",+ "OptimizationContext",+ # Dynamic content detection+ "DetectorConfig",+ "DynamicCategory",+ "DynamicContentDetector",+ "DynamicSpan",+ "detect_dynamic_content",+ # Registry+ "CacheOptimizerRegistry",+ # Provider implementations+ "AnthropicCacheOptimizer",+ "OpenAICacheOptimizer",+ "GoogleCacheOptimizer",+ # Semantic caching+ "SemanticCacheLayer",+ "SemanticCache",+]diff --git a/headroom/cache/anthropic.py b/headroom/cache/anthropic.pynew file mode 100644index 000000000..670ffa5b3--- /dev/null+++ b/headroom/cache/anthropic.py@@ -0,0 +1,518 @@+"""+Anthropic Cache Optimizer.++Implements cache optimization for Anthropic's explicit cache_control mechanism.+Anthropic uses ephemeral cache breakpoints to mark content that should be cached.++Anthropic Caching Characteristics:+- Explicit cache_control: {"type": "ephemeral"} blocks+- Minimum 1024 tokens for caching to be effective+- Maximum 4 cache breakpoints per request+- 5-minute TTL (extended on cache hit)+- Cost: 25% MORE to write to cache, 90% LESS to read++Usage:+ from headroom.cache import AnthropicCacheOptimizer, OptimizationContext++ optimizer = AnthropicCacheOptimizer()+ context = OptimizationContext(provider="anthropic", model="claude-3-opus")++ result = optimizer.optimize(messages, context)+ # result.messages now contains cache_control blocks+"""++from __future__ import annotations++import copy+import re+from dataclasses import dataclass, field+from typing import Any++from .base import (+ BaseCacheOptimizer,+ BreakpointLocation,+ CacheBreakpoint,+ CacheConfig,+ CacheMetrics,+ CacheResult,+ CacheStrategy,+ OptimizationContext,+)+++# Anthropic-specific constants+ANTHROPIC_MIN_CACHEABLE_TOKENS = 1024+ANTHROPIC_MAX_BREAKPOINTS = 4+ANTHROPIC_CACHE_TTL_SECONDS = 300 # 5 minutes+ANTHROPIC_WRITE_COST_MULTIPLIER = 1.25 # 25% more to write+ANTHROPIC_READ_COST_MULTIPLIER = 0.10 # 90% less to read+++@dataclass+class ContentSection:+ """Represents a section of content that may be cacheable."""++ content: str | list[dict[str, Any]]+ section_type: str # "system", "tools", "examples", "user", "assistant"+ message_index: int+ content_index: int | None = None+ token_count: int = 0+ is_cacheable: bool = False+ reason: str = ""+++@dataclass+class BreakpointPlan:+ """Plan for where to insert cache breakpoints."""++ breakpoints: list[CacheBreakpoint] = field(default_factory=list)+ total_cacheable_tokens: int = 0+ estimated_savings_percent: float = 0.0+ warnings: list[str] = field(default_factory=list)+++class AnthropicCacheOptimizer(BaseCacheOptimizer):+ """+ Cache optimizer for Anthropic's explicit cache_control mechanism.++ This optimizer analyzes messages and inserts cache_control blocks at+ optimal positions to maximize cache hit rates and minimize costs.++ Key features:+ - Detects cacheable sections (system prompt, tools, few-shot examples)+ - Respects Anthropic's 1024 token minimum and 4 breakpoint maximum+ - Stabilizes prefixes by moving dates and normalizing whitespace+ - Tracks metrics for monitoring and debugging+ """++ def __init__(self, config: CacheConfig | None = None):+ super().__init__(config)+ if self.config.min_cacheable_tokens < ANTHROPIC_MIN_CACHEABLE_TOKENS:+ self.config.min_cacheable_tokens = ANTHROPIC_MIN_CACHEABLE_TOKENS+ if self.config.max_breakpoints > ANTHROPIC_MAX_BREAKPOINTS:+ self.config.max_breakpoints = ANTHROPIC_MAX_BREAKPOINTS++ @property+ def name(self) -> str:+ return "anthropic-cache-optimizer"++ @property+ def provider(self) -> str:+ return "anthropic"++ @property+ def strategy(self) -> CacheStrategy:+ return CacheStrategy.EXPLICIT_BREAKPOINTS++ def optimize(+ self,+ messages: list[dict[str, Any]],+ context: OptimizationContext,+ config: CacheConfig | None = None,+ ) -> CacheResult:+ """+ Optimize messages for Anthropic's cache.++ Steps:+ 1. Analyze messages to identify cacheable sections+ 2. Stabilize the prefix (moves dates, normalizes whitespace)+ 3. Plan breakpoint placement+ 4. Insert cache_control blocks at optimal positions+ 5. Record metrics for monitoring+ """+ effective_config = config or self.config++ if not effective_config.enabled:+ return CacheResult(+ messages=messages,+ metrics=CacheMetrics(),+ transforms_applied=[],+ )++ optimized_messages = copy.deepcopy(messages)+ transforms_applied: list[str] = []+ warnings: list[str] = []++ # Step 1: Analyze content sections+ sections = self._analyze_sections(optimized_messages)++ # Step 2: Stabilize prefix+ optimized_messages, stabilization_applied = self._stabilize_prefix(+ optimized_messages, effective_config+ )+ transforms_applied.extend(stabilization_applied)++ # Step 3: Plan breakpoint placement+ plan = self._plan_breakpoints(sections, effective_config)+ warnings.extend(plan.warnings)++ # Step 4: Insert cache_control blocks+ optimized_messages = self._insert_breakpoints(+ optimized_messages, plan.breakpoints+ )+ if plan.breakpoints:+ transforms_applied.append(+ f"inserted_{len(plan.breakpoints)}_cache_breakpoints"+ )++ # Step 5: Compute metrics+ prefix_content = self._extract_cacheable_content(optimized_messages)+ prefix_hash = self._compute_prefix_hash(prefix_content)++ cache_hit = False+ if context.previous_prefix_hash:+ cache_hit = prefix_hash == context.previous_prefix_hash+ elif self._previous_prefix_hash:+ cache_hit = prefix_hash == self._previous_prefix_hash++ total_tokens = sum(s.token_count for s in sections)+ cacheable_tokens = plan.total_cacheable_tokens++ metrics = CacheMetrics(+ stable_prefix_tokens=cacheable_tokens,+ stable_prefix_hash=prefix_hash,+ breakpoints_inserted=len(plan.breakpoints),+ breakpoint_locations=plan.breakpoints,+ prefix_changed_from_previous=not cache_hit,+ previous_prefix_hash=self._previous_prefix_hash,+ estimated_cache_hit=cache_hit,+ estimated_savings_percent=plan.estimated_savings_percent if cache_hit else 0.0,+ cacheable_tokens=cacheable_tokens,+ non_cacheable_tokens=total_tokens - cacheable_tokens,+ cache_ttl_remaining_seconds=ANTHROPIC_CACHE_TTL_SECONDS if cache_hit else None,+ )++ self._previous_prefix_hash = prefix_hash+ self._record_metrics(metrics)++ return CacheResult(+ messages=optimized_messages,+ metrics=metrics,+ tokens_before=total_tokens,+ tokens_after=total_tokens,+ transforms_applied=transforms_applied,+ warnings=warnings,+ )++ def _analyze_sections(+ self, messages: list[dict[str, Any]]+ ) -> list[ContentSection]:+ """Analyze messages to identify distinct content sections."""+ sections: list[ContentSection] = []++ for idx, message in enumerate(messages):+ role = message.get("role", "")+ content = message.get("content", "")++ if role == "system":+ section_type = "system"+ elif role == "user":+ section_type = "examples" if self._looks_like_example(message, messages, idx) else "user"+ elif role == "assistant":+ section_type = "examples" if self._looks_like_example(message, messages, idx) else "assistant"+ else:+ section_type = role++ # Handle tools+ if "tools" in message:+ tool_section = ContentSection(+ content=message["tools"],+ section_type="tools",+ message_index=idx,+ token_count=self._estimate_tools_tokens(message["tools"]),+ is_cacheable=True,+ reason="Tool definitions are static and cacheable",+ )+ sections.append(tool_section)++ if isinstance(content, str):+ token_count = self._count_tokens_estimate(content)+ is_cacheable, reason = self._assess_cacheability(+ section_type, token_count, content+ )+ sections.append(ContentSection(+ content=content,+ section_type=section_type,+ message_index=idx,+ token_count=token_count,+ is_cacheable=is_cacheable,+ reason=reason,+ ))++ elif isinstance(content, list):+ for block_idx, block in enumerate(content):+ if isinstance(block, dict) and block.get("type") == "text":+ text = block.get("text", "")+ token_count = self._count_tokens_estimate(text)+ is_cacheable, reason = self._assess_cacheability(+ section_type, token_count, text+ )+ sections.append(ContentSection(+ content=block,+ section_type=section_type,+ message_index=idx,+ content_index=block_idx,+ token_count=token_count,+ is_cacheable=is_cacheable,+ reason=reason,+ ))++ return sections++ def _assess_cacheability(+ self, section_type: str, token_count: int, content: str+ ) -> tuple[bool, str]:+ """Assess whether a section is cacheable."""+ if token_count < self.config.min_cacheable_tokens:+ return False, f"Below minimum tokens ({token_count} < {self.config.min_cacheable_tokens})"++ if section_type == "system":+ return True, "System prompts are highly cacheable"+ if section_type == "tools":+ return True, "Tool definitions are static and cacheable"+ if section_type == "examples":+ return True, "Few-shot examples are typically static"+ if self._has_dynamic_content(content):+ return False, "Contains dynamic content (dates, times, etc.)"+ if section_type == "user":+ return False, "User messages are typically dynamic"++ return True, "Content is large enough for caching"++ def _has_dynamic_content(self, content: str) -> bool:+ """Check if content has dynamic elements."""+ for pattern in self.config.date_patterns:+ if re.search(pattern, content):+ return True+ return False++ def _looks_like_example(+ self,+ message: dict[str, Any],+ messages: list[dict[str, Any]],+ idx: int,+ ) -> bool:+ """Determine if a message looks like a few-shot example."""+ system_idx = -1+ for i, msg in enumerate(messages):+ if msg.get("role") == "system":+ system_idx = i+ break++ if system_idx >= 0 and idx <= system_idx + 4:+ role = message.get("role")+ if role == "user" and idx + 1 < len(messages):+ if messages[idx + 1].get("role") == "assistant":+ return True+ elif role == "assistant" and idx > 0:+ if messages[idx - 1].get("role") == "user":+ return True++ content = message.get("content", "")+ if isinstance(content, str):+ example_markers = ["example:", "for example", "e.g.", "sample:"]+ return any(marker in content.lower() for marker in example_markers)++ return False++ def _estimate_tools_tokens(self, tools: Any) -> int:+ """Estimate token count for tool definitions."""+ import json+ try:+ return self._count_tokens_estimate(json.dumps(tools))+ except (TypeError, ValueError):+ return 0++ def _stabilize_prefix(+ self,+ messages: list[dict[str, Any]],+ config: CacheConfig,+ ) -> tuple[list[dict[str, Any]], list[str]]:+ """Stabilize the prefix by moving dynamic content."""+ transforms: list[str] = []++ for message in messages:+ if message.get("role") != "system":+ continue++ content = message.get("content", "")+ if isinstance(content, str):+ new_content, applied = self._stabilize_text(content, config)+ if new_content != content:+ message["content"] = new_content+ transforms.extend(applied)++ elif isinstance(content, list):+ for block in content:+ if isinstance(block, dict) and block.get("type") == "text":+ text = block.get("text", "")+ new_text, applied = self._stabilize_text(text, config)+ if new_text != text:+ block["text"] = new_text+ transforms.extend(applied)++ return messages, transforms++ def _stabilize_text(+ self, text: str, config: CacheConfig+ ) -> tuple[str, list[str]]:+ """Stabilize a text string."""+ transforms: list[str] = []+ result = text++ extracted_dates: list[str] = []+ for pattern in config.date_patterns:+ matches = re.findall(pattern, result)+ if matches:+ extracted_dates.extend(matches)+ result = re.sub(pattern, "", result)+ transforms.append("extracted_dates")++ if config.normalize_whitespace:+ new_result = re.sub(r"[ \t]+", " ", result)+ if new_result != result:+ result = new_result+ transforms.append("normalized_spaces")++ if config.collapse_blank_lines:+ new_result = re.sub(r"\n{3,}", "\n\n", result)+ if new_result != result:+ result = new_result+ transforms.append("collapsed_blank_lines")++ result = result.strip()++ if extracted_dates:+ result = result + config.dynamic_separator + " ".join(extracted_dates)++ return result, list(set(transforms))++ def _plan_breakpoints(+ self,+ sections: list[ContentSection],+ config: CacheConfig,+ ) -> BreakpointPlan:+ """Plan where to place cache breakpoints."""+ plan = BreakpointPlan()++ cacheable = [s for s in sections if s.is_cacheable]+ if not cacheable:+ plan.warnings.append("No sections meet caching requirements")+ return plan++ priority_order = {"system": 0, "tools": 1, "examples": 2}+ cacheable.sort(key=lambda s: priority_order.get(s.section_type, 3))++ selected: list[ContentSection] = []+ accumulated_tokens = 0++ for section in cacheable:+ if len(selected) >= config.max_breakpoints:+ plan.warnings.append(+ f"Reached maximum breakpoints ({config.max_breakpoints})"+ )+ break++ selected.append(section)+ accumulated_tokens += section.token_count++ for section in selected:+ location = self._section_type_to_location(section.section_type)+ breakpoint = CacheBreakpoint(+ message_index=section.message_index,+ location=location,+ content_index=section.content_index,+ tokens_at_breakpoint=section.token_count,+ reason=section.reason,+ )+ plan.breakpoints.append(breakpoint)++ plan.total_cacheable_tokens = accumulated_tokens+ if accumulated_tokens > 0:+ plan.estimated_savings_percent = 90.0++ return plan++ def _section_type_to_location(self, section_type: str) -> BreakpointLocation:+ """Convert section type to breakpoint location enum."""+ mapping = {+ "system": BreakpointLocation.AFTER_SYSTEM,+ "tools": BreakpointLocation.AFTER_TOOLS,+ "examples": BreakpointLocation.AFTER_EXAMPLES,+ }+ return mapping.get(section_type, BreakpointLocation.CUSTOM)++ def _insert_breakpoints(+ self,+ messages: list[dict[str, Any]],+ breakpoints: list[CacheBreakpoint],+ ) -> list[dict[str, Any]]:+ """Insert cache_control blocks at specified positions."""+ for bp in breakpoints:+ if bp.message_index >= len(messages):+ continue++ message = messages[bp.message_index]+ content = message.get("content", "")++ if isinstance(content, str):+ message["content"] = [+ {+ "type": "text",+ "text": content,+ "cache_control": {"type": "ephemeral"},+ }+ ]+ elif isinstance(content, list):+ if bp.content_index is not None and bp.content_index < len(content):+ block = content[bp.content_index]+ if isinstance(block, dict):+ block["cache_control"] = {"type": "ephemeral"}+ elif content:+ last_block = content[-1]+ if isinstance(last_block, dict):+ last_block["cache_control"] = {"type": "ephemeral"}++ return messages++ def _extract_cacheable_content(self, messages: list[dict[str, Any]]) -> str:+ """Extract content that has cache_control markers for hashing."""+ parts: list[str] = []++ for message in messages:+ content = message.get("content", "")+ if isinstance(content, list):+ for block in content:+ if isinstance(block, dict) and "cache_control" in block:+ text = block.get("text", "")+ if text:+ parts.append(text)+ elif isinstance(content, str) and message.get("role") == "system":+ parts.append(content)++ return "\n".join(parts)++ def estimate_savings(+ self,+ messages: list[dict[str, Any]],+ context: OptimizationContext,+ ) -> float:+ """Estimate potential savings from caching."""+ sections = self._analyze_sections(messages)+ plan = self._plan_breakpoints(sections, self.config)++ if plan.total_cacheable_tokens == 0:+ return 0.0++ total_tokens = sum(s.token_count for s in sections)+ cacheable_ratio = plan.total_cacheable_tokens / total_tokens+ return 90.0 * cacheable_ratio++ def get_cache_write_cost_multiplier(self) -> float:+ return ANTHROPIC_WRITE_COST_MULTIPLIER++ def get_cache_read_cost_multiplier(self) -> float:+ return ANTHROPIC_READ_COST_MULTIPLIER++ def get_cache_ttl_seconds(self) -> int:+ return ANTHROPIC_CACHE_TTL_SECONDSdiff --git a/headroom/cache/base.py b/headroom/cache/base.pynew file mode 100644index 000000000..617c961fa--- /dev/null+++ b/headroom/cache/base.py@@ -0,0 +1,339 @@+"""+Base types and interfaces for cache optimization.++This module defines the core abstractions that all cache optimizers implement.+The design allows for provider-specific implementations while maintaining a+consistent interface for users.+"""++from __future__ import annotations++from abc import ABC, abstractmethod+from dataclasses import dataclass, field+from datetime import datetime+from enum import Enum+from typing import Any, Literal, Protocol, runtime_checkable+++class CacheStrategy(Enum):+ """Cache optimization strategy."""++ # Just stabilize prefix (move dates, normalize whitespace)+ PREFIX_STABILIZATION = "prefix_stabilization"++ # Insert explicit cache breakpoints (Anthropic)+ EXPLICIT_BREAKPOINTS = "explicit_breakpoints"++ # Manage separate cached content objects (Google)+ CACHED_CONTENT = "cached_content"++ # No optimization possible (provider doesn't support caching)+ NONE = "none"+++class BreakpointLocation(Enum):+ """Where to insert cache breakpoints."""++ AFTER_SYSTEM = "after_system"+ AFTER_TOOLS = "after_tools"+ AFTER_EXAMPLES = "after_examples"+ CUSTOM = "custom"+++@dataclass+class CacheBreakpoint:+ """+ Represents a cache breakpoint location.++ For Anthropic, this maps to cache_control blocks.+ For other providers, this is informational.+ """++ # Message index where breakpoint should be inserted+ message_index: int++ # Location type+ location: BreakpointLocation++ # For content arrays, index within the content+ content_index: int | None = None++ # Token count at this breakpoint+ tokens_at_breakpoint: int = 0++ # Reason for this breakpoint+ reason: str = ""+++@dataclass+class CacheConfig:+ """Configuration for cache optimization."""++ # Whether to optimize at all+ enabled: bool = True++ # Strategy to use (auto-detected if None)+ strategy: CacheStrategy | None = None++ # Minimum tokens before caching makes sense+ min_cacheable_tokens: int = 1024++ # Maximum number of breakpoints (Anthropic limit is 4)+ max_breakpoints: int = 4++ # Patterns to extract and move to dynamic section+ date_patterns: list[str] = field(default_factory=lambda: [+ r"Today is \w+ \d{1,2},? \d{4}\.?",+ r"Current date: \d{4}-\d{2}-\d{2}",+ r"The current time is .+\.",+ ])++ # Whether to normalize whitespace+ normalize_whitespace: bool = True++ # Collapse multiple blank lines+ collapse_blank_lines: bool = True++ # Separator between static and dynamic content+ dynamic_separator: str = "\n\n---\n\n"++ # Dynamic content detection tiers (for OpenAI prefix stabilization)+ # - "regex": Fast pattern matching (~0ms) - always recommended+ # - "ner": Named Entity Recognition via spaCy (~5-10ms) - catches names, money, etc.+ # - "semantic": Embedding similarity (~20-50ms) - catches volatile patterns+ # Default is regex-only for speed. Add tiers for better detection at cost of latency.+ dynamic_detection_tiers: list[Literal["regex", "ner", "semantic"]] = field(+ default_factory=lambda: ["regex"]+ )++ # For semantic caching+ semantic_cache_enabled: bool = False+ semantic_similarity_threshold: float = 0.95+ semantic_cache_ttl_seconds: int = 300+++@dataclass+class CacheMetrics:+ """Metrics about cache optimization."""++ # Prefix analysis+ stable_prefix_tokens: int = 0+ stable_prefix_hash: str = ""++ # Breakpoint info+ breakpoints_inserted: int = 0+ breakpoint_locations: list[CacheBreakpoint] = field(default_factory=list)++ # Cache hit estimation+ prefix_changed_from_previous: bool = False+ previous_prefix_hash: str | None = None+ estimated_cache_hit: bool = False++ # Savings estimation+ estimated_savings_percent: float = 0.0+ cacheable_tokens: int = 0+ non_cacheable_tokens: int = 0++ # Provider-specific+ provider_cache_id: str | None = None # For Google's CachedContent+ cache_ttl_remaining_seconds: int | None = None+++@dataclass+class OptimizationContext:+ """Context for optimization request."""++ # Request tracking+ request_id: str = ""+ timestamp: datetime = field(default_factory=datetime.now)++ # Provider info+ provider: str = ""+ model: str = ""++ # Query for relevance (used by semantic cache)+ query: str | None = None++ # Previous request info (for cache hit detection)+ previous_prefix_hash: str | None = None++ # Additional metadata+ metadata: dict[str, Any] = field(default_factory=dict)+++@dataclass+class CacheResult:+ """Result of cache optimization."""++ # Optimized messages+ messages: list[dict[str, Any]]++ # Whether this was a semantic cache hit+ semantic_cache_hit: bool = False++ # Cached response (if semantic cache hit)+ cached_response: Any | None = None++ # Optimization metrics+ metrics: CacheMetrics = field(default_factory=CacheMetrics)++ # Tokens before/after+ tokens_before: int = 0+ tokens_after: int = 0++ # Transforms applied+ transforms_applied: list[str] = field(default_factory=list)++ # Warnings+ warnings: list[str] = field(default_factory=list)+++@runtime_checkable+class CacheOptimizer(Protocol):+ """+ Protocol for cache optimizers.++ All provider-specific optimizers must implement this interface.+ This allows for easy swapping of implementations and plugin registration.+ """++ @property+ def name(self) -> str:+ """Name of this optimizer."""+ ...++ @property+ def provider(self) -> str:+ """Provider this optimizer is for."""+ ...++ @property+ def strategy(self) -> CacheStrategy:+ """The caching strategy this optimizer uses."""+ ...++ def optimize(+ self,+ messages: list[dict[str, Any]],+ context: OptimizationContext,+ config: CacheConfig | None = None,+ ) -> CacheResult:+ """+ Optimize messages for caching.++ Args:+ messages: The messages to optimize.+ context: Optimization context with request info.+ config: Optional configuration override.++ Returns:+ CacheResult with optimized messages and metrics.+ """+ ...++ def get_metrics(self) -> CacheMetrics:+ """Get aggregated metrics from this optimizer."""+ ...++ def estimate_savings(+ self,+ messages: list[dict[str, Any]],+ context: OptimizationContext,+ ) -> float:+ """+ Estimate potential savings from optimization.++ Returns:+ Estimated savings as a percentage (0-100).+ """+ ...+++class BaseCacheOptimizer(ABC):+ """+ Abstract base class for cache optimizers.++ Provides common functionality for all optimizers.+ """++ def __init__(self, config: CacheConfig | None = None):+ self.config = config or CacheConfig()+ self._metrics_history: list[CacheMetrics] = []+ self._previous_prefix_hash: str | None = None++ @property+ @abstractmethod+ def name(self) -> str:+ """Name of this optimizer."""+ ...++ @property+ @abstractmethod+ def provider(self) -> str:+ """Provider this optimizer is for."""+ ...++ @property+ @abstractmethod+ def strategy(self) -> CacheStrategy:+ """The caching strategy this optimizer uses."""+ ...++ @abstractmethod+ def optimize(+ self,+ messages: list[dict[str, Any]],+ context: OptimizationContext,+ config: CacheConfig | None = None,+ ) -> CacheResult:+ """Optimize messages for caching."""+ ...++ def get_metrics(self) -> CacheMetrics:+ """Get aggregated metrics."""+ if not self._metrics_history:+ return CacheMetrics()++ # Return most recent metrics+ return self._metrics_history[-1]++ def estimate_savings(+ self,+ messages: list[dict[str, Any]],+ context: OptimizationContext,+ ) -> float:+ """Estimate potential savings."""+ # Default implementation - subclasses can override+ result = self.optimize(messages, context)+ return result.metrics.estimated_savings_percent++ def _record_metrics(self, metrics: CacheMetrics) -> None:+ """Record metrics for history."""+ self._metrics_history.append(metrics)+ # Keep only last 100 entries+ if len(self._metrics_history) > 100:+ self._metrics_history = self._metrics_history[-100:]++ def _compute_prefix_hash(self, content: str) -> str:+ """Compute a short hash of content."""+ import hashlib+ return hashlib.sha256(content.encode()).hexdigest()[:12]++ def _extract_system_content(self, messages: list[dict[str, Any]]) -> str:+ """Extract content from system messages."""+ parts = []+ for msg in messages:+ if msg.get("role") == "system":+ content = msg.get("content", "")+ if isinstance(content, str):+ parts.append(content)+ elif isinstance(content, list):+ # Handle content blocks+ for block in content:+ if isinstance(block, dict) and block.get("type") == "text":+ parts.append(block.get("text", ""))+ return "\n".join(parts)++ def _count_tokens_estimate(self, text: str) -> int:+ """Rough token count estimate (4 chars per token)."""+ return len(text) // 4diff --git a/headroom/cache/dynamic_detector.py b/headroom/cache/dynamic_detector.pynew file mode 100644index 000000000..01a2d60ab--- /dev/null+++ b/headroom/cache/dynamic_detector.py@@ -0,0 +1,951 @@+"""+Dynamic Content Detector for Cache Optimization.++This module provides a scalable, language-agnostic approach to detecting dynamic+content in prompts. Dynamic content (dates, prices, user data, session info) breaks+cache prefixes. By detecting and moving dynamic content to the end, we maximize+cache hits.++Design Philosophy:+ - NO HARDCODED PATTERNS for locale-specific content (no month names, etc.)+ - Structural detection: "Label: value" patterns where LABEL indicates dynamism+ - Entropy-based detection: High entropy = likely dynamic (UUIDs, tokens, hashes)+ - Universal patterns only: ISO 8601, UUIDs, Unix timestamps (truly universal)++Tiers (configurable, each adds latency):+ Tier 1: Regex (~0ms) - Structural patterns, universal formats, entropy-based+ Tier 2: NER (~5-10ms) - Named Entity Recognition for names, money, orgs+ Tier 3: Semantic (~20-50ms) - Embedding similarity to known dynamic patterns++Usage:+ from headroom.cache.dynamic_detector import DynamicContentDetector++ detector = DynamicContentDetector(tiers=["regex", "ner"])+ result = detector.detect("Session: abc123. User: John paid $500.")++ # result.spans = [+ # DynamicSpan(text="Session: abc123", category="session", tier="regex", ...),+ # DynamicSpan(text="John", category="person", tier="ner", ...),+ # DynamicSpan(text="$500", category="money", tier="ner", ...),+ # ]+"""++from __future__ import annotations++import math+import re+from dataclasses import dataclass, field+from enum import Enum+from typing import Any, Literal++# Optional imports - graceful degradation+_SPACY_AVAILABLE = False+_SENTENCE_TRANSFORMERS_AVAILABLE = False++try:+ import spacy+ _SPACY_AVAILABLE = True+except ImportError:+ spacy = None # type: ignore++try:+ from sentence_transformers import SentenceTransformer+ import numpy as np+ _SENTENCE_TRANSFORMERS_AVAILABLE = True+except ImportError:+ SentenceTransformer = None # type: ignore+ np = None # type: ignore+++class DynamicCategory(str, Enum):+ """Categories of dynamic content."""++ # Tier 1: Structural/Regex detectable+ DATE = "date"+ TIME = "time"+ DATETIME = "datetime"+ TIMESTAMP = "timestamp"+ UUID = "uuid"+ REQUEST_ID = "request_id"+ VERSION = "version"+ SESSION = "session"+ USER_DATA = "user_data"+ IDENTIFIER = "identifier" # Generic high-entropy ID++ # Tier 2: NER detectable+ PERSON = "person"+ MONEY = "money"+ ORG = "org"+ LOCATION = "location"++ # Tier 3: Semantic+ VOLATILE = "volatile" # Semantically detected as changing+ REALTIME = "realtime"++ # Fallback+ UNKNOWN = "unknown"+++@dataclass+class DynamicSpan:+ """A span of dynamic content detected in text."""++ # The actual text matched+ text: str++ # Position in original content+ start: int+ end: int++ # What category of dynamic content+ category: DynamicCategory++ # Which tier detected it+ tier: Literal["regex", "ner", "semantic"]++ # Confidence score (0-1)+ confidence: float = 1.0++ # Additional metadata (pattern name, entity type, etc.)+ metadata: dict[str, Any] = field(default_factory=dict)+++@dataclass+class DetectionResult:+ """Result of dynamic content detection."""++ # All detected spans+ spans: list[DynamicSpan]++ # Content with dynamic parts removed+ static_content: str++ # Content that was extracted (for reinsertion at end)+ dynamic_content: str++ # Which tiers were used+ tiers_used: list[str]++ # Processing time in milliseconds+ processing_time_ms: float = 0.0++ # Any warnings (e.g., "spaCy not available, skipping NER")+ warnings: list[str] = field(default_factory=list)+++@dataclass+class DetectorConfig:+ """Configuration for the dynamic content detector."""++ # Which tiers to enable (order matters - later tiers can use earlier results)+ tiers: list[Literal["regex", "ner", "semantic"]] = field(+ default_factory=lambda: ["regex"]+ )++ # Tier 1: Structural labels that indicate dynamic content+ # These are the KEY names that hint the VALUE is dynamic+ # Users can add domain-specific labels+ dynamic_labels: list[str] = field(default_factory=lambda: [+ # Time-related+ "date", "time", "timestamp", "datetime", "created", "updated",+ "modified", "expires", "last", "current", "today", "now",+ # Identifiers+ "id", "uuid", "guid", "session", "request", "trace", "span",+ "transaction", "correlation", "token", "key", "secret",+ # User-related+ "user", "username", "email", "name", "phone", "address",+ "customer", "client", "employee", "member",+ # System state+ "version", "build", "commit", "branch", "revision",+ "status", "state", "count", "total", "balance", "remaining",+ "load", "queue", "active", "pending",+ # Order/ticket related+ "order", "ticket", "case", "invoice", "reference",+ ])++ # Tier 1: Custom regex patterns (user-provided)+ custom_patterns: list[tuple[str, DynamicCategory]] = field(+ default_factory=list+ )++ # Entropy threshold for detecting random strings (0-1 scale normalized)+ # Higher = more selective (only very random strings)+ entropy_threshold: float = 0.7++ # Minimum length for entropy-based detection+ min_entropy_length: int = 8++ # Tier 2: NER config+ spacy_model: str = "en_core_web_sm"+ ner_entity_types: list[str] = field(+ default_factory=lambda: ["DATE", "TIME", "MONEY", "PERSON", "ORG", "GPE"]+ )++ # Tier 3: Semantic config+ embedding_model: str = "all-MiniLM-L6-v2"+ semantic_threshold: float = 0.7++ # General+ min_span_length: int = 2+ merge_overlapping: bool = True+++def calculate_entropy(s: str) -> float:+ """+ Calculate Shannon entropy of a string, normalized to 0-1.++ Higher entropy = more random/unpredictable = likely dynamic.+ - "aaaaaaa" -> ~0 (low entropy, predictable)+ - "a1b2c3d4" -> ~0.7 (medium entropy)+ - "550e8400-e29b-41d4" -> ~0.9 (high entropy, random-looking)++ Returns:+ Normalized entropy (0-1). Higher = more likely dynamic.+ """+ if not s:+ return 0.0++ # Count character frequencies+ freq: dict[str, int] = {}+ for char in s:+ freq[char] = freq.get(char, 0) + 1++ # Calculate entropy+ length = len(s)+ entropy = 0.0+ for count in freq.values():+ p = count / length+ entropy -= p * math.log2(p)++ # Normalize: max entropy for string of length n with k unique chars+ # is log2(min(n, alphabet_size)). We'll normalize by log2(length)+ # to get a 0-1 scale+ max_entropy = math.log2(length) if length > 1 else 1.0+ return entropy / max_entropy if max_entropy > 0 else 0.0+++class RegexDetector:+ """+ Tier 1: Scalable pattern detection.++ Uses THREE strategies (no hardcoded month names!):+ 1. Structural: "Label: value" patterns where label indicates dynamic content+ 2. Universal: Truly universal formats (ISO 8601, UUID, Unix timestamps)+ 3. Entropy: High-entropy strings (tokens, hashes, IDs)+ """++ # Universal patterns (these formats are language-agnostic)+ UNIVERSAL_PATTERNS = [+ # UUID - truly universal format+ (r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}",+ DynamicCategory.UUID, "uuid"),++ # ISO 8601 datetime (most universal date format)+ (r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?",+ DynamicCategory.DATETIME, "iso_datetime"),++ # ISO 8601 date only+ (r"\d{4}-\d{2}-\d{2}(?!\d)",+ DynamicCategory.DATE, "iso_date"),++ # Unix timestamps (10-13 digits, but NOT within longer numbers)+ (r"(?<![0-9])\d{10,13}(?![0-9])",+ DynamicCategory.TIMESTAMP, "unix_timestamp"),++ # 24-hour time HH:MM:SS or HH:MM+ (r"(?<![0-9])\d{1,2}:\d{2}(?::\d{2})?(?:\s*(?:AM|PM|am|pm))?(?![0-9])",+ DynamicCategory.TIME, "time"),++ # Version numbers with v prefix (unambiguous)+ (r"\bv\d+\.\d+(?:\.\d+)?(?:-[a-zA-Z0-9.]+)?",+ DynamicCategory.VERSION, "version"),++ # API key/token patterns (prefix + random string)+ (r"\b(?:sk|pk|api|key|token|bearer|auth)[-_][a-zA-Z0-9]{16,}",+ DynamicCategory.REQUEST_ID, "api_key"),++ # Common prefixed IDs (req_, sess_, txn_, etc.)+ (r"\b[a-z]{2,6}_[a-zA-Z0-9]{8,}",+ DynamicCategory.REQUEST_ID, "prefixed_id"),++ # Hex strings of common ID lengths (32 = MD5, 40 = SHA1, 64 = SHA256)+ (r"\b[a-fA-F0-9]{32}\b",+ DynamicCategory.IDENTIFIER, "hex_32"),+ (r"\b[a-fA-F0-9]{40}\b",+ DynamicCategory.IDENTIFIER, "hex_40"),+ (r"\b[a-fA-F0-9]{64}\b",+ DynamicCategory.IDENTIFIER, "hex_64"),++ # JWT tokens (three base64 sections separated by dots)+ (r"eyJ[a-zA-Z0-9_-]+\.eyJ[a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+",+ DynamicCategory.REQUEST_ID, "jwt"),+ ]++ def __init__(self, config: DetectorConfig):+ """Initialize regex detector."""+ self.config = config++ # Compile universal patterns+ self._universal_patterns: list[tuple[re.Pattern[str], DynamicCategory, str]] = [+ (re.compile(pattern), category, name)+ for pattern, category, name in self.UNIVERSAL_PATTERNS+ ]++ # Build structural pattern from dynamic labels+ # Pattern: "label" followed by separator then value+ labels_pattern = "|".join(re.escape(label) for label in config.dynamic_labels)+ self._structural_pattern = re.compile(+ rf"(?P<label>(?:{labels_pattern}))(?P<sep>\s*[:=]\s*|\s+)(?P<value>[^\n,;]+)",+ re.IGNORECASE+ )++ # Compile custom patterns+ self._custom_patterns: list[tuple[re.Pattern[str], DynamicCategory]] = [+ (re.compile(pattern, re.IGNORECASE), category)+ for pattern, category in config.custom_patterns+ ]++ def detect(self, content: str) -> list[DynamicSpan]:+ """Detect dynamic content using structural, universal, and entropy detection."""+ spans: list[DynamicSpan] = []+ seen_ranges: set[tuple[int, int]] = set()++ # 1. Universal patterns first (most specific)+ for pattern, category, pattern_name in self._universal_patterns:+ for match in pattern.finditer(content):+ start, end = match.start(), match.end()+ if self._is_overlapping(start, end, seen_ranges):+ continue+ if end - start < self.config.min_span_length:+ continue++ spans.append(DynamicSpan(+ text=match.group(),+ start=start,+ end=end,+ category=category,+ tier="regex",+ confidence=1.0,+ metadata={"pattern": pattern_name, "method": "universal"},+ ))+ seen_ranges.add((start, end))++ # 2. Structural detection: "Label: value" patterns+ for match in self._structural_pattern.finditer(content):+ # Get the full match range+ start, end = match.start(), match.end()++ # Skip if overlaps with universal patterns+ if self._is_overlapping(start, end, seen_ranges):+ continue++ label = match.group("label").lower()+ value = match.group("value").strip()++ # Determine category from label+ category = self._categorize_label(label)++ # Only add the value portion (keep label as static)+ value_start = match.start("value")+ value_end = match.end("value")++ # Skip if value is too short or empty+ if value_end - value_start < self.config.min_span_length:+ continue+ if not value.strip():+ continue++ spans.append(DynamicSpan(+ text=value,+ start=value_start,+ end=value_end,+ category=category,+ tier="regex",+ confidence=0.9,+ metadata={"pattern": "structural", "method": "structural", "label": label},+ ))+ seen_ranges.add((value_start, value_end))++ # 3. Entropy-based detection for remaining potential IDs+ spans.extend(self._detect_high_entropy(content, seen_ranges))++ # 4. Custom patterns+ for pattern, category in self._custom_patterns:+ for match in pattern.finditer(content):+ start, end = match.start(), match.end()+ if self._is_overlapping(start, end, seen_ranges):+ continue+ if end - start < self.config.min_span_length:+ continue++ spans.append(DynamicSpan(+ text=match.group(),+ start=start,+ end=end,+ category=category,+ tier="regex",+ confidence=0.8,+ metadata={"pattern": "custom", "method": "custom"},+ ))+ seen_ranges.add((start, end))++ return sorted(spans, key=lambda s: s.start)++ def _detect_high_entropy(+ self,+ content: str,+ seen_ranges: set[tuple[int, int]],+ ) -> list[DynamicSpan]:+ """+ Detect high-entropy strings that look like IDs/tokens.++ Finds alphanumeric sequences and checks their entropy.+ High entropy = likely random/generated = dynamic.+ """+ spans: list[DynamicSpan] = []++ # Find alphanumeric sequences (potential IDs)+ # Must be at least min_entropy_length chars, mix of letters/numbers+ pattern = re.compile(r"\b[a-zA-Z0-9_-]{8,}\b")++ for match in pattern.finditer(content):+ start, end = match.start(), match.end()+ text = match.group()++ # Skip if already detected+ if self._is_overlapping(start, end, seen_ranges):+ continue++ # Skip if too short+ if len(text) < self.config.min_entropy_length:+ continue++ # Skip if all letters or all numbers (not random-looking)+ if text.isalpha() or text.isdigit():+ continue++ # Skip common words that might look like IDs+ if text.lower() in {"username", "password", "localhost", "undefined"}:+ continue++ # Calculate entropy+ entropy = calculate_entropy(text)++ if entropy >= self.config.entropy_threshold:+ spans.append(DynamicSpan(+ text=text,+ start=start,+ end=end,+ category=DynamicCategory.IDENTIFIER,+ tier="regex",+ confidence=entropy, # Use entropy as confidence+ metadata={"pattern": "entropy", "method": "entropy", "entropy": entropy},+ ))+ seen_ranges.add((start, end))++ return spans++ def _is_overlapping(+ self,+ start: int,+ end: int,+ seen_ranges: set[tuple[int, int]],+ ) -> bool:+ """Check if range overlaps with any existing range."""+ return any(+ not (end <= s or start >= e)+ for s, e in seen_ranges+ )++ def _categorize_label(self, label: str) -> DynamicCategory:+ """Categorize based on the label name."""+ label = label.lower()++ # Time-related+ if label in {"date", "datetime", "created", "updated", "modified", "expires", "today"}:+ return DynamicCategory.DATE+ if label in {"time", "timestamp", "now"}:+ return DynamicCategory.TIMESTAMP+ if label == "current":+ return DynamicCategory.DATETIME++ # Identifiers+ if label in {"id", "uuid", "guid"}:+ return DynamicCategory.UUID+ if label in {"session", "request", "trace", "span", "transaction", "correlation"}:+ return DynamicCategory.SESSION+ if label in {"token", "key", "secret"}:+ return DynamicCategory.REQUEST_ID++ # User-related+ if label in {"user", "username", "email", "name", "phone", "address",+ "customer", "client", "employee", "member"}:+ return DynamicCategory.USER_DATA++ # System state+ if label in {"version", "build", "commit", "branch", "revision"}:+ return DynamicCategory.VERSION+ if label in {"status", "state", "count", "total", "balance", "remaining",+ "load", "queue", "active", "pending"}:+ return DynamicCategory.VOLATILE++ # Order/ticket+ if label in {"order", "ticket", "case", "invoice", "reference"}:+ return DynamicCategory.REQUEST_ID++ return DynamicCategory.UNKNOWN+++class NERDetector:+ """Tier 2: spaCy-based Named Entity Recognition."""++ # Map spaCy entity types to our categories+ ENTITY_MAP = {+ "DATE": DynamicCategory.DATE,+ "TIME": DynamicCategory.TIME,+ "MONEY": DynamicCategory.MONEY,+ "PERSON": DynamicCategory.PERSON,+ "ORG": DynamicCategory.ORG,+ "GPE": DynamicCategory.LOCATION, # Geo-Political Entity+ "LOC": DynamicCategory.LOCATION,+ "FAC": DynamicCategory.LOCATION, # Facility+ "CARDINAL": DynamicCategory.UNKNOWN, # Numbers+ "ORDINAL": DynamicCategory.UNKNOWN,+ }++ def __init__(self, config: DetectorConfig):+ """Initialize NER detector, loading spaCy model."""+ self.config = config+ self._nlp = None+ self._load_error: str | None = None++ if not _SPACY_AVAILABLE:+ self._load_error = "spaCy not installed. Install with: pip install spacy && python -m spacy download en_core_web_sm"+ return++ try:+ self._nlp = spacy.load(config.spacy_model)+ except OSError:+ self._load_error = f"spaCy model '{config.spacy_model}' not found. Install with: python -m spacy download {config.spacy_model}"++ @property+ def is_available(self) -> bool:+ """Check if NER is available."""+ return self._nlp is not None++ def detect(+ self,+ content: str,+ existing_spans: list[DynamicSpan] | None = None,+ ) -> tuple[list[DynamicSpan], str | None]:+ """+ Detect dynamic content using NER.++ Args:+ content: Text to analyze.+ existing_spans: Spans already detected (to avoid duplicates).++ Returns:+ Tuple of (new_spans, warning_message).+ """+ if not self.is_available:+ return [], self._load_error++ # Get existing ranges to avoid duplicates+ existing_ranges = set()+ if existing_spans:+ existing_ranges = {(s.start, s.end) for s in existing_spans}++ doc = self._nlp(content)+ spans: list[DynamicSpan] = []++ for ent in doc.ents:+ # Skip entity types we don't care about+ if ent.label_ not in self.config.ner_entity_types:+ continue++ # Skip if already detected by regex+ if (ent.start_char, ent.end_char) in existing_ranges:+ continue++ # Check for overlap with existing spans+ overlaps = any(+ not (ent.end_char <= s or ent.start_char >= e)+ for s, e in existing_ranges+ )+ if overlaps:+ continue++ # Map to our category+ category = self.ENTITY_MAP.get(ent.label_, DynamicCategory.UNKNOWN)++ # Skip unknown categories+ if category == DynamicCategory.UNKNOWN:+ continue++ spans.append(DynamicSpan(+ text=ent.text,+ start=ent.start_char,+ end=ent.end_char,+ category=category,+ tier="ner",+ confidence=0.9,+ metadata={"entity_type": ent.label_},+ ))+ existing_ranges.add((ent.start_char, ent.end_char))++ return sorted(spans, key=lambda s: s.start), None+++class SemanticDetector:+ """Tier 3: Embedding-based semantic detection."""++ # Known phrases that indicate dynamic content+ # These are SEMANTIC patterns, not literal strings to match+ DYNAMIC_EXEMPLARS = [+ # Time-sensitive+ "The current date is",+ "As of today",+ "Updated on",+ "Last refreshed",+ "Real-time data",+ "Live prices",+ "Current stock price",++ # Session-specific+ "Your session ID",+ "Your account balance",+ "Your recent orders",+ "Your conversation history",++ # User-specific+ "Hello [user]",+ "Dear customer",+ "Your name is",++ # System state+ "Server status",+ "System load",+ "Queue length",+ "Active users",+ ]++ def __init__(self, config: DetectorConfig):+ """Initialize semantic detector with embedding model."""+ self.config = config+ self._model = None+ self._exemplar_embeddings = None+ self._load_error: str | None = None++ if not _SENTENCE_TRANSFORMERS_AVAILABLE:+ self._load_error = "sentence-transformers not installed. Install with: pip install sentence-transformers"+ return++ try:+ self._model = SentenceTransformer(config.embedding_model)+ # Pre-compute exemplar embeddings+ self._exemplar_embeddings = self._model.encode(+ self.DYNAMIC_EXEMPLARS,+ convert_to_numpy=True,+ )+ except Exception as e:+ self._load_error = f"Failed to load embedding model: {e}"++ @property+ def is_available(self) -> bool:+ """Check if semantic detection is available."""+ return self._model is not None++ def detect(+ self,+ content: str,+ existing_spans: list[DynamicSpan] | None = None,+ ) -> tuple[list[DynamicSpan], str | None]:+ """+ Detect dynamic content using semantic similarity.++ Splits content into sentences and checks each against known+ dynamic patterns using embedding similarity.++ Args:+ content: Text to analyze.+ existing_spans: Spans already detected (to avoid duplicates).++ Returns:+ Tuple of (new_spans, warning_message).+ """+ if not self.is_available:+ return [], self._load_error++ # Simple sentence splitting (could use spaCy if available)+ sentences = self._split_sentences(content)+ spans: list[DynamicSpan] = []++ # Get existing ranges+ existing_ranges = set()+ if existing_spans:+ existing_ranges = {(s.start, s.end) for s in existing_spans}++ # Encode all sentences+ if not sentences:+ return [], None++ sentence_texts = [s[0] for s in sentences]+ sentence_embeddings = self._model.encode(+ sentence_texts,+ convert_to_numpy=True,+ )++ # Compute similarities+ similarities = np.dot(sentence_embeddings, self._exemplar_embeddings.T)++ for i, (text, start, end) in enumerate(sentences):+ # Get max similarity to any exemplar+ max_sim = float(np.max(similarities[i]))++ if max_sim < self.config.semantic_threshold:+ continue++ # Check overlap with existing spans+ overlaps = any(+ not (end <= s or start >= e)+ for s, e in existing_ranges+ )+ if overlaps:+ continue++ # Find which exemplar matched best+ best_exemplar_idx = int(np.argmax(similarities[i]))+ best_exemplar = self.DYNAMIC_EXEMPLARS[best_exemplar_idx]++ # Determine category based on exemplar+ category = self._categorize_exemplar(best_exemplar)++ spans.append(DynamicSpan(+ text=text,+ start=start,+ end=end,+ category=category,+ tier="semantic",+ confidence=max_sim,+ metadata={+ "matched_exemplar": best_exemplar,+ "similarity": max_sim,+ },+ ))+ existing_ranges.add((start, end))++ return sorted(spans, key=lambda s: s.start), None++ def _split_sentences(self, content: str) -> list[tuple[str, int, int]]:+ """Split content into sentences with positions."""+ sentences: list[tuple[str, int, int]] = []+ pattern = r'[^.!?\n]+[.!?\n]?'+ for match in re.finditer(pattern, content):+ text = match.group().strip()+ if len(text) > 10:+ sentences.append((text, match.start(), match.end()))+ return sentences++ def _categorize_exemplar(self, exemplar: str) -> DynamicCategory:+ """Categorize based on which exemplar matched."""+ exemplar_lower = exemplar.lower()++ if any(w in exemplar_lower for w in ["date", "today", "updated", "refreshed"]):+ return DynamicCategory.DATE+ elif any(w in exemplar_lower for w in ["price", "stock", "live", "real-time"]):+ return DynamicCategory.REALTIME+ elif any(w in exemplar_lower for w in ["session", "account", "your"]):+ return DynamicCategory.SESSION+ elif any(w in exemplar_lower for w in ["status", "load", "queue", "active"]):+ return DynamicCategory.VOLATILE+ else:+ return DynamicCategory.VOLATILE+++class DynamicContentDetector:+ """+ Unified dynamic content detector with tiered detection.++ Key Design Principles:+ - NO hardcoded locale-specific patterns (no month names)+ - Structural detection: Labels indicate what's dynamic+ - Universal patterns: ISO 8601, UUIDs, Unix timestamps+ - Entropy-based: High entropy = random/generated = dynamic++ Usage:+ # Fast mode (regex only - structural + universal + entropy)+ detector = DynamicContentDetector(DetectorConfig(tiers=["regex"]))++ # Balanced mode (regex + NER for names/money)+ detector = DynamicContentDetector(DetectorConfig(tiers=["regex", "ner"]))++ # Full mode (all tiers)+ detector = DynamicContentDetector(DetectorConfig(+ tiers=["regex", "ner", "semantic"]+ ))++ result = detector.detect("Session: abc123. User: John paid $500.")+ """++ def __init__(self, config: DetectorConfig | None = None):+ """Initialize detector with configuration."""+ self.config = config or DetectorConfig()++ # Initialize detectors based on enabled tiers+ self._regex_detector: RegexDetector | None = None+ self._ner_detector: NERDetector | None = None+ self._semantic_detector: SemanticDetector | None = None++ if "regex" in self.config.tiers:+ self._regex_detector = RegexDetector(self.config)++ if "ner" in self.config.tiers:+ self._ner_detector = NERDetector(self.config)++ if "semantic" in self.config.tiers:+ self._semantic_detector = SemanticDetector(self.config)++ def detect(self, content: str) -> DetectionResult:+ """+ Detect dynamic content in text.++ Runs enabled tiers in order, accumulating spans.+ Each tier can see what previous tiers detected.++ Args:+ content: Text to analyze.++ Returns:+ DetectionResult with spans, static/dynamic content split, etc.+ """+ import time+ start_time = time.perf_counter()++ all_spans: list[DynamicSpan] = []+ tiers_used: list[str] = []+ warnings: list[str] = []++ # Tier 1: Regex (structural + universal + entropy)+ if self._regex_detector:+ regex_spans = self._regex_detector.detect(content)+ all_spans.extend(regex_spans)+ tiers_used.append("regex")++ # Tier 2: NER+ if self._ner_detector:+ ner_spans, ner_warning = self._ner_detector.detect(content, all_spans)+ all_spans.extend(ner_spans)+ if ner_warning:+ warnings.append(ner_warning)+ elif ner_spans or self._ner_detector.is_available:+ tiers_used.append("ner")++ # Tier 3: Semantic+ if self._semantic_detector:+ sem_spans, sem_warning = self._semantic_detector.detect(content, all_spans)+ all_spans.extend(sem_spans)+ if sem_warning:+ warnings.append(sem_warning)+ elif sem_spans or self._semantic_detector.is_available:+ tiers_used.append("semantic")++ # Sort by position+ all_spans = sorted(all_spans, key=lambda s: s.start)++ # Build static and dynamic content+ static_content, dynamic_content = self._split_content(content, all_spans)++ processing_time = (time.perf_counter() - start_time) * 1000++ return DetectionResult(+ spans=all_spans,+ static_content=static_content,+ dynamic_content=dynamic_content,+ tiers_used=tiers_used,+ processing_time_ms=processing_time,+ warnings=warnings,+ )++ def _split_content(+ self,+ content: str,+ spans: list[DynamicSpan],+ ) -> tuple[str, str]:+ """Split content into static and dynamic parts."""+ if not spans:+ return content, ""++ static = content+ dynamic_parts: list[str] = []++ for span in reversed(spans):+ dynamic_parts.append(span.text)+ static = static[:span.start] + static[span.end:]++ static = self._clean_static_content(static)+ dynamic_parts.reverse()+ dynamic = "\n".join(dynamic_parts)++ return static, dynamic++ def _clean_static_content(self, content: str) -> str:+ """Clean up static content after span removal."""+ lines = content.split("\n")+ cleaned_lines: list[str] = []+ prev_blank = False++ for line in lines:+ is_blank = not line.strip()+ if is_blank and prev_blank:+ continue+ cleaned_lines.append(line.rstrip())+ prev_blank = is_blank++ return "\n".join(cleaned_lines).strip()++ @property+ def available_tiers(self) -> list[str]:+ """Get list of actually available tiers (dependencies installed)."""+ available = []++ if self._regex_detector:+ available.append("regex")++ if self._ner_detector and self._ner_detector.is_available:+ available.append("ner")++ if self._semantic_detector and self._semantic_detector.is_available:+ available.append("semantic")++ return available+++# Convenience function+def detect_dynamic_content(+ content: str,+ tiers: list[Literal["regex", "ner", "semantic"]] | None = None,+) -> DetectionResult:+ """+ Detect dynamic content in text.++ Convenience function that creates a detector with specified tiers.++ Args:+ content: Text to analyze.+ tiers: Which tiers to use. Default: ["regex"] for speed.++ Returns:+ DetectionResult with detected spans and split content.++ Example:+ >>> result = detect_dynamic_content(+ ... "Session: abc123xyz. User: John paid $500.",+ ... tiers=["regex", "ner"]+ ... )+ >>> print(result.static_content)+ >>> print(result.dynamic_content)+ """+ config = DetectorConfig(tiers=tiers or ["regex"])+ detector = DynamicContentDetector(config)+ return detector.detect(content)diff --git a/headroom/cache/google.py b/headroom/cache/google.pynew file mode 100644index 000000000..84e325c0f--- /dev/null+++ b/headroom/cache/google.py@@ -0,0 +1,896 @@+"""+Google Cache Optimizer for CachedContent API.++Google's Gemini API offers explicit cached content management through+the `genai.caching.CachedContent` API. Key characteristics:++- Minimum 32K tokens required for caching+- 75% discount on cached input tokens+- Storage costs (pay per hour for cached content)+- User-defined TTL (default 1 hour)+- Returns cache_id for subsequent requests++This optimizer provides cache lifecycle management utilities without+making actual API calls - users integrate with the google-generativeai+package themselves.++Usage:+ optimizer = GoogleCacheOptimizer()++ # Check if content is cacheable+ analysis = optimizer.analyze_cacheability(messages, context)++ # Optimize and get cache recommendation+ result = optimizer.optimize(messages, context)++ # After user creates cache via Google API, register it+ optimizer.register_cache(+ cache_id="cached-content-xyz",+ content_hash=result.metrics.stable_prefix_hash,+ token_count=50000,+ expires_at=datetime.now() + timedelta(hours=1),+ )++ # Check if existing cache can be reused+ cache_info = optimizer.get_reusable_cache(content_hash)++ # Extend cache TTL+ optimizer.extend_cache_ttl(cache_id, additional_seconds=3600)++ # Clean up expired caches+ optimizer.cleanup_expired_caches()+"""++from __future__ import annotations++import logging+from dataclasses import dataclass, field+from datetime import datetime, timedelta+from typing import Any++from .base import (+ BaseCacheOptimizer,+ CacheConfig,+ CacheMetrics,+ CacheResult,+ CacheStrategy,+ OptimizationContext,+)++logger = logging.getLogger(__name__)+++# Google-specific constants+GOOGLE_MIN_CACHE_TOKENS = 32_768 # 32K tokens minimum+GOOGLE_CACHE_DISCOUNT = 0.75 # 75% discount on cached tokens+GOOGLE_DEFAULT_TTL_SECONDS = 3600 # 1 hour default+GOOGLE_MAX_TTL_SECONDS = 86400 * 7 # 7 days maximum+++@dataclass+class CachedContentInfo:+ """+ Information about a cached content object.++ Tracks the lifecycle of a Google CachedContent resource.+ """++ # Google's cache identifier+ cache_id: str++ # Hash of the content for matching+ content_hash: str++ # Timestamps+ created_at: datetime+ expires_at: datetime++ # Token count in the cached content+ token_count: int++ # Optional model used (some caches are model-specific)+ model: str | None = None++ # Display name for the cached content+ display_name: str | None = None++ # Metadata for tracking+ metadata: dict[str, Any] = field(default_factory=dict)++ @property+ def is_expired(self) -> bool:+ """Check if cache has expired."""+ return datetime.now() >= self.expires_at++ @property+ def ttl_remaining_seconds(self) -> int:+ """Seconds remaining until expiry."""+ remaining = (self.expires_at - datetime.now()).total_seconds()+ return max(0, int(remaining))++ @property+ def age_seconds(self) -> int:+ """Age of the cache in seconds."""+ return int((datetime.now() - self.created_at).total_seconds())++ def to_dict(self) -> dict[str, Any]:+ """Serialize to dictionary."""+ return {+ "cache_id": self.cache_id,+ "content_hash": self.content_hash,+ "created_at": self.created_at.isoformat(),+ "expires_at": self.expires_at.isoformat(),+ "token_count": self.token_count,+ "model": self.model,+ "display_name": self.display_name,+ "metadata": self.metadata,+ }++ @classmethod+ def from_dict(cls, data: dict[str, Any]) -> CachedContentInfo:+ """Deserialize from dictionary."""+ return cls(+ cache_id=data["cache_id"],+ content_hash=data["content_hash"],+ created_at=datetime.fromisoformat(data["created_at"]),+ expires_at=datetime.fromisoformat(data["expires_at"]),+ token_count=data["token_count"],+ model=data.get("model"),+ display_name=data.get("display_name"),+ metadata=data.get("metadata", {}),+ )+++@dataclass+class CacheabilityAnalysis:+ """+ Analysis of whether content is suitable for Google caching.++ Provides detailed information about caching viability and+ potential savings.+ """++ # Whether content meets minimum threshold+ is_cacheable: bool++ # Token counts+ total_tokens: int+ cacheable_tokens: int++ # Shortfall if not cacheable+ tokens_below_minimum: int = 0++ # Estimated savings+ estimated_hourly_storage_cost_usd: float = 0.0+ estimated_savings_per_request_percent: float = 0.0++ # Recommendations+ recommendations: list[str] = field(default_factory=list)++ # Content hash for cache matching+ content_hash: str = ""+++class GoogleCacheOptimizer(BaseCacheOptimizer):+ """+ Cache optimizer for Google's Gemini CachedContent API.++ This optimizer provides:+ 1. Analysis of whether content meets Google's caching requirements+ 2. Cache lifecycle management (register, lookup, extend, delete)+ 3. Optimization recommendations+ 4. Integration utilities for the google-generativeai SDK++ The optimizer does NOT make actual API calls - it provides the+ infrastructure for users to manage caches themselves.++ Example workflow:+ optimizer = GoogleCacheOptimizer()++ # Analyze content+ result = optimizer.optimize(messages, context)++ if result.metrics.cacheable_tokens >= GOOGLE_MIN_CACHE_TOKENS:+ # User creates cache via Google SDK+ cached_content = genai.caching.CachedContent.create(+ model="gemini-1.5-pro",+ contents=contents,+ ttl=timedelta(hours=1),+ )++ # Register with optimizer for tracking+ optimizer.register_cache(+ cache_id=cached_content.name,+ content_hash=result.metrics.stable_prefix_hash,+ token_count=result.metrics.cacheable_tokens,+ expires_at=datetime.now() + timedelta(hours=1),+ )++ # Later, check for reusable cache+ cache = optimizer.get_reusable_cache(content_hash)+ if cache:+ # Use cache.cache_id in API call+ pass+ """++ def __init__(self, config: CacheConfig | None = None):+ """+ Initialize Google cache optimizer.++ Args:+ config: Optional cache configuration+ """+ super().__init__(config)++ # Override minimum tokens for Google's requirements+ if self.config.min_cacheable_tokens < GOOGLE_MIN_CACHE_TOKENS:+ self.config.min_cacheable_tokens = GOOGLE_MIN_CACHE_TOKENS++ # Cache registry: content_hash -> CachedContentInfo+ self._cache_registry: dict[str, CachedContentInfo] = {}++ # Also index by cache_id for direct lookup+ self._cache_by_id: dict[str, CachedContentInfo] = {}++ # Statistics+ self._caches_created: int = 0+ self._caches_reused: int = 0+ self._caches_expired: int = 0++ @property+ def name(self) -> str:+ """Name of this optimizer."""+ return "google-cached-content"++ @property+ def provider(self) -> str:+ """Provider this optimizer is for."""+ return "google"++ @property+ def strategy(self) -> CacheStrategy:+ """The caching strategy this optimizer uses."""+ return CacheStrategy.CACHED_CONTENT++ def optimize(+ self,+ messages: list[dict[str, Any]],+ context: OptimizationContext,+ config: CacheConfig | None = None,+ ) -> CacheResult:+ """+ Optimize messages for Google caching.++ This method:+ 1. Analyzes content for cacheability+ 2. Checks for existing reusable caches+ 3. Returns optimization metrics and recommendations++ Args:+ messages: The messages to optimize+ context: Optimization context+ config: Optional configuration override++ Returns:+ CacheResult with analysis and cache information+ """+ effective_config = config or self.config++ # Extract cacheable content (system messages + static context)+ cacheable_content = self._extract_cacheable_content(messages)+ content_hash = self._compute_prefix_hash(cacheable_content)++ # Estimate tokens+ total_tokens = self._count_tokens_estimate(+ self._messages_to_text(messages)+ )+ cacheable_tokens = self._count_tokens_estimate(cacheable_content)++ # Check for existing cache+ existing_cache = self.get_reusable_cache(content_hash)++ # Build metrics+ metrics = CacheMetrics(+ stable_prefix_tokens=cacheable_tokens,+ stable_prefix_hash=content_hash,+ prefix_changed_from_previous=(+ context.previous_prefix_hash != content_hash+ if context.previous_prefix_hash+ else False+ ),+ previous_prefix_hash=context.previous_prefix_hash,+ cacheable_tokens=cacheable_tokens,+ non_cacheable_tokens=total_tokens - cacheable_tokens,+ )++ # Calculate estimated savings+ if cacheable_tokens >= GOOGLE_MIN_CACHE_TOKENS:+ metrics.estimated_savings_percent = GOOGLE_CACHE_DISCOUNT * 100+ metrics.estimated_cache_hit = existing_cache is not None++ # Add cache info if available+ if existing_cache:+ metrics.provider_cache_id = existing_cache.cache_id+ metrics.cache_ttl_remaining_seconds = existing_cache.ttl_remaining_seconds+ self._caches_reused += 1++ # Build warnings+ warnings: list[str] = []+ if cacheable_tokens < GOOGLE_MIN_CACHE_TOKENS:+ shortfall = GOOGLE_MIN_CACHE_TOKENS - cacheable_tokens+ warnings.append(+ f"Content has {cacheable_tokens:,} tokens, needs {shortfall:,} more "+ f"to meet Google's 32K minimum for caching"+ )++ if existing_cache and existing_cache.ttl_remaining_seconds < 300:+ warnings.append(+ f"Existing cache expires in {existing_cache.ttl_remaining_seconds}s - "+ f"consider extending TTL"+ )++ # Record metrics+ self._record_metrics(metrics)+ self._previous_prefix_hash = content_hash++ # Build transforms applied list+ transforms: list[str] = ["content_analysis"]+ if existing_cache:+ transforms.append("cache_lookup")++ return CacheResult(+ messages=messages, # Messages unchanged - caching is separate+ semantic_cache_hit=False,+ metrics=metrics,+ tokens_before=total_tokens,+ tokens_after=total_tokens, # Token count doesn't change+ transforms_applied=transforms,+ warnings=warnings,+ )++ def analyze_cacheability(+ self,+ messages: list[dict[str, Any]],+ context: OptimizationContext,+ ) -> CacheabilityAnalysis:+ """+ Analyze content for Google cache suitability.++ Provides detailed analysis including:+ - Whether content meets minimum requirements+ - Estimated costs and savings+ - Recommendations for improving cacheability++ Args:+ messages: Messages to analyze+ context: Optimization context++ Returns:+ CacheabilityAnalysis with detailed information+ """+ cacheable_content = self._extract_cacheable_content(messages)+ content_hash = self._compute_prefix_hash(cacheable_content)++ total_tokens = self._count_tokens_estimate(+ self._messages_to_text(messages)+ )+ cacheable_tokens = self._count_tokens_estimate(cacheable_content)++ is_cacheable = cacheable_tokens >= GOOGLE_MIN_CACHE_TOKENS+ tokens_below_minimum = max(0, GOOGLE_MIN_CACHE_TOKENS - cacheable_tokens)++ # Build recommendations+ recommendations: list[str] = []++ if not is_cacheable:+ recommendations.append(+ f"Add {tokens_below_minimum:,} more tokens to static content "+ f"to enable caching"+ )+ recommendations.append(+ "Consider adding detailed examples or documentation to system prompt"+ )+ else:+ recommendations.append(+ f"Content is cacheable. Create cache with google-generativeai SDK"+ )++ # Storage cost estimation (rough - actual pricing varies)+ # Assuming ~$0.001 per 1000 tokens per hour (simplified)+ hourly_cost = (cacheable_tokens / 1000) * 0.001+ recommendations.append(+ f"Estimated storage cost: ~${hourly_cost:.4f}/hour"+ )++ # Break-even analysis+ if hourly_cost > 0:+ # Assuming $0.01 per 1000 input tokens base price+ base_cost_per_request = (cacheable_tokens / 1000) * 0.01+ savings_per_request = base_cost_per_request * GOOGLE_CACHE_DISCOUNT+ break_even_requests = hourly_cost / savings_per_request if savings_per_request > 0 else float('inf')+ recommendations.append(+ f"Break-even: ~{int(break_even_requests)} requests/hour"+ )++ return CacheabilityAnalysis(+ is_cacheable=is_cacheable,+ total_tokens=total_tokens,+ cacheable_tokens=cacheable_tokens,+ tokens_below_minimum=tokens_below_minimum,+ estimated_savings_per_request_percent=(+ GOOGLE_CACHE_DISCOUNT * 100 if is_cacheable else 0.0+ ),+ recommendations=recommendations,+ content_hash=content_hash,+ )++ # -------------------------------------------------------------------------+ # Cache Registry Management+ # -------------------------------------------------------------------------++ def register_cache(+ self,+ cache_id: str,+ content_hash: str,+ token_count: int,+ expires_at: datetime,+ *,+ model: str | None = None,+ display_name: str | None = None,+ metadata: dict[str, Any] | None = None,+ ) -> CachedContentInfo:+ """+ Register a cache after creating it via Google's API.++ Call this after successfully creating a CachedContent resource+ to enable cache reuse detection.++ Args:+ cache_id: Google's cache identifier (e.g., "cachedContents/xyz")+ content_hash: Hash of cached content (from optimize() metrics)+ token_count: Number of tokens in cached content+ expires_at: When the cache expires+ model: Optional model the cache was created for+ display_name: Optional display name+ metadata: Optional additional metadata++ Returns:+ CachedContentInfo for the registered cache++ Example:+ # After creating cache via Google SDK+ cached_content = genai.caching.CachedContent.create(...)++ info = optimizer.register_cache(+ cache_id=cached_content.name,+ content_hash=result.metrics.stable_prefix_hash,+ token_count=result.metrics.cacheable_tokens,+ expires_at=datetime.now() + timedelta(hours=1),+ )+ """+ # Remove any existing cache with same content hash+ old_cache = self._cache_registry.get(content_hash)+ if old_cache:+ self._cache_by_id.pop(old_cache.cache_id, None)+ logger.debug(+ f"Replacing existing cache for hash {content_hash}: "+ f"{old_cache.cache_id} -> {cache_id}"+ )++ cache_info = CachedContentInfo(+ cache_id=cache_id,+ content_hash=content_hash,+ created_at=datetime.now(),+ expires_at=expires_at,+ token_count=token_count,+ model=model,+ display_name=display_name,+ metadata=metadata or {},+ )++ self._cache_registry[content_hash] = cache_info+ self._cache_by_id[cache_id] = cache_info+ self._caches_created += 1++ logger.info(+ f"Registered cache {cache_id} with {token_count:,} tokens, "+ f"expires in {cache_info.ttl_remaining_seconds}s"+ )++ return cache_info++ def get_reusable_cache(+ self,+ content_hash: str,+ *,+ min_ttl_seconds: int = 60,+ ) -> CachedContentInfo | None:+ """+ Check if a reusable cache exists for the given content.++ Args:+ content_hash: Hash of the content to look up+ min_ttl_seconds: Minimum remaining TTL to consider reusable++ Returns:+ CachedContentInfo if reusable cache exists, None otherwise+ """+ cache_info = self._cache_registry.get(content_hash)++ if cache_info is None:+ return None++ if cache_info.is_expired:+ self._remove_cache_internal(content_hash)+ return None++ if cache_info.ttl_remaining_seconds < min_ttl_seconds:+ logger.debug(+ f"Cache {cache_info.cache_id} has insufficient TTL "+ f"({cache_info.ttl_remaining_seconds}s < {min_ttl_seconds}s)"+ )+ return None++ return cache_info++ def get_cache_by_id(self, cache_id: str) -> CachedContentInfo | None:+ """+ Look up cache information by cache ID.++ Args:+ cache_id: Google's cache identifier++ Returns:+ CachedContentInfo if found, None otherwise+ """+ return self._cache_by_id.get(cache_id)++ def extend_cache_ttl(+ self,+ cache_id: str,+ new_expires_at: datetime,+ ) -> CachedContentInfo | None:+ """+ Update the expiry time for a cache after extending via Google API.++ Call this after successfully calling update() on the CachedContent+ to extend its TTL.++ Args:+ cache_id: Google's cache identifier+ new_expires_at: New expiry time++ Returns:+ Updated CachedContentInfo or None if not found++ Example:+ # After extending via Google SDK+ cached_content.update(ttl=timedelta(hours=2))++ optimizer.extend_cache_ttl(+ cache_id=cached_content.name,+ new_expires_at=datetime.now() + timedelta(hours=2),+ )+ """+ cache_info = self._cache_by_id.get(cache_id)+ if cache_info is None:+ logger.warning(f"Cannot extend unknown cache: {cache_id}")+ return None++ old_expires = cache_info.expires_at+ cache_info.expires_at = new_expires_at++ logger.info(+ f"Extended cache {cache_id} TTL from {old_expires} to {new_expires_at}"+ )++ return cache_info++ def remove_cache(self, cache_id: str) -> bool:+ """+ Remove a cache from the registry.++ Call this after deleting the cache via Google API.++ Args:+ cache_id: Google's cache identifier++ Returns:+ True if cache was removed, False if not found+ """+ cache_info = self._cache_by_id.get(cache_id)+ if cache_info is None:+ return False++ self._cache_by_id.pop(cache_id, None)+ self._cache_registry.pop(cache_info.content_hash, None)++ logger.info(f"Removed cache {cache_id} from registry")+ return True++ def _remove_cache_internal(self, content_hash: str) -> None:+ """Remove cache by content hash (internal use)."""+ cache_info = self._cache_registry.pop(content_hash, None)+ if cache_info:+ self._cache_by_id.pop(cache_info.cache_id, None)+ self._caches_expired += 1++ def cleanup_expired_caches(self) -> list[str]:+ """+ Remove all expired caches from the registry.++ Returns:+ List of removed cache IDs (for user to delete via Google API)++ Example:+ expired_ids = optimizer.cleanup_expired_caches()+ for cache_id in expired_ids:+ # User deletes via Google SDK+ genai.caching.CachedContent.get(cache_id).delete()+ """+ expired_ids: list[str] = []++ # Find expired caches+ for content_hash, cache_info in list(self._cache_registry.items()):+ if cache_info.is_expired:+ expired_ids.append(cache_info.cache_id)+ self._remove_cache_internal(content_hash)++ if expired_ids:+ logger.info(f"Cleaned up {len(expired_ids)} expired caches")++ return expired_ids++ def list_caches(+ self,+ *,+ include_expired: bool = False,+ ) -> list[CachedContentInfo]:+ """+ List all registered caches.++ Args:+ include_expired: Whether to include expired caches++ Returns:+ List of CachedContentInfo objects+ """+ caches = list(self._cache_registry.values())++ if not include_expired:+ caches = [c for c in caches if not c.is_expired]++ # Sort by expiry time+ caches.sort(key=lambda c: c.expires_at)++ return caches++ def get_statistics(self) -> dict[str, Any]:+ """+ Get cache usage statistics.++ Returns:+ Dictionary with cache statistics+ """+ active_caches = [c for c in self._cache_registry.values() if not c.is_expired]+ total_cached_tokens = sum(c.token_count for c in active_caches)++ return {+ "active_caches": len(active_caches),+ "total_cached_tokens": total_cached_tokens,+ "caches_created": self._caches_created,+ "caches_reused": self._caches_reused,+ "caches_expired": self._caches_expired,+ "cache_hit_rate": (+ self._caches_reused / (self._caches_reused + self._caches_created)+ if (self._caches_reused + self._caches_created) > 0+ else 0.0+ ),+ }++ # -------------------------------------------------------------------------+ # Cache Creation Helpers+ # -------------------------------------------------------------------------++ def prepare_cache_creation(+ self,+ messages: list[dict[str, Any]],+ context: OptimizationContext,+ ttl_seconds: int = GOOGLE_DEFAULT_TTL_SECONDS,+ ) -> dict[str, Any] | None:+ """+ Prepare parameters for creating a Google cache.++ Returns a dictionary with suggested parameters for+ genai.caching.CachedContent.create().++ Args:+ messages: Messages to cache+ context: Optimization context+ ttl_seconds: Desired TTL in seconds++ Returns:+ Dictionary with cache creation parameters, or None if not cacheable++ Example:+ params = optimizer.prepare_cache_creation(messages, context)+ if params:+ cached_content = genai.caching.CachedContent.create(**params)+ """+ analysis = self.analyze_cacheability(messages, context)++ if not analysis.is_cacheable:+ logger.debug(+ f"Content not cacheable: {analysis.tokens_below_minimum} "+ f"tokens below minimum"+ )+ return None++ cacheable_content = self._extract_cacheable_content(messages)++ return {+ "contents": cacheable_content,+ "ttl": timedelta(seconds=min(ttl_seconds, GOOGLE_MAX_TTL_SECONDS)),+ "display_name": f"headroom-cache-{analysis.content_hash[:8]}",+ "_headroom_metadata": {+ "content_hash": analysis.content_hash,+ "token_count": analysis.cacheable_tokens,+ "created_by": "headroom",+ },+ }++ def build_request_with_cache(+ self,+ messages: list[dict[str, Any]],+ cache_id: str,+ ) -> dict[str, Any]:+ """+ Build request parameters using an existing cache.++ Returns a dictionary suggesting how to structure the API call+ when using cached content.++ Args:+ messages: Full message list+ cache_id: Cache ID to use++ Returns:+ Dictionary with suggested request structure+ """+ # Extract only the non-cached (dynamic) content+ dynamic_messages = self._extract_dynamic_messages(messages)++ return {+ "cached_content": cache_id,+ "contents": dynamic_messages,+ "_headroom_note": (+ "Use cached_content parameter with GenerativeModel "+ "to leverage the cache"+ ),+ }++ # -------------------------------------------------------------------------+ # Content Extraction Helpers+ # -------------------------------------------------------------------------++ def _extract_cacheable_content(self, messages: list[dict[str, Any]]) -> str:+ """+ Extract content suitable for caching.++ Includes:+ - System messages+ - Static context (tools, examples)++ Excludes:+ - Recent conversation turns+ - Dynamic content (dates, user-specific data)+ """+ cacheable_parts: list[str] = []++ for msg in messages:+ role = msg.get("role", "")++ # System messages are always cacheable+ if role == "system":+ content = self._extract_message_content(msg)+ if content:+ cacheable_parts.append(content)++ # First few user/assistant turns with examples might be cacheable+ # but we're conservative - only include system by default++ return "\n\n".join(cacheable_parts)++ def _extract_dynamic_messages(+ self,+ messages: list[dict[str, Any]],+ ) -> list[dict[str, Any]]:+ """+ Extract messages that should NOT be cached.++ These are the conversation turns after the cached prefix.+ """+ dynamic: list[dict[str, Any]] = []++ for msg in messages:+ if msg.get("role") != "system":+ dynamic.append(msg)++ return dynamic++ def _extract_message_content(self, message: dict[str, Any]) -> str:+ """Extract text content from a message."""+ content = message.get("content", "")++ if isinstance(content, str):+ return content++ if isinstance(content, list):+ parts = []+ for block in content:+ if isinstance(block, dict):+ if block.get("type") == "text":+ parts.append(block.get("text", ""))+ elif isinstance(block, str):+ parts.append(block)+ return "\n".join(parts)++ return ""++ def _messages_to_text(self, messages: list[dict[str, Any]]) -> str:+ """Convert all messages to text for token counting."""+ parts = []+ for msg in messages:+ content = self._extract_message_content(msg)+ if content:+ parts.append(f"{msg.get('role', 'unknown')}: {content}")+ return "\n\n".join(parts)++ # -------------------------------------------------------------------------+ # Serialization for Persistence+ # -------------------------------------------------------------------------++ def export_cache_registry(self) -> list[dict[str, Any]]:+ """+ Export cache registry for persistence.++ Returns:+ List of cache info dictionaries+ """+ return [info.to_dict() for info in self._cache_registry.values()]++ def import_cache_registry(+ self,+ cache_data: list[dict[str, Any]],+ *,+ skip_expired: bool = True,+ ) -> int:+ """+ Import caches from persisted data.++ Args:+ cache_data: List of cache info dictionaries+ skip_expired: Whether to skip already-expired caches++ Returns:+ Number of caches imported+ """+ imported = 0++ for data in cache_data:+ try:+ cache_info = CachedContentInfo.from_dict(data)++ if skip_expired and cache_info.is_expired:+ continue++ self._cache_registry[cache_info.content_hash] = cache_info+ self._cache_by_id[cache_info.cache_id] = cache_info+ imported += 1++ except (KeyError, ValueError) as e:+ logger.warning(f"Failed to import cache entry: {e}")+ continue++ logger.info(f"Imported {imported} caches from persisted data")+ return importeddiff --git a/headroom/cache/openai.py b/headroom/cache/openai.pynew file mode 100644index 000000000..6f23806b2--- /dev/null+++ b/headroom/cache/openai.py@@ -0,0 +1,591 @@+"""+OpenAI Cache Optimizer.++This module implements cache optimization for OpenAI's automatic prefix caching.+Unlike Anthropic, OpenAI's caching is fully automatic - users cannot control what+gets cached. The only optimization strategy is to stabilize prefixes to maximize+cache hit rates.++OpenAI Caching Details:+ - Fully automatic - no explicit cache control available+ - 50% discount on cached input tokens+ - Requires prompts > 1024 tokens to activate+ - 5-60 minute TTL (varies based on usage patterns)+ - Cache is prefix-based - changes invalidate downstream cache++Optimization Strategy:+ Since we can't control caching explicitly, we focus on PREFIX_STABILIZATION:+ - Extract dynamic content (dates, timestamps) and move to end+ - Normalize whitespace for consistent hashing+ - Remove random IDs from system prompts+ - Track prefix stability to estimate cache hit probability++Dynamic Content Detection Tiers:+ - Tier 1 (regex): Always on, ~0ms - dates, UUIDs, timestamps+ - Tier 2 (ner): Optional, ~5-10ms - names, money, organizations+ - Tier 3 (semantic): Optional, ~20-50ms - volatile patterns via embeddings++Usage:+ # Default: regex only (fastest)+ optimizer = OpenAICacheOptimizer()++ # With NER (requires spacy)+ optimizer = OpenAICacheOptimizer(+ config=CacheConfig(dynamic_detection_tiers=["regex", "ner"])+ )++ # Full detection (requires spacy + sentence-transformers)+ optimizer = OpenAICacheOptimizer(+ config=CacheConfig(dynamic_detection_tiers=["regex", "ner", "semantic"])+ )+"""++from __future__ import annotations++import re+from copy import deepcopy+from dataclasses import dataclass, field+from typing import Any, Literal++from .base import (+ BaseCacheOptimizer,+ CacheConfig,+ CacheMetrics,+ CacheResult,+ CacheStrategy,+ OptimizationContext,+)+from .dynamic_detector import (+ DetectorConfig,+ DynamicContentDetector,+ DynamicSpan,+)+++@dataclass+class PrefixAnalysis:+ """+ Analysis of prefix stability.++ Used to determine likelihood of cache hits and track changes+ between requests.+ """++ # Hash of the stabilized prefix+ prefix_hash: str++ # Estimated token count of stable prefix+ stable_tokens: int++ # Dynamic content that was extracted+ dynamic_spans: list[DynamicSpan] = field(default_factory=list)++ # Whether prefix changed from previous request+ changed_from_previous: bool = False++ # Previous hash for comparison+ previous_hash: str | None = None++ # Detection processing time+ detection_time_ms: float = 0.0+++class OpenAICacheOptimizer(BaseCacheOptimizer):+ """+ Cache optimizer for OpenAI's automatic prefix caching.++ OpenAI automatically caches prompt prefixes for requests > 1024 tokens.+ Since caching is automatic, this optimizer focuses on maximizing cache+ hit rates by stabilizing prefixes.++ Key Optimizations:+ 1. Extract dynamic content (dates, times) and move to end of messages+ 2. Normalize whitespace for consistent formatting+ 3. Remove random IDs and timestamps from system prompts+ 4. Track prefix changes to estimate cache hit probability++ Usage:+ optimizer = OpenAICacheOptimizer()+ result = optimizer.optimize(messages, context)++ # Check if prefix was stable (likely cache hit)+ if not result.metrics.prefix_changed_from_previous:+ print("Likely cache hit - prefix unchanged")++ # Estimate savings+ savings = result.metrics.estimated_savings_percent+ print(f"Estimated savings: {savings:.1f}%")++ Attributes:+ name: Identifier for this optimizer+ provider: The provider this optimizer targets ("openai")+ strategy: Always CacheStrategy.PREFIX_STABILIZATION+ """++ # OpenAI-specific constants+ MIN_TOKENS_FOR_CACHING = 1024+ CACHE_DISCOUNT_PERCENT = 50.0++ def __init__(self, config: CacheConfig | None = None):+ """+ Initialize the OpenAI cache optimizer.++ Args:+ config: Optional cache configuration. If not provided,+ sensible defaults are used.++ The optimizer uses the DynamicContentDetector with configurable tiers:+ - "regex": Fast pattern matching (~0ms) - always on+ - "ner": Named Entity Recognition (~5-10ms) - requires spacy+ - "semantic": Embedding similarity (~20-50ms) - requires sentence-transformers++ Configure tiers via config.dynamic_detection_tiers.+ """+ super().__init__(config)++ # Initialize the tiered dynamic content detector+ detector_config = DetectorConfig(+ tiers=self.config.dynamic_detection_tiers, # type: ignore+ )+ self._detector = DynamicContentDetector(detector_config)++ @property+ def name(self) -> str:+ """Name of this optimizer."""+ return "openai-prefix-stabilizer"++ @property+ def provider(self) -> str:+ """Provider this optimizer is for."""+ return "openai"++ @property+ def strategy(self) -> CacheStrategy:+ """The caching strategy this optimizer uses."""+ return CacheStrategy.PREFIX_STABILIZATION++ def optimize(+ self,+ messages: list[dict[str, Any]],+ context: OptimizationContext,+ config: CacheConfig | None = None,+ ) -> CacheResult:+ """+ Optimize messages for OpenAI's prefix caching.++ This method stabilizes the message prefix to maximize cache hit rates.+ Since OpenAI caching is automatic, we focus on ensuring the prefix+ remains consistent across requests.++ Args:+ messages: List of message dictionaries in OpenAI format.+ context: Optimization context with request metadata.+ config: Optional configuration override.++ Returns:+ CacheResult containing:+ - Optimized messages with stabilized prefixes+ - Metrics about prefix stability and estimated savings+ - List of transforms applied+ - Any warnings encountered++ Example:+ >>> optimizer = OpenAICacheOptimizer()+ >>> messages = [+ ... {"role": "system", "content": "Today is Jan 1, 2024. You are helpful."},+ ... {"role": "user", "content": "Hello!"}+ ... ]+ >>> context = OptimizationContext(provider="openai", model="gpt-4")+ >>> result = optimizer.optimize(messages, context)+ >>> # Date moved to end, prefix stabilized+ """+ effective_config = config or self.config++ # Handle disabled optimization+ if not effective_config.enabled:+ return CacheResult(+ messages=messages,+ metrics=CacheMetrics(),+ transforms_applied=[],+ )++ # Deep copy to avoid mutating input+ optimized_messages = deepcopy(messages)+ transforms_applied: list[str] = []+ warnings: list[str] = []++ # Track all extracted spans across messages+ all_spans: list[DynamicSpan] = []+ total_detection_time = 0.0++ # Process system messages for prefix stabilization+ for i, msg in enumerate(optimized_messages):+ if msg.get("role") == "system":+ content = msg.get("content", "")++ if isinstance(content, str):+ # Use tiered dynamic content detector+ result = self._detector.detect(content)+ all_spans.extend(result.spans)+ total_detection_time += result.processing_time_ms++ # Add any detector warnings+ warnings.extend(result.warnings)++ if result.spans:+ transforms_applied.append(+ f"extracted_{len(result.spans)}_dynamic_elements"+ )+ transforms_applied.extend(+ f"tier_{tier}" for tier in result.tiers_used+ )++ # Get static content with dynamic parts removed+ stabilized = result.static_content++ # Normalize whitespace+ if effective_config.normalize_whitespace:+ stabilized = self._normalize_whitespace(+ stabilized,+ collapse_blank_lines=effective_config.collapse_blank_lines,+ )+ transforms_applied.append("normalized_whitespace")++ # If we extracted dynamic content, append it at the end+ if result.dynamic_content:+ dynamic_section = self._format_dynamic_section(+ result.dynamic_content,+ separator=effective_config.dynamic_separator,+ )+ stabilized = stabilized.rstrip() + dynamic_section++ optimized_messages[i]["content"] = stabilized++ elif isinstance(content, list):+ # Handle content blocks (less common for OpenAI)+ new_content = []+ for block in content:+ if isinstance(block, dict) and block.get("type") == "text":+ text = block.get("text", "")+ result = self._detector.detect(text)+ all_spans.extend(result.spans)+ total_detection_time += result.processing_time_ms+ warnings.extend(result.warnings)++ stabilized = result.static_content++ if effective_config.normalize_whitespace:+ stabilized = self._normalize_whitespace(stabilized)++ if result.dynamic_content:+ dynamic_section = self._format_dynamic_section(+ result.dynamic_content,+ separator=effective_config.dynamic_separator,+ )+ stabilized = stabilized.rstrip() + dynamic_section++ new_content.append({**block, "text": stabilized})+ else:+ new_content.append(block)++ optimized_messages[i]["content"] = new_content+ if all_spans:+ transforms_applied.append("processed_content_blocks")++ # Analyze prefix stability+ analysis = self._analyze_prefix(optimized_messages, context)++ # Calculate token estimates+ tokens_before = self._estimate_total_tokens(messages)+ tokens_after = self._estimate_total_tokens(optimized_messages)++ # Build metrics+ metrics = CacheMetrics(+ stable_prefix_tokens=analysis.stable_tokens,+ stable_prefix_hash=analysis.prefix_hash,+ prefix_changed_from_previous=analysis.changed_from_previous,+ previous_prefix_hash=analysis.previous_hash,+ estimated_cache_hit=not analysis.changed_from_previous,+ cacheable_tokens=self._calculate_cacheable_tokens(analysis.stable_tokens),+ non_cacheable_tokens=max(0, tokens_after - analysis.stable_tokens),+ estimated_savings_percent=self._calculate_savings_percent(+ analysis.stable_tokens,+ tokens_after,+ likely_cache_hit=not analysis.changed_from_previous,+ ),+ )++ # Add warnings for suboptimal cases+ if tokens_after < self.MIN_TOKENS_FOR_CACHING:+ warnings.append(+ f"Prompt has ~{tokens_after} tokens, below OpenAI's {self.MIN_TOKENS_FOR_CACHING} "+ f"token minimum for caching. Consider adding more static context."+ )++ if analysis.changed_from_previous:+ warnings.append(+ "Prefix changed from previous request - cache miss likely. "+ "Consider reviewing what content is changing between requests."+ )++ # Record metrics and update state+ self._record_metrics(metrics)+ self._previous_prefix_hash = analysis.prefix_hash++ return CacheResult(+ messages=optimized_messages,+ metrics=metrics,+ tokens_before=tokens_before,+ tokens_after=tokens_after,+ transforms_applied=list(set(transforms_applied)), # Dedupe+ warnings=warnings,+ )++ def estimate_savings(+ self,+ messages: list[dict[str, Any]],+ context: OptimizationContext,+ ) -> float:+ """+ Estimate potential cost savings from caching.++ OpenAI provides 50% discount on cached tokens. This method estimates+ what portion of tokens are likely to be cached based on prefix+ stability and token count.++ Args:+ messages: Messages to analyze.+ context: Optimization context.++ Returns:+ Estimated savings as a percentage (0-100).+ Returns 0 if prompt is below caching threshold.++ Example:+ >>> savings = optimizer.estimate_savings(messages, context)+ >>> print(f"Potential savings: {savings:.1f}%")+ """+ total_tokens = self._estimate_total_tokens(messages)++ # No savings if below threshold+ if total_tokens < self.MIN_TOKENS_FOR_CACHING:+ return 0.0++ # Extract system content for prefix analysis+ system_content = self._extract_system_content(messages)+ system_tokens = self._count_tokens_estimate(system_content)++ # Estimate cacheable portion (system + early messages)+ # OpenAI caches the longest matching prefix+ cacheable_ratio = min(1.0, system_tokens / total_tokens)++ # Check if prefix is stable+ current_hash = self._compute_prefix_hash(system_content)+ likely_hit = (+ self._previous_prefix_hash is not None+ and current_hash == self._previous_prefix_hash+ )++ if likely_hit:+ # 50% savings on cacheable portion+ return cacheable_ratio * self.CACHE_DISCOUNT_PERCENT+ else:+ # First request or prefix changed - no immediate savings+ # but return expected savings for future requests+ return cacheable_ratio * self.CACHE_DISCOUNT_PERCENT * 0.5++ def _normalize_whitespace(+ self,+ content: str,+ collapse_blank_lines: bool = True,+ ) -> str:+ """+ Normalize whitespace in content.++ Ensures consistent whitespace formatting to improve prefix matching.+ This helps when the same logical content has minor formatting differences.++ Args:+ content: Text to normalize.+ collapse_blank_lines: If True, multiple blank lines become one.++ Returns:+ Content with normalized whitespace.+ """+ # Normalize line endings+ result = content.replace("\r\n", "\n").replace("\r", "\n")++ # Collapse multiple spaces (but preserve indentation)+ lines = result.split("\n")+ normalized_lines = []++ for line in lines:+ # Preserve leading whitespace, normalize trailing+ stripped = line.rstrip()+ if stripped:+ # Find leading whitespace+ leading = len(line) - len(line.lstrip())+ # Collapse multiple spaces in content (not indentation)+ content_part = " ".join(stripped.split())+ normalized_lines.append(" " * leading + content_part[leading:] if leading else content_part)+ else:+ normalized_lines.append("")++ result = "\n".join(normalized_lines)++ # Collapse multiple blank lines+ if collapse_blank_lines:+ while "\n\n\n" in result:+ result = result.replace("\n\n\n", "\n\n")++ return result.strip()++ def _format_dynamic_section(+ self,+ dynamic_content: str,+ separator: str = "\n\n---\n\n",+ ) -> str:+ """+ Format extracted dynamic content as a section to append.++ Creates a clearly marked section containing dynamic values,+ appended to the end of the message to preserve prefix stability.++ Args:+ dynamic_content: The dynamic content string to append.+ separator: Separator to use before the dynamic section.++ Returns:+ Formatted dynamic section string.+ """+ if not dynamic_content or not dynamic_content.strip():+ return ""++ # Format as a context section+ return f"{separator}[Current Context]\n{dynamic_content.strip()}\n"++ def _analyze_prefix(+ self,+ messages: list[dict[str, Any]],+ context: OptimizationContext,+ ) -> PrefixAnalysis:+ """+ Analyze the prefix for stability metrics.++ Computes hash of the stable prefix portion and compares with+ previous requests to estimate cache hit likelihood.++ Args:+ messages: Messages to analyze.+ context: Optimization context with previous hash.++ Returns:+ PrefixAnalysis with stability metrics.+ """+ # Extract prefix content (system messages + structure)+ prefix_parts = []++ for msg in messages:+ if msg.get("role") == "system":+ content = msg.get("content", "")+ if isinstance(content, str):+ prefix_parts.append(content)+ elif isinstance(content, list):+ for block in content:+ if isinstance(block, dict) and block.get("type") == "text":+ prefix_parts.append(block.get("text", ""))++ prefix_content = "\n".join(prefix_parts)+ prefix_hash = self._compute_prefix_hash(prefix_content)+ stable_tokens = self._count_tokens_estimate(prefix_content)++ # Check for changes from previous request+ previous_hash = context.previous_prefix_hash or self._previous_prefix_hash+ changed = previous_hash is not None and prefix_hash != previous_hash++ return PrefixAnalysis(+ prefix_hash=prefix_hash,+ stable_tokens=stable_tokens,+ changed_from_previous=changed,+ previous_hash=previous_hash,+ )++ def _calculate_cacheable_tokens(self, stable_prefix_tokens: int) -> int:+ """+ Calculate how many tokens are likely cacheable.++ OpenAI only caches prompts > 1024 tokens, and caches in chunks.++ Args:+ stable_prefix_tokens: Number of tokens in stable prefix.++ Returns:+ Estimated cacheable token count.+ """+ if stable_prefix_tokens < self.MIN_TOKENS_FOR_CACHING:+ return 0++ # OpenAI caches in 128-token chunks (aligned)+ # Return the aligned cacheable amount+ return (stable_prefix_tokens // 128) * 128++ def _calculate_savings_percent(+ self,+ stable_tokens: int,+ total_tokens: int,+ likely_cache_hit: bool,+ ) -> float:+ """+ Calculate estimated savings percentage.++ Args:+ stable_tokens: Tokens in stable prefix.+ total_tokens: Total tokens in request.+ likely_cache_hit: Whether a cache hit is likely.++ Returns:+ Estimated savings as percentage (0-100).+ """+ if total_tokens == 0:+ return 0.0++ cacheable = self._calculate_cacheable_tokens(stable_tokens)+ if cacheable == 0:+ return 0.0++ cacheable_ratio = cacheable / total_tokens++ if likely_cache_hit:+ # Full 50% savings on cacheable portion+ return cacheable_ratio * self.CACHE_DISCOUNT_PERCENT+ else:+ # No savings on first request, but show potential+ return 0.0++ def _estimate_total_tokens(self, messages: list[dict[str, Any]]) -> int:+ """+ Estimate total tokens in messages.++ Args:+ messages: Messages to count.++ Returns:+ Estimated token count.+ """+ total = 0+ for msg in messages:+ content = msg.get("content", "")+ if isinstance(content, str):+ total += self._count_tokens_estimate(content)+ elif isinstance(content, list):+ for block in content:+ if isinstance(block, dict):+ if block.get("type") == "text":+ total += self._count_tokens_estimate(+ block.get("text", "")+ )+ elif block.get("type") == "image_url":+ # Rough estimate for images+ total += 85 # Base cost+ return total+diff --git a/headroom/cache/registry.py b/headroom/cache/registry.pynew file mode 100644index 000000000..26856052e--- /dev/null+++ b/headroom/cache/registry.py@@ -0,0 +1,181 @@+"""+Cache Optimizer Registry.++Provides a plugin system for registering and retrieving cache optimizers.+This allows users to swap implementations and register custom optimizers.+"""++from __future__ import annotations++from typing import Type++from .base import CacheOptimizer, BaseCacheOptimizer, CacheConfig+++class CacheOptimizerRegistry:+ """+ Registry for cache optimizer plugins.++ This registry allows:+ - Registration of custom optimizers+ - Retrieval by provider name+ - Tier-based selection (oss vs enterprise)++ Usage:+ # Get default optimizer for provider+ optimizer = CacheOptimizerRegistry.get("anthropic")++ # Get enterprise version if available+ optimizer = CacheOptimizerRegistry.get("anthropic", tier="enterprise")++ # Register custom optimizer+ CacheOptimizerRegistry.register("my-provider", MyOptimizer)+ """++ _optimizers: dict[str, Type[BaseCacheOptimizer]] = {}+ _instances: dict[str, BaseCacheOptimizer] = {}++ @classmethod+ def register(+ cls,+ name: str,+ optimizer_class: Type[BaseCacheOptimizer],+ *,+ override: bool = False,+ ) -> None:+ """+ Register a cache optimizer.++ Args:+ name: Name to register under (e.g., "anthropic", "anthropic-enterprise")+ optimizer_class: The optimizer class to register+ override: Whether to override existing registration++ Raises:+ ValueError: If name already registered and override=False+ """+ if name in cls._optimizers and not override:+ raise ValueError(+ f"Optimizer '{name}' already registered. "+ f"Use override=True to replace."+ )+ cls._optimizers[name] = optimizer_class+ # Clear cached instance if exists+ cls._instances.pop(name, None)++ @classmethod+ def unregister(cls, name: str) -> None:+ """+ Unregister a cache optimizer.++ Args:+ name: Name to unregister+ """+ cls._optimizers.pop(name, None)+ cls._instances.pop(name, None)++ @classmethod+ def get(+ cls,+ provider: str,+ tier: str = "oss",+ config: CacheConfig | None = None,+ *,+ cached: bool = True,+ ) -> BaseCacheOptimizer:+ """+ Get a cache optimizer for a provider.++ Args:+ provider: Provider name (e.g., "anthropic", "openai", "google")+ tier: Tier to get ("oss" or "enterprise")+ config: Optional configuration+ cached: Whether to return cached instance++ Returns:+ Cache optimizer instance++ Raises:+ KeyError: If no optimizer registered for provider/tier+ """+ # Build the lookup key+ if tier != "oss":+ key = f"{provider}-{tier}"+ # Fall back to OSS if enterprise not available+ if key not in cls._optimizers:+ key = provider+ else:+ key = provider++ if key not in cls._optimizers:+ available = list(cls._optimizers.keys())+ raise KeyError(+ f"No optimizer registered for '{key}'. "+ f"Available: {available}"+ )++ # Return cached instance if requested+ cache_key = f"{key}:{id(config)}" if config else key+ if cached and cache_key in cls._instances:+ return cls._instances[cache_key]++ # Create new instance+ optimizer_class = cls._optimizers[key]+ instance = optimizer_class(config)++ if cached:+ cls._instances[cache_key] = instance++ return instance++ @classmethod+ def list_providers(cls) -> list[str]:+ """List all registered provider names (excluding tier suffixes)."""+ providers = set()+ for name in cls._optimizers:+ # Remove tier suffix if present+ base_name = name.split("-")[0]+ providers.add(base_name)+ return sorted(providers)++ @classmethod+ def list_all(cls) -> list[str]:+ """List all registered optimizer names."""+ return sorted(cls._optimizers.keys())++ @classmethod+ def is_registered(cls, name: str) -> bool:+ """Check if an optimizer is registered."""+ return name in cls._optimizers++ @classmethod+ def clear(cls) -> None:+ """Clear all registrations. Mainly for testing."""+ cls._optimizers.clear()+ cls._instances.clear()++ @classmethod+ def reset_to_defaults(cls) -> None:+ """Reset to default registrations."""+ cls.clear()+ _register_defaults()+++def _register_defaults() -> None:+ """Register default optimizers."""+ # Import here to avoid circular imports+ from .anthropic import AnthropicCacheOptimizer+ from .openai import OpenAICacheOptimizer+ from .google import GoogleCacheOptimizer++ CacheOptimizerRegistry.register("anthropic", AnthropicCacheOptimizer)+ CacheOptimizerRegistry.register("openai", OpenAICacheOptimizer)+ CacheOptimizerRegistry.register("google", GoogleCacheOptimizer)+++# Auto-register defaults on module import+# Wrapped in try/except to allow partial imports during development+try:+ _register_defaults()+except ImportError:+ passdiff --git a/headroom/cache/semantic.py b/headroom/cache/semantic.pynew file mode 100644index 000000000..17ecafec1--- /dev/null+++ b/headroom/cache/semantic.py@@ -0,0 +1,447 @@+"""+Semantic Cache Layer.++Provides query-level semantic caching using embedding similarity.+This is COMPLEMENTARY to provider prompt caching - it caches complete+responses for semantically similar queries.++How it works:+1. When a query comes in, compute its embedding+2. Search for similar queries in the cache (cosine similarity)+3. If similarity > threshold, return cached response+4. Otherwise, proceed with normal optimization++Key difference from Prompt Caching:+- Prompt Caching: Provider caches KV-cache for prefix (same prompt = faster)+- Semantic Caching: We cache responses for similar queries (similar query = cached answer)++Usage:+ from headroom.cache import SemanticCacheLayer, CacheOptimizerRegistry++ # Get provider optimizer+ provider_optimizer = CacheOptimizerRegistry.get("anthropic")++ # Wrap with semantic layer+ semantic = SemanticCacheLayer(+ provider_optimizer,+ similarity_threshold=0.95,+ )++ result = semantic.process(messages, context)+ if result.semantic_cache_hit:+ # Use result.cached_response directly+ pass+"""++from __future__ import annotations++import hashlib+import time+from collections import OrderedDict+from dataclasses import dataclass, field+from typing import Any, Callable++from .base import (+ BaseCacheOptimizer,+ CacheConfig,+ CacheMetrics,+ CacheResult,+ OptimizationContext,+)+++@dataclass+class CacheEntry:+ """Entry in the semantic cache."""++ # Query embedding+ embedding: list[float]++ # Original query text+ query: str++ # Cached response+ response: Any++ # Metadata+ created_at: float+ last_accessed: float+ access_count: int = 1++ # Hash of the full messages for exact matching+ messages_hash: str = ""+++@dataclass+class SemanticCacheConfig:+ """Configuration for semantic caching."""++ # Similarity threshold for cache hit (0.0 - 1.0)+ similarity_threshold: float = 0.95++ # Maximum entries in cache+ max_entries: int = 1000++ # TTL in seconds (0 = no expiry)+ ttl_seconds: int = 300++ # Whether to use exact hash matching as fallback+ use_exact_matching: bool = True++ # Embedding model (if using embeddings)+ embedding_model: str = "all-MiniLM-L6-v2"+++class SemanticCache:+ """+ In-memory semantic cache with LRU eviction.++ Stores query embeddings and responses, supporting both+ semantic similarity search and exact hash matching.+ """++ def __init__(+ self,+ config: SemanticCacheConfig | None = None,+ embedding_fn: Callable[[str], list[float]] | None = None,+ ):+ """+ Initialize the semantic cache.++ Args:+ config: Cache configuration+ embedding_fn: Optional custom embedding function.+ If not provided, uses simple hash-based matching.+ """+ self.config = config or SemanticCacheConfig()+ self._embedding_fn = embedding_fn++ # LRU cache: key -> CacheEntry+ self._cache: OrderedDict[str, CacheEntry] = OrderedDict()++ # Exact hash index: messages_hash -> key+ self._hash_index: dict[str, str] = {}++ # Statistics+ self._hits = 0+ self._misses = 0+ self._evictions = 0++ def get(+ self,+ query: str,+ messages_hash: str | None = None,+ ) -> CacheEntry | None:+ """+ Look up a cached entry.++ Args:+ query: Query text to search for+ messages_hash: Optional exact hash for fast lookup++ Returns:+ CacheEntry if found, None otherwise+ """+ self._cleanup_expired()++ # Try exact hash match first+ if messages_hash and self.config.use_exact_matching:+ key = self._hash_index.get(messages_hash)+ if key and key in self._cache:+ entry = self._cache[key]+ self._touch(key)+ self._hits += 1+ return entry++ # Try semantic similarity if we have embedding function+ if self._embedding_fn:+ query_embedding = self._embedding_fn(query)+ best_match, best_similarity = self._find_similar(query_embedding)++ if best_similarity >= self.config.similarity_threshold:+ self._touch(best_match)+ self._hits += 1+ return self._cache[best_match]++ self._misses += 1+ return None++ def put(+ self,+ query: str,+ response: Any,+ messages_hash: str | None = None,+ ) -> str:+ """+ Store a response in the cache.++ Args:+ query: Query text+ response: Response to cache+ messages_hash: Optional exact hash for fast lookup++ Returns:+ Cache key for the entry+ """+ self._cleanup_expired()++ # Evict if at capacity+ while len(self._cache) >= self.config.max_entries:+ self._evict_oldest()++ # Generate embedding if available+ embedding: list[float] = []+ if self._embedding_fn:+ embedding = self._embedding_fn(query)++ # Create cache key+ key = self._generate_key(query)++ now = time.time()+ entry = CacheEntry(+ embedding=embedding,+ query=query,+ response=response,+ created_at=now,+ last_accessed=now,+ messages_hash=messages_hash or "",+ )++ self._cache[key] = entry++ # Index by hash for fast exact matching+ if messages_hash:+ self._hash_index[messages_hash] = key++ return key++ def invalidate(self, key: str) -> bool:+ """Invalidate a cache entry by key."""+ if key in self._cache:+ entry = self._cache.pop(key)+ if entry.messages_hash:+ self._hash_index.pop(entry.messages_hash, None)+ return True+ return False++ def clear(self) -> None:+ """Clear all cache entries."""+ self._cache.clear()+ self._hash_index.clear()++ def get_stats(self) -> dict[str, Any]:+ """Get cache statistics."""+ total = self._hits + self._misses+ hit_rate = self._hits / total if total > 0 else 0.0++ return {+ "entries": len(self._cache),+ "max_entries": self.config.max_entries,+ "hits": self._hits,+ "misses": self._misses,+ "hit_rate": hit_rate,+ "evictions": self._evictions,+ }++ def _find_similar(+ self,+ query_embedding: list[float],+ ) -> tuple[str, float]:+ """Find the most similar cached entry."""+ best_key = ""+ best_similarity = -1.0++ for key, entry in self._cache.items():+ if not entry.embedding:+ continue++ similarity = self._cosine_similarity(query_embedding, entry.embedding)+ if similarity > best_similarity:+ best_similarity = similarity+ best_key = key++ return best_key, best_similarity++ def _cosine_similarity(+ self,+ a: list[float],+ b: list[float],+ ) -> float:+ """Compute cosine similarity between two vectors."""+ if len(a) != len(b) or not a:+ return 0.0++ dot_product = sum(x * y for x, y in zip(a, b))+ norm_a = sum(x * x for x in a) ** 0.5+ norm_b = sum(x * x for x in b) ** 0.5++ if norm_a == 0 or norm_b == 0:+ return 0.0++ return dot_product / (norm_a * norm_b)++ def _touch(self, key: str) -> None:+ """Update access time and move to end of LRU."""+ if key in self._cache:+ entry = self._cache.pop(key)+ entry.last_accessed = time.time()+ entry.access_count += 1+ self._cache[key] = entry++ def _evict_oldest(self) -> None:+ """Evict the oldest (least recently used) entry."""+ if self._cache:+ key, entry = self._cache.popitem(last=False)+ if entry.messages_hash:+ self._hash_index.pop(entry.messages_hash, None)+ self._evictions += 1++ def _cleanup_expired(self) -> None:+ """Remove expired entries."""+ if self.config.ttl_seconds <= 0:+ return++ now = time.time()+ expired = [+ key for key, entry in self._cache.items()+ if now - entry.created_at > self.config.ttl_seconds+ ]++ for key in expired:+ entry = self._cache.pop(key)+ if entry.messages_hash:+ self._hash_index.pop(entry.messages_hash, None)++ def _generate_key(self, query: str) -> str:+ """Generate a cache key for a query."""+ return hashlib.sha256(query.encode()).hexdigest()[:16]+++class SemanticCacheLayer:+ """+ Layer that adds semantic caching on top of provider optimizers.++ This layer checks for semantically similar queries before+ delegating to the underlying provider optimizer.+ """++ def __init__(+ self,+ provider_optimizer: BaseCacheOptimizer,+ similarity_threshold: float = 0.95,+ max_entries: int = 1000,+ ttl_seconds: int = 300,+ embedding_fn: Callable[[str], list[float]] | None = None,+ ):+ """+ Initialize the semantic cache layer.++ Args:+ provider_optimizer: Underlying provider optimizer+ similarity_threshold: Similarity threshold for cache hits+ max_entries: Maximum cache entries+ ttl_seconds: Cache TTL in seconds+ embedding_fn: Optional embedding function+ """+ self.provider_optimizer = provider_optimizer++ cache_config = SemanticCacheConfig(+ similarity_threshold=similarity_threshold,+ max_entries=max_entries,+ ttl_seconds=ttl_seconds,+ )+ self.cache = SemanticCache(cache_config, embedding_fn)++ def process(+ self,+ messages: list[dict[str, Any]],+ context: OptimizationContext,+ config: CacheConfig | None = None,+ ) -> CacheResult:+ """+ Process messages through semantic cache and provider optimizer.++ Args:+ messages: Messages to process+ context: Optimization context+ config: Optional configuration override++ Returns:+ CacheResult with semantic_cache_hit=True if cache hit+ """+ # Extract query for semantic matching+ query = context.query or self._extract_query(messages)+ messages_hash = self._compute_messages_hash(messages)++ # Check semantic cache+ cached = self.cache.get(query, messages_hash)+ if cached:+ return CacheResult(+ messages=messages,+ semantic_cache_hit=True,+ cached_response=cached.response,+ metrics=CacheMetrics(+ estimated_cache_hit=True,+ estimated_savings_percent=100.0,+ ),+ transforms_applied=["semantic_cache_hit"],+ )++ # Delegate to provider optimizer+ result = self.provider_optimizer.optimize(messages, context, config)++ return result++ def store_response(+ self,+ messages: list[dict[str, Any]],+ response: Any,+ context: OptimizationContext | None = None,+ ) -> str:+ """+ Store a response in the semantic cache.++ Call this after receiving a response from the LLM to enable+ future cache hits.++ Args:+ messages: Original messages+ response: Response from LLM+ context: Optional context with query++ Returns:+ Cache key+ """+ query = (context.query if context else None) or self._extract_query(messages)+ messages_hash = self._compute_messages_hash(messages)++ return self.cache.put(query, response, messages_hash)++ def get_stats(self) -> dict[str, Any]:+ """Get combined statistics."""+ return {+ "semantic_cache": self.cache.get_stats(),+ "provider_optimizer": self.provider_optimizer.name,+ }++ def _extract_query(self, messages: list[dict[str, Any]]) -> str:+ """Extract the last user query from messages."""+ for msg in reversed(messages):+ if msg.get("role") == "user":+ content = msg.get("content", "")+ if isinstance(content, str):+ return content+ elif isinstance(content, list):+ for block in content:+ if isinstance(block, dict) and block.get("type") == "text":+ return block.get("text", "")+ return ""++ def _compute_messages_hash(self, messages: list[dict[str, Any]]) -> str:+ """Compute a hash of all messages."""+ import json+ try:+ content = json.dumps(messages, sort_keys=True)+ return hashlib.sha256(content.encode()).hexdigest()[:24]+ except (TypeError, ValueError):+ return ""diff --git a/headroom/client.py b/headroom/client.pyindex a77cda7b2..900675e82 100644--- a/headroom/client.py+++ b/headroom/client.py@@ -6,6 +6,13 @@ from collections.abc import Iteratorfrom datetime import datetimefrom typing import Any+from .cache import (+ BaseCacheOptimizer,+ CacheOptimizerRegistry,+ CacheConfig,+ OptimizationContext,+ SemanticCacheLayer,+)from .config import (HeadroomConfig,HeadroomMode,@@ -256,6 +263,9 @@ class HeadroomClient:store_url: str = "sqlite:///headroom.db",default_mode: str = "audit",model_context_limits: dict[str, int] | None = None,+ cache_optimizer: BaseCacheOptimizer | None = None,+ enable_cache_optimizer: bool = True,+ enable_semantic_cache: bool = False,):"""Initialize HeadroomClient.@@ -266,6 +276,10 @@ class HeadroomClient:store_url: Storage URL (sqlite:// or jsonl://).default_mode: Default mode ("audit" | "optimize").model_context_limits: Override context limits for models.+ cache_optimizer: Optional custom cache optimizer. If None and+ enable_cache_optimizer=True, auto-detects from provider.+ enable_cache_optimizer: Enable provider-specific cache optimization.+ enable_semantic_cache: Enable query-level semantic caching."""self._original = original_clientself._provider = provider@@ -276,6 +290,8 @@ class HeadroomClient:self._config = HeadroomConfig()self._config.store_url = store_urlself._config.default_mode = self._default_mode+ self._config.cache_optimizer.enabled = enable_cache_optimizer+ self._config.cache_optimizer.enable_semantic_cache = enable_semantic_cacheif model_context_limits:self._config.model_context_limits.update(model_context_limits)@@ -286,6 +302,34 @@ class HeadroomClient:# Initialize transform pipelineself._pipeline = TransformPipeline(self._config, provider=self._provider)+ # Initialize cache optimizer+ self._cache_optimizer: BaseCacheOptimizer | None = None+ self._semantic_cache_layer: SemanticCacheLayer | None = None++ if enable_cache_optimizer:+ if cache_optimizer is not None:+ self._cache_optimizer = cache_optimizer+ else:+ # Auto-detect from provider+ provider_name = self._provider.name.lower()+ if CacheOptimizerRegistry.is_registered(provider_name):+ cache_config = CacheConfig(+ min_cacheable_tokens=self._config.cache_optimizer.min_cacheable_tokens,+ )+ self._cache_optimizer = CacheOptimizerRegistry.get(+ provider_name,+ config=cache_config,+ )++ # Wrap with semantic cache if enabled+ if enable_semantic_cache and self._cache_optimizer is not None:+ self._semantic_cache_layer = SemanticCacheLayer(+ self._cache_optimizer,+ similarity_threshold=self._config.cache_optimizer.semantic_cache_similarity,+ max_entries=self._config.cache_optimizer.semantic_cache_max_entries,+ ttl_seconds=self._config.cache_optimizer.semantic_cache_ttl_seconds,+ )+# Public API - OpenAI styleself.chat = type("Chat", (), {"completions": ChatCompletions(self)})()# Public API - Anthropic style@@ -339,6 +383,16 @@ class HeadroomClient:# Compute stable prefix hashstable_prefix_hash = compute_prefix_hash(messages)+ # Cache optimizer metrics (populated later if optimizer is used)+ cache_optimizer_used = None+ cache_optimizer_strategy = None+ cacheable_tokens = 0+ breakpoints_inserted = 0+ estimated_cache_hit = False+ estimated_savings_percent = 0.0+ semantic_cache_hit = False+ cached_response = None+# Apply transforms if in optimize modeif mode == HeadroomMode.OPTIMIZE:output_buffer = headroom_output_buffer_tokens or self._config.rolling_window.output_buffer_tokens@@ -356,6 +410,54 @@ class HeadroomClient:tokens_after = result.tokens_aftertransforms_applied = result.transforms_applied+ # Apply provider-specific cache optimization+ if self._cache_optimizer is not None or self._semantic_cache_layer is not None:+ cache_context = OptimizationContext(+ provider=self._provider.name.lower(),+ model=model,+ query=self._extract_query(optimized_messages),+ )++ # Check semantic cache first (if enabled)+ if self._semantic_cache_layer is not None:+ cache_result = self._semantic_cache_layer.process(+ optimized_messages, cache_context+ )+ semantic_cache_hit = cache_result.semantic_cache_hit+ if semantic_cache_hit:+ cached_response = cache_result.cached_response++ # Update metrics from cache result+ cache_optimizer_used = cache_result.metrics.optimizer_name or self._cache_optimizer.name+ cache_optimizer_strategy = cache_result.metrics.strategy+ cacheable_tokens = cache_result.metrics.cacheable_tokens+ breakpoints_inserted = cache_result.metrics.breakpoints_inserted+ estimated_cache_hit = cache_result.metrics.estimated_cache_hit+ estimated_savings_percent = cache_result.metrics.estimated_savings_percent++ # Apply optimized messages (with cache_control blocks for Anthropic)+ if cache_result.messages:+ optimized_messages = cache_result.messages++ elif self._cache_optimizer is not None:+ # Direct cache optimizer (no semantic layer)+ cache_result = self._cache_optimizer.optimize(+ optimized_messages, cache_context+ )+ cache_optimizer_used = self._cache_optimizer.name+ cache_optimizer_strategy = self._cache_optimizer.strategy.value+ cacheable_tokens = cache_result.metrics.cacheable_tokens+ breakpoints_inserted = cache_result.metrics.breakpoints_inserted+ estimated_cache_hit = cache_result.metrics.estimated_cache_hit+ estimated_savings_percent = cache_result.metrics.estimated_savings_percent++ if cache_result.messages:+ optimized_messages = cache_result.messages++ transforms_applied.extend(+ f"cache_optimizer:{t}" for t in (cache_result.transforms_applied or [])+ )+# Recalculate prefix hash after optimizationstable_prefix_hash = compute_prefix_hash(optimized_messages)else:@@ -379,8 +481,21 @@ class HeadroomClient:cache_alignment_score=cache_alignment_score,transforms_applied=transforms_applied,messages_hash=compute_messages_hash(messages),+ # Cache optimizer metrics+ cache_optimizer_used=cache_optimizer_used,+ cache_optimizer_strategy=cache_optimizer_strategy,+ cacheable_tokens=cacheable_tokens,+ breakpoints_inserted=breakpoints_inserted,+ estimated_cache_hit=estimated_cache_hit,+ estimated_savings_percent=estimated_savings_percent,+ semantic_cache_hit=semantic_cache_hit,)+ # Return cached response if semantic cache hit+ if semantic_cache_hit and cached_response is not None:+ self._storage.save(metrics)+ return cached_response+# Call underlying client based on API styletry:if api_style == "anthropic":@@ -494,6 +609,65 @@ class HeadroomClient:# Note: output tokens unknown for streamsself._storage.save(metrics)+ def _extract_query(self, messages: list[dict[str, Any]]) -> str:+ """Extract query from messages for semantic caching.++ Returns the last user message content as the query.+ """+ for msg in reversed(messages):+ if msg.get("role") == "user":+ content = msg.get("content", "")+ if isinstance(content, str):+ return content+ elif isinstance(content, list):+ # Content block format+ for block in content:+ if isinstance(block, dict) and block.get("type") == "text":+ return block.get("text", "")+ return ""+ return ""++ def _store_response_in_semantic_cache(+ self,+ messages: list[dict[str, Any]],+ response: Any,+ model: str,+ ) -> None:+ """Store response in semantic cache for future hits."""+ if self._semantic_cache_layer is not None:+ cache_context = OptimizationContext(+ provider=self._provider.name.lower(),+ model=model,+ query=self._extract_query(messages),+ )+ # Extract response content for caching+ response_data = self._extract_response_content(response)+ if response_data:+ self._semantic_cache_layer.store_response(+ messages, response_data, cache_context+ )++ def _extract_response_content(self, response: Any) -> dict[str, Any] | None:+ """Extract cacheable content from API response."""+ try:+ # OpenAI format+ if hasattr(response, "choices") and response.choices:+ choice = response.choices[0]+ if hasattr(choice, "message"):+ return {+ "role": "assistant",+ "content": choice.message.content,+ }+ # Anthropic format+ elif hasattr(response, "content"):+ return {+ "role": "assistant",+ "content": response.content,+ }+ except Exception:+ pass+ return None+def _simulate(self,*,diff --git a/headroom/config.py b/headroom/config.pyindex 770d544fa..9b6d6e8b9 100644--- a/headroom/config.py+++ b/headroom/config.py@@ -193,6 +193,31 @@ class SmartCrusherConfig:relevance: RelevanceScorerConfig = field(default_factory=RelevanceScorerConfig)+@dataclass+class CacheOptimizerConfig:+ """Configuration for provider-specific cache optimization.++ The CacheOptimizer system provides provider-specific caching strategies:+ - Anthropic: Explicit cache_control breakpoints for prompt caching+ - OpenAI: Prefix stabilization for automatic prefix caching+ - Google: CachedContent API lifecycle management++ This is COMPLEMENTARY to the CacheAligner transform - CacheAligner does+ basic prefix stabilization (date extraction, whitespace normalization),+ while CacheOptimizer applies provider-specific optimizations.++ Enable this for maximum cache hit rates when you know your provider.+ """++ enabled: bool = True # Enable provider-specific cache optimization+ auto_detect_provider: bool = True # Auto-detect from HeadroomClient provider+ min_cacheable_tokens: int = 1024 # Minimum tokens for caching (provider may override)+ enable_semantic_cache: bool = False # Enable query-level semantic caching+ semantic_cache_similarity: float = 0.95 # Similarity threshold for semantic cache+ semantic_cache_max_entries: int = 1000 # Max semantic cache entries+ semantic_cache_ttl_seconds: int = 300 # Semantic cache TTL++@dataclassclass HeadroomConfig:"""Main configuration for HeadroomClient."""@@ -206,6 +231,7 @@ class HeadroomConfig:smart_crusher: SmartCrusherConfig = field(default_factory=SmartCrusherConfig)cache_aligner: CacheAlignerConfig = field(default_factory=CacheAlignerConfig)rolling_window: RollingWindowConfig = field(default_factory=RollingWindowConfig)+ cache_optimizer: CacheOptimizerConfig = field(default_factory=CacheOptimizerConfig)# Debugging - opt-in diff artifact generationgenerate_diff_artifact: bool = False # Enable to get detailed transform diffs@@ -370,11 +396,20 @@ class RequestMetrics:# Waste signalswaste_signals: dict[str, int] = field(default_factory=dict)- # Cache metrics+ # Cache metrics (basic)stable_prefix_hash: str = ""cache_alignment_score: float = 0.0cached_tokens: int | None = None # From API response if available+ # Cache optimizer metrics (provider-specific)+ cache_optimizer_used: str | None = None # e.g., "anthropic-cache-optimizer"+ cache_optimizer_strategy: str | None = None # e.g., "explicit_breakpoints"+ cacheable_tokens: int = 0 # Tokens eligible for caching+ breakpoints_inserted: int = 0 # Cache breakpoints added (Anthropic)+ estimated_cache_hit: bool = False # Whether prefix matches previous+ estimated_savings_percent: float = 0.0 # Estimated savings if cached+ semantic_cache_hit: bool = False # Whether semantic cache was hit+# Transform detailstransforms_applied: list[str] = field(default_factory=list)tool_units_dropped: int = 0diff --git a/tests/test_cache/__init__.py b/tests/test_cache/__init__.pynew file mode 100644index 000000000..875b77f8a--- /dev/null+++ b/tests/test_cache/__init__.py@@ -0,0 +1 @@+"""Tests for the cache optimization module."""diff --git a/tests/test_cache/test_anthropic.py b/tests/test_cache/test_anthropic.pynew file mode 100644index 000000000..81cdcb126--- /dev/null+++ b/tests/test_cache/test_anthropic.py@@ -0,0 +1,184 @@+"""Tests for AnthropicCacheOptimizer."""++import pytest+from headroom.cache import (+ AnthropicCacheOptimizer,+ CacheConfig,+ OptimizationContext,+)+from headroom.cache.base import CacheStrategy, BreakpointLocation+++class TestAnthropicCacheOptimizer:+ """Test AnthropicCacheOptimizer functionality."""++ @pytest.fixture+ def optimizer(self):+ """Create optimizer instance."""+ return AnthropicCacheOptimizer()++ @pytest.fixture+ def context(self):+ """Create optimization context."""+ return OptimizationContext(+ provider="anthropic",+ model="claude-3-opus",+ )++ def test_optimizer_properties(self, optimizer):+ """Test optimizer properties."""+ assert optimizer.name == "anthropic-cache-optimizer"+ assert optimizer.provider == "anthropic"+ assert optimizer.strategy == CacheStrategy.EXPLICIT_BREAKPOINTS++ def test_enforces_minimum_tokens(self):+ """Test that Anthropic minimum is enforced."""+ config = CacheConfig(min_cacheable_tokens=100)+ optimizer = AnthropicCacheOptimizer(config)+ assert optimizer.config.min_cacheable_tokens >= 1024++ def test_enforces_maximum_breakpoints(self):+ """Test that Anthropic maximum breakpoints is enforced."""+ config = CacheConfig(max_breakpoints=10)+ optimizer = AnthropicCacheOptimizer(config)+ assert optimizer.config.max_breakpoints <= 4++ def test_optimize_simple_messages(self, optimizer, context):+ """Test optimizing simple messages."""+ messages = [+ {"role": "system", "content": "You are a helpful assistant. " * 500},+ {"role": "user", "content": "Hello!"},+ ]++ result = optimizer.optimize(messages, context)++ assert result.messages is not None+ assert len(result.messages) == 2+ assert result.metrics.stable_prefix_hash != ""++ def test_optimize_inserts_cache_control(self, optimizer, context):+ """Test that optimization inserts cache_control blocks."""+ # Large system prompt to trigger caching+ messages = [+ {"role": "system", "content": "You are a helpful assistant. " * 500},+ {"role": "user", "content": "Hello!"},+ ]++ result = optimizer.optimize(messages, context)++ # Check if cache_control was inserted+ system_content = result.messages[0]["content"]+ if isinstance(system_content, list):+ has_cache_control = any(+ "cache_control" in block+ for block in system_content+ if isinstance(block, dict)+ )+ assert has_cache_control++ def test_optimize_with_dates(self, optimizer, context):+ """Test optimization extracts dates."""+ messages = [+ {+ "role": "system",+ "content": "Today is January 7, 2026. You are a helpful assistant. " * 300,+ },+ {"role": "user", "content": "Hello!"},+ ]++ result = optimizer.optimize(messages, context)++ # Dates should be moved to end+ assert "extracted_dates" in result.transforms_applied or result.metrics.breakpoints_inserted >= 0++ def test_optimize_disabled(self, context):+ """Test optimization when disabled."""+ config = CacheConfig(enabled=False)+ optimizer = AnthropicCacheOptimizer(config)++ messages = [+ {"role": "system", "content": "Test"},+ {"role": "user", "content": "Hello!"},+ ]++ result = optimizer.optimize(messages, context)++ assert result.transforms_applied == []++ def test_prefix_hash_tracking(self, optimizer, context):+ """Test that prefix hash is tracked between calls."""+ messages = [+ {"role": "system", "content": "You are a helpful assistant. " * 500},+ {"role": "user", "content": "Hello!"},+ ]++ result1 = optimizer.optimize(messages, context)+ result2 = optimizer.optimize(messages, context)++ # Second call should detect stable prefix+ assert result2.metrics.previous_prefix_hash == result1.metrics.stable_prefix_hash++ def test_estimate_savings(self, optimizer, context):+ """Test savings estimation."""+ messages = [+ {"role": "system", "content": "You are a helpful assistant. " * 500},+ {"role": "user", "content": "Hello!"},+ ]++ savings = optimizer.estimate_savings(messages, context)+ assert savings >= 0.0+ assert savings <= 100.0++ def test_content_block_format(self, optimizer, context):+ """Test handling of content block format."""+ messages = [+ {+ "role": "system",+ "content": [+ {"type": "text", "text": "You are a helpful assistant. " * 500}+ ],+ },+ {"role": "user", "content": "Hello!"},+ ]++ result = optimizer.optimize(messages, context)+ assert result.messages is not None++ def test_tools_are_cacheable(self, optimizer, context):+ """Test that tools are identified as cacheable."""+ messages = [+ {"role": "system", "content": "You are helpful. " * 300},+ {+ "role": "user",+ "content": "Use tools",+ "tools": [+ {+ "name": "search",+ "description": "Search the web " * 200,+ "input_schema": {"type": "object"},+ }+ ],+ },+ ]++ result = optimizer.optimize(messages, context)+ assert result.metrics.cacheable_tokens > 0++ def test_metrics_history(self, optimizer, context):+ """Test that metrics are recorded."""+ messages = [+ {"role": "system", "content": "You are helpful. " * 500},+ {"role": "user", "content": "Hello!"},+ ]++ optimizer.optimize(messages, context)+ metrics = optimizer.get_metrics()++ assert metrics is not None+ assert metrics.stable_prefix_hash != ""++ def test_cache_constants(self, optimizer):+ """Test cache-related constants."""+ assert optimizer.get_cache_write_cost_multiplier() == 1.25+ assert optimizer.get_cache_read_cost_multiplier() == 0.10+ assert optimizer.get_cache_ttl_seconds() == 300diff --git a/tests/test_cache/test_base.py b/tests/test_cache/test_base.pynew file mode 100644index 000000000..b2d04d49c--- /dev/null+++ b/tests/test_cache/test_base.py@@ -0,0 +1,141 @@+"""Tests for cache base types and interfaces."""++import pytest+from headroom.cache.base import (+ CacheStrategy,+ BreakpointLocation,+ CacheBreakpoint,+ CacheConfig,+ CacheMetrics,+ CacheResult,+ OptimizationContext,+)+++class TestCacheStrategy:+ """Test CacheStrategy enum."""++ def test_strategies_exist(self):+ """Test all expected strategies exist."""+ assert CacheStrategy.PREFIX_STABILIZATION.value == "prefix_stabilization"+ assert CacheStrategy.EXPLICIT_BREAKPOINTS.value == "explicit_breakpoints"+ assert CacheStrategy.CACHED_CONTENT.value == "cached_content"+ assert CacheStrategy.NONE.value == "none"+++class TestCacheConfig:+ """Test CacheConfig dataclass."""++ def test_default_values(self):+ """Test default configuration values."""+ config = CacheConfig()+ assert config.enabled is True+ assert config.min_cacheable_tokens == 1024+ assert config.max_breakpoints == 4+ assert config.normalize_whitespace is True+ assert config.collapse_blank_lines is True++ def test_custom_values(self):+ """Test custom configuration."""+ config = CacheConfig(+ enabled=False,+ min_cacheable_tokens=2048,+ max_breakpoints=2,+ )+ assert config.enabled is False+ assert config.min_cacheable_tokens == 2048+ assert config.max_breakpoints == 2++ def test_date_patterns(self):+ """Test date patterns are set."""+ config = CacheConfig()+ assert len(config.date_patterns) > 0+ assert any("Today" in p for p in config.date_patterns)+++class TestCacheMetrics:+ """Test CacheMetrics dataclass."""++ def test_default_values(self):+ """Test default metrics values."""+ metrics = CacheMetrics()+ assert metrics.stable_prefix_tokens == 0+ assert metrics.breakpoints_inserted == 0+ assert metrics.estimated_cache_hit is False+ assert metrics.estimated_savings_percent == 0.0++ def test_custom_values(self):+ """Test custom metrics."""+ metrics = CacheMetrics(+ stable_prefix_tokens=5000,+ breakpoints_inserted=2,+ estimated_cache_hit=True,+ estimated_savings_percent=90.0,+ )+ assert metrics.stable_prefix_tokens == 5000+ assert metrics.breakpoints_inserted == 2+ assert metrics.estimated_cache_hit is True+ assert metrics.estimated_savings_percent == 90.0+++class TestCacheBreakpoint:+ """Test CacheBreakpoint dataclass."""++ def test_breakpoint_creation(self):+ """Test creating a breakpoint."""+ bp = CacheBreakpoint(+ message_index=0,+ location=BreakpointLocation.AFTER_SYSTEM,+ tokens_at_breakpoint=2000,+ reason="System prompt is cacheable",+ )+ assert bp.message_index == 0+ assert bp.location == BreakpointLocation.AFTER_SYSTEM+ assert bp.tokens_at_breakpoint == 2000+ assert bp.content_index is None+++class TestCacheResult:+ """Test CacheResult dataclass."""++ def test_result_creation(self):+ """Test creating a cache result."""+ messages = [{"role": "system", "content": "Hello"}]+ result = CacheResult(+ messages=messages,+ metrics=CacheMetrics(cacheable_tokens=1000),+ transforms_applied=["normalized_whitespace"],+ )+ assert result.messages == messages+ assert result.metrics.cacheable_tokens == 1000+ assert "normalized_whitespace" in result.transforms_applied++ def test_semantic_cache_hit(self):+ """Test semantic cache hit result."""+ result = CacheResult(+ messages=[],+ semantic_cache_hit=True,+ cached_response={"text": "cached response"},+ )+ assert result.semantic_cache_hit is True+ assert result.cached_response["text"] == "cached response"+++class TestOptimizationContext:+ """Test OptimizationContext dataclass."""++ def test_context_creation(self):+ """Test creating optimization context."""+ context = OptimizationContext(+ provider="anthropic",+ model="claude-3-opus",+ request_id="req-123",+ )+ assert context.provider == "anthropic"+ assert context.model == "claude-3-opus"+ assert context.request_id == "req-123"++ def test_default_timestamp(self):+ """Test default timestamp is set."""+ context = OptimizationContext()+ assert context.timestamp is not Nonediff --git a/tests/test_cache/test_client_integration.py b/tests/test_cache/test_client_integration.pynew file mode 100644index 000000000..d5fa7606a--- /dev/null+++ b/tests/test_cache/test_client_integration.py@@ -0,0 +1,221 @@+"""Tests for HeadroomClient cache optimizer integration."""++import pytest+import tempfile+import os+from unittest.mock import MagicMock, patch+from headroom import (+ HeadroomClient,+ AnthropicCacheOptimizer,+ CacheOptimizerRegistry,+)+from headroom.providers import AnthropicProvider, OpenAIProvider+++def temp_db():+ """Create a temporary database file."""+ fd, path = tempfile.mkstemp(suffix=".db")+ os.close(fd)+ yield f"sqlite:///{path}"+ if os.path.exists(path):+ os.unlink(path)+++class MockTokenCounter:+ """Mock token counter for testing."""++ def count_tokens(self, text: str) -> int:+ return len(text) // 4++ def count_messages(self, messages: list) -> int:+ total = 0+ for msg in messages:+ content = msg.get("content", "")+ if isinstance(content, str):+ total += len(content) // 4+ elif isinstance(content, list):+ for block in content:+ if isinstance(block, dict):+ total += len(block.get("text", "")) // 4+ return total+++class MockAnthropicProvider:+ """Mock Anthropic provider for testing."""++ name = "anthropic"++ def get_token_counter(self, model: str):+ return MockTokenCounter()++ def get_context_limit(self, model: str) -> int:+ return 200000+++class MockOpenAIProvider:+ """Mock OpenAI provider for testing."""++ name = "openai"++ def get_token_counter(self, model: str):+ return MockTokenCounter()++ def get_context_limit(self, model: str) -> int:+ return 128000+++class TestHeadroomClientCacheIntegration:+ """Test HeadroomClient cache optimizer integration."""++ def test_auto_detect_anthropic_optimizer(self, temp_db):+ """Test that Anthropic optimizer is auto-detected."""+ mock_client = MagicMock()+ provider = MockAnthropicProvider()++ client = HeadroomClient(+ original_client=mock_client,+ provider=provider,+ store_url=temp_db,+ enable_cache_optimizer=True,+ )++ assert client._cache_optimizer is not None+ assert client._cache_optimizer.name == "anthropic-cache-optimizer"++ def test_auto_detect_openai_optimizer(self, temp_db):+ """Test that OpenAI optimizer is auto-detected."""+ mock_client = MagicMock()+ provider = MockOpenAIProvider()++ client = HeadroomClient(+ original_client=mock_client,+ provider=provider,+ store_url=temp_db,+ enable_cache_optimizer=True,+ )++ assert client._cache_optimizer is not None+ assert client._cache_optimizer.name == "openai-prefix-stabilizer"++ def test_custom_optimizer(self, temp_db):+ """Test using a custom optimizer."""+ mock_client = MagicMock()+ provider = MockAnthropicProvider()+ custom_optimizer = AnthropicCacheOptimizer()++ client = HeadroomClient(+ original_client=mock_client,+ provider=provider,+ store_url=temp_db,+ cache_optimizer=custom_optimizer,+ )++ assert client._cache_optimizer is custom_optimizer++ def test_disable_cache_optimizer(self, temp_db):+ """Test disabling cache optimizer."""+ mock_client = MagicMock()+ provider = MockAnthropicProvider()++ client = HeadroomClient(+ original_client=mock_client,+ provider=provider,+ store_url=temp_db,+ enable_cache_optimizer=False,+ )++ assert client._cache_optimizer is None++ def test_semantic_cache_layer_creation(self, temp_db):+ """Test semantic cache layer is created when enabled."""+ mock_client = MagicMock()+ provider = MockAnthropicProvider()++ client = HeadroomClient(+ original_client=mock_client,+ provider=provider,+ store_url=temp_db,+ enable_cache_optimizer=True,+ enable_semantic_cache=True,+ )++ assert client._semantic_cache_layer is not None+ assert client._cache_optimizer is not None++ def test_extract_query_from_string_content(self, temp_db):+ """Test query extraction from string content."""+ mock_client = MagicMock()+ provider = MockAnthropicProvider()++ client = HeadroomClient(+ original_client=mock_client,+ provider=provider,+ store_url=temp_db,+ )++ messages = [+ {"role": "system", "content": "You are helpful."},+ {"role": "user", "content": "What is 2+2?"},+ ]++ query = client._extract_query(messages)+ assert query == "What is 2+2?"++ def test_extract_query_from_content_blocks(self, temp_db):+ """Test query extraction from content block format."""+ mock_client = MagicMock()+ provider = MockAnthropicProvider()++ client = HeadroomClient(+ original_client=mock_client,+ provider=provider,+ store_url=temp_db,+ )++ messages = [+ {"role": "system", "content": "You are helpful."},+ {+ "role": "user",+ "content": [{"type": "text", "text": "What is 2+2?"}],+ },+ ]++ query = client._extract_query(messages)+ assert query == "What is 2+2?"++ def test_extract_query_last_user_message(self, temp_db):+ """Test that query extraction uses last user message."""+ mock_client = MagicMock()+ provider = MockAnthropicProvider()++ client = HeadroomClient(+ original_client=mock_client,+ provider=provider,+ store_url=temp_db,+ )++ messages = [+ {"role": "user", "content": "First question"},+ {"role": "assistant", "content": "First answer"},+ {"role": "user", "content": "Second question"},+ ]++ query = client._extract_query(messages)+ assert query == "Second question"++ def test_config_propagation(self, temp_db):+ """Test that config is properly propagated."""+ mock_client = MagicMock()+ provider = MockAnthropicProvider()++ client = HeadroomClient(+ original_client=mock_client,+ provider=provider,+ store_url=temp_db,+ enable_cache_optimizer=True,+ enable_semantic_cache=True,+ )++ assert client._config.cache_optimizer.enabled is True+ assert client._config.cache_optimizer.enable_semantic_cache is Truediff --git a/tests/test_cache/test_dynamic_detector.py b/tests/test_cache/test_dynamic_detector.pynew file mode 100644index 000000000..2617f3d8e--- /dev/null+++ b/tests/test_cache/test_dynamic_detector.py@@ -0,0 +1,505 @@+"""Tests for the dynamic content detector."""++import pytest+from headroom.cache.dynamic_detector import (+ DetectionResult,+ DetectorConfig,+ DynamicCategory,+ DynamicContentDetector,+ DynamicSpan,+ RegexDetector,+ detect_dynamic_content,+)+++class TestRegexDetector:+ """Test the Tier 1 regex detector."""++ @pytest.fixture+ def detector(self):+ """Create a regex detector."""+ config = DetectorConfig(tiers=["regex"])+ return RegexDetector(config)++ def test_iso_date(self, detector):+ """Test ISO date detection."""+ spans = detector.detect("The date is 2024-01-15.")+ assert len(spans) == 1+ assert spans[0].text == "2024-01-15"+ assert spans[0].category == DynamicCategory.DATE+ assert spans[0].tier == "regex"++ def test_structural_detection(self, detector):+ """Test structural detection via 'Label: value' patterns."""+ # New scalable approach: detect via structural "Today: value" pattern+ spans = detector.detect("Date: 2024-01-15")+ assert len(spans) == 1+ assert spans[0].text == "2024-01-15"+ assert spans[0].category == DynamicCategory.DATE++ # Test user label detection+ spans = detector.detect("User: [email protected]")+ user_spans = [s for s in spans if s.category == DynamicCategory.USER_DATA]+ assert len(user_spans) == 1++ def test_datetime_iso(self, detector):+ """Test ISO datetime detection."""+ spans = detector.detect("Timestamp: 2024-01-15T10:30:00Z")+ assert len(spans) == 1+ assert spans[0].text == "2024-01-15T10:30:00Z"+ assert spans[0].category == DynamicCategory.DATETIME++ def test_uuid(self, detector):+ """Test UUID detection."""+ spans = detector.detect("ID: 550e8400-e29b-41d4-a716-446655440000")+ assert len(spans) == 1+ assert spans[0].text == "550e8400-e29b-41d4-a716-446655440000"+ assert spans[0].category == DynamicCategory.UUID++ def test_request_id(self, detector):+ """Test request ID detection."""+ spans = detector.detect("Request: req_abc123def456ghi789")+ assert len(spans) == 1+ assert "req_" in spans[0].text+ assert spans[0].category == DynamicCategory.REQUEST_ID++ def test_unix_timestamp(self, detector):+ """Test Unix timestamp detection."""+ spans = detector.detect("Time: 1705312200")+ assert len(spans) == 1+ assert spans[0].text == "1705312200"+ assert spans[0].category == DynamicCategory.TIMESTAMP++ def test_time(self, detector):+ """Test time detection."""+ spans = detector.detect("Meeting at 10:30 AM")+ assert len(spans) == 1+ assert spans[0].text == "10:30 AM"+ assert spans[0].category == DynamicCategory.TIME++ def test_version(self, detector):+ """Test version number detection."""+ spans = detector.detect("Running v2.3.1-beta")+ assert len(spans) == 1+ assert spans[0].text == "v2.3.1-beta"+ assert spans[0].category == DynamicCategory.VERSION++ def test_date_prefix_pattern(self, detector):+ """Test full date prefix phrase detection."""+ spans = detector.detect("Today is Monday, January 15, 2024. You are an assistant.")+ assert len(spans) >= 1+ # Should detect the full phrase+ date_spans = [s for s in spans if s.category == DynamicCategory.DATE]+ assert len(date_spans) >= 1++ def test_multiple_dynamic_elements(self, detector):+ """Test detecting multiple dynamic elements."""+ content = """+ Date: 2024-01-15+ Time: 10:30:00+ Request ID: req_abc123def456ghi789xyz+ UUID: 550e8400-e29b-41d4-a716-446655440000+ """+ spans = detector.detect(content)+ assert len(spans) == 4+ categories = {s.category for s in spans}+ assert DynamicCategory.DATE in categories+ assert DynamicCategory.TIME in categories+ assert DynamicCategory.REQUEST_ID in categories+ assert DynamicCategory.UUID in categories++ def test_no_false_positives_on_static(self, detector):+ """Test that static content doesn't trigger false positives."""+ spans = detector.detect("You are a helpful assistant. Answer questions clearly.")+ assert len(spans) == 0++ def test_positions_are_correct(self, detector):+ """Test that span positions are correct."""+ content = "Date: 2024-01-15"+ spans = detector.detect(content)+ assert len(spans) == 1+ assert content[spans[0].start:spans[0].end] == spans[0].text+++class TestDynamicContentDetector:+ """Test the unified dynamic content detector."""++ def test_regex_only(self):+ """Test detector with regex tier only."""+ config = DetectorConfig(tiers=["regex"])+ detector = DynamicContentDetector(config)++ result = detector.detect("Today is 2024-01-15. You are helpful.")++ assert len(result.spans) == 1+ assert result.spans[0].text == "2024-01-15"+ assert "regex" in result.tiers_used+ assert result.processing_time_ms < 10 # Should be very fast++ def test_static_dynamic_split(self):+ """Test that content is properly split."""+ config = DetectorConfig(tiers=["regex"])+ detector = DynamicContentDetector(config)++ result = detector.detect("Today is 2024-01-15. You are helpful.")++ assert "2024-01-15" not in result.static_content+ assert "2024-01-15" in result.dynamic_content+ assert "You are helpful" in result.static_content++ def test_complex_content(self):+ """Test with realistic system prompt."""+ config = DetectorConfig(tiers=["regex"])+ detector = DynamicContentDetector(config)++ content = """You are a helpful AI assistant.+Today is January 15, 2024.+Current session: sess_abc123def456ghi789xyz++Instructions:+1. Be concise+2. Be accurate+3. Be helpful++Request ID: req_xyz789abc123def456ghi"""++ result = detector.detect(content)++ # Should find date, session ID, request ID+ assert len(result.spans) >= 2+ categories = {s.category for s in result.spans}+ assert DynamicCategory.DATE in categories or DynamicCategory.REQUEST_ID in categories++ def test_empty_content(self):+ """Test with empty content."""+ detector = DynamicContentDetector()+ result = detector.detect("")++ assert len(result.spans) == 0+ assert result.static_content == ""+ assert result.dynamic_content == ""++ def test_no_dynamic_content(self):+ """Test with fully static content."""+ detector = DynamicContentDetector()+ content = "You are a helpful assistant. Answer questions clearly and concisely."++ result = detector.detect(content)++ assert len(result.spans) == 0+ assert result.static_content == content+ assert result.dynamic_content == ""++ def test_custom_patterns(self):+ """Test adding custom regex patterns."""+ config = DetectorConfig(+ tiers=["regex"],+ custom_patterns=[+ (r"CUSTOM_\d{4}", DynamicCategory.REQUEST_ID),+ ],+ )+ detector = DynamicContentDetector(config)++ result = detector.detect("Code: CUSTOM_1234")++ custom_spans = [s for s in result.spans if s.text == "CUSTOM_1234"]+ assert len(custom_spans) == 1++ def test_available_tiers(self):+ """Test that available_tiers reflects actual availability."""+ config = DetectorConfig(tiers=["regex", "ner", "semantic"])+ detector = DynamicContentDetector(config)++ # Regex should always be available+ assert "regex" in detector.available_tiers++ # NER and semantic depend on optional dependencies+ # They may or may not be available++ def test_warnings_for_missing_dependencies(self):+ """Test that warnings are generated for missing dependencies."""+ config = DetectorConfig(tiers=["regex", "ner", "semantic"])+ detector = DynamicContentDetector(config)++ result = detector.detect("Test content")++ # If NER/semantic not installed, should have warnings+ # (This test passes either way - it's informational)+ # If deps ARE installed, no warnings. If not, warnings present.+++class TestConvenienceFunction:+ """Test the detect_dynamic_content convenience function."""++ def test_basic_usage(self):+ """Test basic convenience function usage."""+ result = detect_dynamic_content("Date: 2024-01-15")++ assert isinstance(result, DetectionResult)+ assert len(result.spans) == 1+ assert result.spans[0].text == "2024-01-15"++ def test_with_tiers(self):+ """Test specifying tiers."""+ result = detect_dynamic_content(+ "Date: 2024-01-15",+ tiers=["regex"],+ )++ assert "regex" in result.tiers_used+++class TestEntropyDetection:+ """Test entropy-based detection for random IDs/tokens."""++ def test_high_entropy_string(self):+ """Test that high-entropy strings are detected."""+ from headroom.cache.dynamic_detector import calculate_entropy++ # High entropy strings (random-looking)+ assert calculate_entropy("abc123xyz789def") > 0.7+ assert calculate_entropy("550e8400e29b41d4") > 0.7++ # Low entropy strings (repetitive)+ assert calculate_entropy("aaaaaaaaaa") < 0.3+ assert calculate_entropy("abababab") < 0.6++ def test_entropy_detection_finds_ids(self):+ """Test that entropy detection finds random IDs."""+ detector = DynamicContentDetector()++ # Random-looking ID that isn't covered by universal patterns+ result = detector.detect("Auth: xK7mN2pQr9sT4vW")++ # Should find the ID via entropy or structural detection+ assert len(result.spans) >= 1++ def test_entropy_skips_common_words(self):+ """Test that common words aren't flagged as high-entropy."""+ detector = DynamicContentDetector()++ # These words have mixed case/numbers but aren't IDs+ result = detector.detect("Use username and password correctly.")++ # "username" and "password" shouldn't be detected+ flagged_words = [s.text for s in result.spans]+ assert "username" not in flagged_words+ assert "password" not in flagged_words+++class TestEdgeCases:+ """Test edge cases and tricky inputs."""++ def test_overlapping_patterns(self):+ """Test that overlapping patterns don't cause duplicates."""+ detector = DynamicContentDetector()++ # ISO datetime contains ISO date - shouldn't match both+ result = detector.detect("Time: 2024-01-15T10:30:00Z")++ # Should match datetime, not date separately+ assert len(result.spans) == 1+ assert result.spans[0].category == DynamicCategory.DATETIME++ def test_adjacent_dynamic_content(self):+ """Test adjacent dynamic elements."""+ detector = DynamicContentDetector()++ result = detector.detect("2024-01-15 10:30:00")++ # Should find both date and time+ assert len(result.spans) == 2++ def test_very_long_content(self):+ """Test with long content."""+ detector = DynamicContentDetector()++ # Create long content with some dynamic parts+ static_parts = ["This is static text. "] * 100+ content = "".join(static_parts) + "Date: 2024-01-15. " + "".join(static_parts)++ result = detector.detect(content)++ assert len(result.spans) == 1+ assert result.processing_time_ms < 100 # Should still be fast++ def test_special_characters(self):+ """Test content with special characters."""+ detector = DynamicContentDetector()++ content = "Date: 2024-01-15\nUUID: 550e8400-e29b-41d4-a716-446655440000\n\n---\n"+ result = detector.detect(content)++ assert len(result.spans) == 2++ def test_unicode_content(self):+ """Test with Unicode content."""+ detector = DynamicContentDetector()++ content = "日期: 2024-01-15. Héllo wörld!"+ result = detector.detect(content)++ # Should still find the date+ assert len(result.spans) == 1+ assert result.spans[0].text == "2024-01-15"+++class TestCacheAlignmentScenarios:+ """Test scenarios relevant to cache alignment."""++ def test_system_prompt_dates(self):+ """Test extracting dates from system prompts."""+ detector = DynamicContentDetector()++ content = """You are Claude, an AI assistant by Anthropic.+Today is Monday, January 15, 2024.+Current time: 10:30 AM PST.++Your task is to help users with coding questions."""++ result = detector.detect(content)++ # Should extract date and time+ assert len(result.spans) >= 1++ # Static content should not have dates+ assert "2024" not in result.static_content or "January" in result.static_content++ # Dynamic content should have the dates+ assert "January" in result.dynamic_content or "2024-01-15" in result.dynamic_content or "10:30" in result.dynamic_content++ def test_request_metadata(self):+ """Test extracting request metadata."""+ detector = DynamicContentDetector()++ content = """Request ID: req_abc123xyz789+Trace ID: 550e8400-e29b-41d4-a716-446655440000+Timestamp: 1705312200++Process the following query:"""++ result = detector.detect(content)++ # Should find request ID, UUID, timestamp+ categories = {s.category for s in result.spans}+ assert len(result.spans) >= 2++ def test_mixed_static_dynamic(self):+ """Test content with interspersed static and dynamic parts."""+ detector = DynamicContentDetector()++ content = """You are helpful (static).+Today is 2024-01-15 (dynamic).+Always be accurate (static).+Session: sess_abc123xyz789 (dynamic).+Never lie (static)."""++ result = detector.detect(content)++ # Should find date and session ID+ assert len(result.spans) >= 1++ # Static content should preserve the static parts+ assert "helpful" in result.static_content+ assert "accurate" in result.static_content+++class TestNERDetector:+ """Test Tier 2 NER detector (if spaCy available)."""++ @pytest.fixture+ def ner_detector(self):+ """Create detector with NER enabled."""+ from headroom.cache.dynamic_detector import NERDetector, _SPACY_AVAILABLE++ if not _SPACY_AVAILABLE:+ pytest.skip("spaCy not installed")++ config = DetectorConfig(tiers=["ner"])+ detector = NERDetector(config)++ if not detector.is_available:+ pytest.skip("spaCy model not available")++ return detector++ def test_person_detection(self, ner_detector):+ """Test detecting person names."""+ spans, _ = ner_detector.detect("John Smith sent the message.")++ person_spans = [s for s in spans if s.category == DynamicCategory.PERSON]+ # NER might or might not detect "John Smith" depending on model+ # This is more of an integration test++ def test_money_detection(self, ner_detector):+ """Test detecting money amounts."""+ spans, _ = ner_detector.detect("The total is $500.00")++ money_spans = [s for s in spans if s.category == DynamicCategory.MONEY]+ # May or may not detect depending on spaCy model+++class TestSemanticDetector:+ """Test Tier 3 semantic detector (if sentence-transformers available)."""++ @pytest.fixture+ def semantic_detector(self):+ """Create detector with semantic enabled."""+ from headroom.cache.dynamic_detector import SemanticDetector, _SENTENCE_TRANSFORMERS_AVAILABLE++ if not _SENTENCE_TRANSFORMERS_AVAILABLE:+ pytest.skip("sentence-transformers not installed")++ config = DetectorConfig(tiers=["semantic"])+ detector = SemanticDetector(config)++ if not detector.is_available:+ pytest.skip("Embedding model not available")++ return detector++ def test_realtime_detection(self, semantic_detector):+ """Test detecting real-time/volatile content."""+ content = "The current stock price is updated every minute."+ spans, _ = semantic_detector.detect(content)++ # Should detect this as volatile/realtime+ # Depends on similarity threshold+++class TestIntegrationWithAllTiers:+ """Integration tests using all available tiers."""++ def test_all_tiers_together(self):+ """Test running all tiers on complex content."""+ config = DetectorConfig(tiers=["regex", "ner", "semantic"])+ detector = DynamicContentDetector(config)++ content = """Today is January 15, 2024.+John paid $500 for the service.+Request ID: req_abc123xyz789.+The stock price updates in real-time.+Be helpful and accurate."""++ result = detector.detect(content)++ # Should find at least the regex matches+ assert len(result.spans) >= 1++ # Check processing time is reasonable+ # NER + semantic might add 50-100ms+ assert result.processing_time_ms < 5000 # Very generous timeout++ # Should have used at least regex+ assert "regex" in result.tiers_used++ def test_tier_precedence(self):+ """Test that earlier tiers take precedence."""+ config = DetectorConfig(tiers=["regex", "ner"])+ detector = DynamicContentDetector(config)++ # Date should be caught by regex, not NER+ result = detector.detect("Date: 2024-01-15")++ assert len(result.spans) == 1+ assert result.spans[0].tier == "regex"diff --git a/tests/test_cache/test_google.py b/tests/test_cache/test_google.pynew file mode 100644index 000000000..0e9e5489b--- /dev/null+++ b/tests/test_cache/test_google.py@@ -0,0 +1,270 @@+"""Tests for GoogleCacheOptimizer."""++import pytest+from datetime import datetime, timedelta+from headroom.cache import GoogleCacheOptimizer, CacheConfig, OptimizationContext+from headroom.cache.base import CacheStrategy+from headroom.cache.google import (+ GOOGLE_MIN_CACHE_TOKENS,+ GOOGLE_CACHE_DISCOUNT,+ CachedContentInfo,+ CacheabilityAnalysis,+)+++class TestGoogleCacheOptimizer:+ """Test GoogleCacheOptimizer functionality."""++ @pytest.fixture+ def optimizer(self):+ """Create optimizer instance."""+ return GoogleCacheOptimizer()++ @pytest.fixture+ def context(self):+ """Create optimization context."""+ return OptimizationContext(+ provider="google",+ model="gemini-1.5-pro",+ )++ def test_optimizer_properties(self, optimizer):+ """Test optimizer properties."""+ assert optimizer.name == "google-cached-content"+ assert optimizer.provider == "google"+ assert optimizer.strategy == CacheStrategy.CACHED_CONTENT++ def test_enforces_minimum_tokens(self):+ """Test that Google's minimum is enforced."""+ config = CacheConfig(min_cacheable_tokens=100)+ optimizer = GoogleCacheOptimizer(config)+ assert optimizer.config.min_cacheable_tokens >= GOOGLE_MIN_CACHE_TOKENS++ def test_optimize_below_threshold(self, optimizer, context):+ """Test optimization with content below threshold."""+ messages = [+ {"role": "system", "content": "Short system prompt."},+ {"role": "user", "content": "Hello!"},+ ]++ result = optimizer.optimize(messages, context)++ assert result.metrics.cacheable_tokens < GOOGLE_MIN_CACHE_TOKENS+ assert any("32K" in w for w in result.warnings)++ def test_optimize_above_threshold(self, optimizer, context):+ """Test optimization with content above threshold."""+ # Create content above 32K tokens+ messages = [+ {"role": "system", "content": "You are helpful. " * 15000},+ {"role": "user", "content": "Hello!"},+ ]++ result = optimizer.optimize(messages, context)++ assert result.metrics.cacheable_tokens > 0+ if result.metrics.cacheable_tokens >= GOOGLE_MIN_CACHE_TOKENS:+ assert result.metrics.estimated_savings_percent == GOOGLE_CACHE_DISCOUNT * 100++ def test_analyze_cacheability(self, optimizer, context):+ """Test cacheability analysis."""+ messages = [+ {"role": "system", "content": "Short"},+ {"role": "user", "content": "Hello!"},+ ]++ analysis = optimizer.analyze_cacheability(messages, context)++ assert isinstance(analysis, CacheabilityAnalysis)+ assert analysis.total_tokens > 0+ assert analysis.tokens_below_minimum > 0+ assert analysis.is_cacheable is False+ assert len(analysis.recommendations) > 0++ def test_cache_registration(self, optimizer):+ """Test registering a cache."""+ cache_info = optimizer.register_cache(+ cache_id="test-cache-123",+ content_hash="abc123",+ token_count=50000,+ expires_at=datetime.now() + timedelta(hours=1),+ )++ assert cache_info.cache_id == "test-cache-123"+ assert cache_info.content_hash == "abc123"+ assert cache_info.token_count == 50000+ assert not cache_info.is_expired+ assert cache_info.ttl_remaining_seconds > 0++ def test_cache_lookup(self, optimizer):+ """Test looking up a registered cache."""+ optimizer.register_cache(+ cache_id="test-cache-456",+ content_hash="def456",+ token_count=50000,+ expires_at=datetime.now() + timedelta(hours=1),+ )++ found = optimizer.get_reusable_cache("def456")+ assert found is not None+ assert found.cache_id == "test-cache-456"++ def test_cache_lookup_expired(self, optimizer):+ """Test that expired caches are not returned."""+ optimizer.register_cache(+ cache_id="expired-cache",+ content_hash="expired123",+ token_count=50000,+ expires_at=datetime.now() - timedelta(hours=1),+ )++ found = optimizer.get_reusable_cache("expired123")+ assert found is None++ def test_cache_lookup_insufficient_ttl(self, optimizer):+ """Test that caches with insufficient TTL are not returned."""+ optimizer.register_cache(+ cache_id="short-ttl-cache",+ content_hash="shortttl123",+ token_count=50000,+ expires_at=datetime.now() + timedelta(seconds=30),+ )++ # Default min_ttl is 60 seconds+ found = optimizer.get_reusable_cache("shortttl123")+ assert found is None++ def test_extend_cache_ttl(self, optimizer):+ """Test extending cache TTL."""+ optimizer.register_cache(+ cache_id="extend-cache",+ content_hash="extend123",+ token_count=50000,+ expires_at=datetime.now() + timedelta(hours=1),+ )++ new_expires = datetime.now() + timedelta(hours=2)+ updated = optimizer.extend_cache_ttl("extend-cache", new_expires)++ assert updated is not None+ assert updated.expires_at == new_expires++ def test_remove_cache(self, optimizer):+ """Test removing a cache."""+ optimizer.register_cache(+ cache_id="remove-cache",+ content_hash="remove123",+ token_count=50000,+ expires_at=datetime.now() + timedelta(hours=1),+ )++ removed = optimizer.remove_cache("remove-cache")+ assert removed is True++ found = optimizer.get_reusable_cache("remove123")+ assert found is None++ def test_cleanup_expired_caches(self, optimizer):+ """Test cleaning up expired caches."""+ # Register expired cache+ optimizer.register_cache(+ cache_id="cleanup-expired",+ content_hash="cleanup123",+ token_count=50000,+ expires_at=datetime.now() - timedelta(hours=1),+ )++ expired_ids = optimizer.cleanup_expired_caches()+ assert "cleanup-expired" in expired_ids++ def test_list_caches(self, optimizer):+ """Test listing caches."""+ optimizer.register_cache(+ cache_id="list-cache-1",+ content_hash="list1",+ token_count=50000,+ expires_at=datetime.now() + timedelta(hours=1),+ )+ optimizer.register_cache(+ cache_id="list-cache-2",+ content_hash="list2",+ token_count=50000,+ expires_at=datetime.now() + timedelta(hours=2),+ )++ caches = optimizer.list_caches()+ assert len(caches) >= 2++ def test_get_statistics(self, optimizer):+ """Test getting statistics."""+ optimizer.register_cache(+ cache_id="stats-cache",+ content_hash="stats123",+ token_count=50000,+ expires_at=datetime.now() + timedelta(hours=1),+ )++ stats = optimizer.get_statistics()+ assert "active_caches" in stats+ assert "caches_created" in stats+ assert stats["caches_created"] >= 1++ def test_prepare_cache_creation(self, optimizer, context):+ """Test preparing cache creation parameters."""+ messages = [+ {"role": "system", "content": "You are helpful. " * 15000},+ {"role": "user", "content": "Hello!"},+ ]++ params = optimizer.prepare_cache_creation(messages, context)++ if params is not None:+ assert "contents" in params+ assert "ttl" in params+ assert "display_name" in params++ def test_build_request_with_cache(self, optimizer):+ """Test building request with cache."""+ messages = [+ {"role": "system", "content": "System prompt"},+ {"role": "user", "content": "User message"},+ ]++ request = optimizer.build_request_with_cache(messages, "cache-123")++ assert "cached_content" in request+ assert request["cached_content"] == "cache-123"+ assert "contents" in request++ def test_export_import_registry(self, optimizer):+ """Test exporting and importing cache registry."""+ optimizer.register_cache(+ cache_id="export-cache",+ content_hash="export123",+ token_count=50000,+ expires_at=datetime.now() + timedelta(hours=1),+ )++ exported = optimizer.export_cache_registry()+ assert len(exported) >= 1++ new_optimizer = GoogleCacheOptimizer()+ imported = new_optimizer.import_cache_registry(exported)+ assert imported >= 1++ def test_cached_content_info_serialization(self):+ """Test CachedContentInfo serialization."""+ info = CachedContentInfo(+ cache_id="test",+ content_hash="hash",+ created_at=datetime.now(),+ expires_at=datetime.now() + timedelta(hours=1),+ token_count=50000,+ )++ data = info.to_dict()+ restored = CachedContentInfo.from_dict(data)++ assert restored.cache_id == info.cache_id+ assert restored.content_hash == info.content_hash+ assert restored.token_count == info.token_countdiff --git a/tests/test_cache/test_openai.py b/tests/test_cache/test_openai.pynew file mode 100644index 000000000..7e26b646e--- /dev/null+++ b/tests/test_cache/test_openai.py@@ -0,0 +1,196 @@+"""Tests for OpenAICacheOptimizer."""++import pytest+from headroom.cache import OpenAICacheOptimizer, CacheConfig, OptimizationContext+from headroom.cache.base import CacheStrategy+++class TestOpenAICacheOptimizer:+ """Test OpenAICacheOptimizer functionality."""++ @pytest.fixture+ def optimizer(self):+ """Create optimizer instance."""+ return OpenAICacheOptimizer()++ @pytest.fixture+ def context(self):+ """Create optimization context."""+ return OptimizationContext(+ provider="openai",+ model="gpt-4",+ )++ def test_optimizer_properties(self, optimizer):+ """Test optimizer properties."""+ assert optimizer.name == "openai-prefix-stabilizer"+ assert optimizer.provider == "openai"+ assert optimizer.strategy == CacheStrategy.PREFIX_STABILIZATION++ def test_optimize_simple_messages(self, optimizer, context):+ """Test optimizing simple messages."""+ messages = [+ {"role": "system", "content": "You are a helpful assistant."},+ {"role": "user", "content": "Hello!"},+ ]++ result = optimizer.optimize(messages, context)++ assert result.messages is not None+ assert len(result.messages) == 2+ assert result.metrics.stable_prefix_hash != ""++ def test_date_extraction(self, optimizer, context):+ """Test that dates are extracted from system prompt."""+ messages = [+ {+ "role": "system",+ "content": "Today is January 7, 2026. You are a helpful assistant.",+ },+ {"role": "user", "content": "Hello!"},+ ]++ result = optimizer.optimize(messages, context)++ # Check that date was extracted and moved+ system_content = result.messages[0]["content"]+ # The date should be moved to a dynamic section at the end+ assert "You are a helpful assistant" in system_content++ def test_whitespace_normalization(self, optimizer, context):+ """Test whitespace normalization."""+ messages = [+ {+ "role": "system",+ "content": "You are a helpful assistant.",+ },+ {"role": "user", "content": "Hello!"},+ ]++ result = optimizer.optimize(messages, context)++ # Whitespace should be normalized+ system_content = result.messages[0]["content"]+ assert " " not in system_content # Multiple spaces collapsed++ def test_optimize_disabled(self, context):+ """Test optimization when disabled."""+ config = CacheConfig(enabled=False)+ optimizer = OpenAICacheOptimizer(config)++ messages = [+ {"role": "system", "content": "Test"},+ {"role": "user", "content": "Hello!"},+ ]++ result = optimizer.optimize(messages, context)+ assert result.transforms_applied == []++ def test_prefix_stability_tracking(self, optimizer, context):+ """Test that prefix stability is tracked."""+ messages = [+ {"role": "system", "content": "You are a helpful assistant."},+ {"role": "user", "content": "Hello!"},+ ]++ # First call+ result1 = optimizer.optimize(messages, context)++ # Second call with same messages+ result2 = optimizer.optimize(messages, context)++ # Second call should detect stable prefix+ assert result2.metrics.estimated_cache_hit is True+ assert result2.metrics.prefix_changed_from_previous is False++ def test_prefix_change_detection(self, optimizer, context):+ """Test detection of prefix changes."""+ messages1 = [+ {"role": "system", "content": "You are a helpful assistant."},+ {"role": "user", "content": "Hello!"},+ ]+ messages2 = [+ {"role": "system", "content": "You are a different assistant."},+ {"role": "user", "content": "Hello!"},+ ]++ result1 = optimizer.optimize(messages1, context)+ result2 = optimizer.optimize(messages2, context)++ # Second call should detect prefix change+ assert result2.metrics.prefix_changed_from_previous is True++ def test_token_threshold_warning(self, optimizer, context):+ """Test warning when below token threshold."""+ messages = [+ {"role": "system", "content": "Short."},+ {"role": "user", "content": "Hi"},+ ]++ result = optimizer.optimize(messages, context)++ # Should have warning about being below threshold+ assert any("1024" in w for w in result.warnings)++ def test_estimate_savings_below_threshold(self, optimizer, context):+ """Test savings estimation below threshold."""+ messages = [+ {"role": "system", "content": "Short system prompt."},+ {"role": "user", "content": "Hello!"},+ ]++ savings = optimizer.estimate_savings(messages, context)+ assert savings == 0.0 # Below threshold++ def test_estimate_savings_above_threshold(self, optimizer, context):+ """Test savings estimation above threshold."""+ messages = [+ {"role": "system", "content": "You are helpful. " * 500},+ {"role": "user", "content": "Hello!"},+ ]++ # First call to establish baseline+ optimizer.optimize(messages, context)++ # Second call should show savings+ savings = optimizer.estimate_savings(messages, context)+ assert savings > 0.0++ def test_uuid_pattern_detection(self, optimizer, context):+ """Test detection of UUIDs in content."""+ messages = [+ {+ "role": "system",+ "content": "Request ID: 12345678-1234-1234-1234-123456789012. Be helpful.",+ },+ {"role": "user", "content": "Hello!"},+ ]++ result = optimizer.optimize(messages, context)+ # UUID should be detected as dynamic content+ assert result.metrics.stable_prefix_hash != ""++ def test_content_block_format(self, optimizer, context):+ """Test handling of content block format."""+ messages = [+ {+ "role": "system",+ "content": [{"type": "text", "text": "You are helpful."}],+ },+ {"role": "user", "content": "Hello!"},+ ]++ result = optimizer.optimize(messages, context)+ assert result.messages is not None++ def test_metrics_recording(self, optimizer, context):+ """Test that metrics are recorded."""+ messages = [+ {"role": "system", "content": "You are helpful."},+ {"role": "user", "content": "Hello!"},+ ]++ optimizer.optimize(messages, context)+ metrics = optimizer.get_metrics()++ assert metrics.stable_prefix_hash != ""diff --git a/tests/test_cache/test_registry.py b/tests/test_cache/test_registry.pynew file mode 100644index 000000000..c383e325a--- /dev/null+++ b/tests/test_cache/test_registry.py@@ -0,0 +1,135 @@+"""Tests for CacheOptimizerRegistry."""++import pytest+from headroom.cache import (+ CacheOptimizerRegistry,+ AnthropicCacheOptimizer,+ OpenAICacheOptimizer,+ GoogleCacheOptimizer,+ CacheConfig,+)+from headroom.cache.base import BaseCacheOptimizer, CacheStrategy, CacheResult, OptimizationContext+++class MockOptimizer(BaseCacheOptimizer):+ """Mock optimizer for testing."""++ @property+ def name(self) -> str:+ return "mock-optimizer"++ @property+ def provider(self) -> str:+ return "mock"++ @property+ def strategy(self) -> CacheStrategy:+ return CacheStrategy.NONE++ def optimize(self, messages, context, config=None):+ return CacheResult(messages=messages)+++class TestCacheOptimizerRegistry:+ """Test CacheOptimizerRegistry functionality."""++ def test_default_providers_registered(self):+ """Test that default providers are registered on import."""+ providers = CacheOptimizerRegistry.list_all()+ assert "anthropic" in providers+ assert "openai" in providers+ assert "google" in providers++ def test_get_anthropic(self):+ """Test getting Anthropic optimizer."""+ optimizer = CacheOptimizerRegistry.get("anthropic")+ assert isinstance(optimizer, AnthropicCacheOptimizer)+ assert optimizer.provider == "anthropic"+ assert optimizer.strategy == CacheStrategy.EXPLICIT_BREAKPOINTS++ def test_get_openai(self):+ """Test getting OpenAI optimizer."""+ optimizer = CacheOptimizerRegistry.get("openai")+ assert isinstance(optimizer, OpenAICacheOptimizer)+ assert optimizer.provider == "openai"+ assert optimizer.strategy == CacheStrategy.PREFIX_STABILIZATION++ def test_get_google(self):+ """Test getting Google optimizer."""+ optimizer = CacheOptimizerRegistry.get("google")+ assert isinstance(optimizer, GoogleCacheOptimizer)+ assert optimizer.provider == "google"+ assert optimizer.strategy == CacheStrategy.CACHED_CONTENT++ def test_get_with_config(self):+ """Test getting optimizer with custom config."""+ config = CacheConfig(min_cacheable_tokens=2048)+ optimizer = CacheOptimizerRegistry.get("anthropic", config=config, cached=False)+ assert optimizer.config.min_cacheable_tokens >= 1024 # Anthropic enforces minimum++ def test_register_custom_optimizer(self):+ """Test registering a custom optimizer."""+ CacheOptimizerRegistry.register("mock", MockOptimizer)+ try:+ optimizer = CacheOptimizerRegistry.get("mock")+ assert isinstance(optimizer, MockOptimizer)+ finally:+ CacheOptimizerRegistry.unregister("mock")++ def test_register_duplicate_raises(self):+ """Test that registering duplicate without override raises."""+ CacheOptimizerRegistry.register("test-dup", MockOptimizer)+ try:+ with pytest.raises(ValueError):+ CacheOptimizerRegistry.register("test-dup", MockOptimizer)+ finally:+ CacheOptimizerRegistry.unregister("test-dup")++ def test_register_with_override(self):+ """Test registering with override."""+ CacheOptimizerRegistry.register("test-override", MockOptimizer)+ try:+ CacheOptimizerRegistry.register("test-override", MockOptimizer, override=True)+ optimizer = CacheOptimizerRegistry.get("test-override")+ assert isinstance(optimizer, MockOptimizer)+ finally:+ CacheOptimizerRegistry.unregister("test-override")++ def test_get_unknown_provider_raises(self):+ """Test getting unknown provider raises KeyError."""+ with pytest.raises(KeyError):+ CacheOptimizerRegistry.get("unknown-provider")++ def test_list_providers(self):+ """Test listing providers."""+ providers = CacheOptimizerRegistry.list_providers()+ assert "anthropic" in providers+ assert "openai" in providers+ assert "google" in providers++ def test_is_registered(self):+ """Test is_registered check."""+ assert CacheOptimizerRegistry.is_registered("anthropic")+ assert not CacheOptimizerRegistry.is_registered("nonexistent")++ def test_cached_instances(self):+ """Test that cached instances are reused."""+ opt1 = CacheOptimizerRegistry.get("anthropic", cached=True)+ opt2 = CacheOptimizerRegistry.get("anthropic", cached=True)+ assert opt1 is opt2++ def test_uncached_instances(self):+ """Test that uncached instances are not reused."""+ opt1 = CacheOptimizerRegistry.get("anthropic", cached=False)+ opt2 = CacheOptimizerRegistry.get("anthropic", cached=False)+ assert opt1 is not opt2++ def test_tier_based_selection(self):+ """Test tier-based optimizer selection."""+ # OSS tier should work+ oss_opt = CacheOptimizerRegistry.get("anthropic", tier="oss")+ assert oss_opt is not None++ # Enterprise tier falls back to OSS if not registered+ ent_opt = CacheOptimizerRegistry.get("anthropic", tier="enterprise")+ assert ent_opt is not Nonediff --git a/tests/test_cache/test_semantic.py b/tests/test_cache/test_semantic.pynew file mode 100644index 000000000..3cda666bd--- /dev/null+++ b/tests/test_cache/test_semantic.py@@ -0,0 +1,297 @@+"""Tests for SemanticCache and SemanticCacheLayer."""++import pytest+import time+from headroom.cache import (+ SemanticCacheLayer,+ SemanticCache,+ AnthropicCacheOptimizer,+ OptimizationContext,+)+from headroom.cache.semantic import SemanticCacheConfig, CacheEntry+++class TestSemanticCacheConfig:+ """Test SemanticCacheConfig."""++ def test_default_values(self):+ """Test default configuration values."""+ config = SemanticCacheConfig()+ assert config.similarity_threshold == 0.95+ assert config.max_entries == 1000+ assert config.ttl_seconds == 300+ assert config.use_exact_matching is True+++class TestSemanticCache:+ """Test SemanticCache functionality."""++ @pytest.fixture+ def cache(self):+ """Create cache instance."""+ config = SemanticCacheConfig(+ max_entries=10,+ ttl_seconds=60,+ )+ return SemanticCache(config)++ def test_put_and_get_exact_match(self, cache):+ """Test storing and retrieving with exact hash matching."""+ response = {"text": "Hello, how can I help?"}+ cache.put("What is the weather?", response, messages_hash="hash123")++ entry = cache.get("What is the weather?", messages_hash="hash123")+ assert entry is not None+ assert entry.response == response++ def test_get_miss(self, cache):+ """Test cache miss."""+ entry = cache.get("Unknown query", messages_hash="unknown")+ assert entry is None++ def test_lru_eviction(self):+ """Test LRU eviction when at capacity."""+ config = SemanticCacheConfig(max_entries=3)+ cache = SemanticCache(config)++ # Fill cache+ cache.put("query1", "response1", messages_hash="h1")+ cache.put("query2", "response2", messages_hash="h2")+ cache.put("query3", "response3", messages_hash="h3")++ # Access query1 to make it recently used+ cache.get("query1", messages_hash="h1")++ # Add new entry, should evict query2 (oldest unused)+ cache.put("query4", "response4", messages_hash="h4")++ # query1 should still be there (recently accessed)+ assert cache.get("query1", messages_hash="h1") is not None+ # query2 should be evicted+ assert cache.get("query2", messages_hash="h2") is None+ # query3 and query4 should be there+ assert cache.get("query3", messages_hash="h3") is not None+ assert cache.get("query4", messages_hash="h4") is not None++ def test_ttl_expiration(self):+ """Test TTL expiration."""+ config = SemanticCacheConfig(ttl_seconds=1)+ cache = SemanticCache(config)++ cache.put("expiring query", "response", messages_hash="exp1")++ # Should be available immediately+ assert cache.get("expiring query", messages_hash="exp1") is not None++ # Wait for TTL+ time.sleep(1.1)++ # Should be expired+ assert cache.get("expiring query", messages_hash="exp1") is None++ def test_invalidate(self, cache):+ """Test invalidating an entry."""+ key = cache.put("query", "response", messages_hash="inv1")++ assert cache.get("query", messages_hash="inv1") is not None++ cache.invalidate(key)++ assert cache.get("query", messages_hash="inv1") is None++ def test_clear(self, cache):+ """Test clearing cache."""+ cache.put("query1", "response1", messages_hash="c1")+ cache.put("query2", "response2", messages_hash="c2")++ cache.clear()++ stats = cache.get_stats()+ assert stats["entries"] == 0++ def test_stats(self, cache):+ """Test statistics."""+ cache.put("query", "response", messages_hash="s1")+ cache.get("query", messages_hash="s1") # hit+ cache.get("unknown", messages_hash="unknown") # miss++ stats = cache.get_stats()++ assert stats["entries"] == 1+ assert stats["hits"] == 1+ assert stats["misses"] == 1+ assert stats["hit_rate"] == 0.5++ def test_access_count(self, cache):+ """Test that access count is tracked."""+ cache.put("query", "response", messages_hash="ac1")++ # Access multiple times+ for _ in range(5):+ entry = cache.get("query", messages_hash="ac1")++ # Initial count is 1, plus 5 accesses = 6+ assert entry.access_count == 6++ def test_semantic_similarity_with_embedding_fn(self):+ """Test semantic similarity with custom embedding function."""++ def mock_embedding(text: str) -> list[float]:+ # Simple mock: return consistent embedding for similar queries+ if "weather" in text.lower():+ return [1.0, 0.0, 0.0]+ elif "time" in text.lower():+ return [0.0, 1.0, 0.0]+ else:+ return [0.0, 0.0, 1.0]++ config = SemanticCacheConfig(similarity_threshold=0.9)+ cache = SemanticCache(config, embedding_fn=mock_embedding)++ # Store a weather query+ cache.put("What is the weather today?", "It's sunny", messages_hash="w1")++ # Similar weather query should hit+ entry = cache.get("How is the weather?")+ assert entry is not None+ assert entry.response == "It's sunny"++ # Different query should miss+ entry = cache.get("What time is it?")+ assert entry is None+++class TestSemanticCacheLayer:+ """Test SemanticCacheLayer functionality."""++ @pytest.fixture+ def layer(self):+ """Create cache layer with Anthropic optimizer."""+ optimizer = AnthropicCacheOptimizer()+ return SemanticCacheLayer(+ optimizer,+ similarity_threshold=0.95,+ max_entries=100,+ ttl_seconds=60,+ )++ @pytest.fixture+ def context(self):+ """Create optimization context."""+ return OptimizationContext(+ provider="anthropic",+ model="claude-3-opus",+ )++ def test_process_no_cache_hit(self, layer, context):+ """Test processing with no cache hit."""+ messages = [+ {"role": "system", "content": "You are helpful."},+ {"role": "user", "content": "Hello!"},+ ]++ result = layer.process(messages, context)++ assert result.semantic_cache_hit is False+ assert result.cached_response is None++ def test_process_with_cache_hit(self, layer, context):+ """Test processing with cache hit."""+ messages = [+ {"role": "system", "content": "You are helpful."},+ {"role": "user", "content": "What is 2+2?"},+ ]++ # First, store a response+ layer.store_response(messages, {"text": "4"}, context)++ # Now process same messages+ result = layer.process(messages, context)++ assert result.semantic_cache_hit is True+ assert result.cached_response == {"text": "4"}++ def test_store_response(self, layer, context):+ """Test storing a response."""+ messages = [+ {"role": "system", "content": "You are helpful."},+ {"role": "user", "content": "Tell me a joke"},+ ]++ key = layer.store_response(messages, {"text": "Why did..."}, context)++ assert key is not None+ assert len(key) > 0++ def test_get_stats(self, layer, context):+ """Test getting statistics."""+ messages = [+ {"role": "system", "content": "You are helpful."},+ {"role": "user", "content": "Hello"},+ ]++ layer.process(messages, context)+ stats = layer.get_stats()++ assert "semantic_cache" in stats+ assert "provider_optimizer" in stats+ assert stats["provider_optimizer"] == "anthropic-cache-optimizer"++ def test_query_extraction(self, layer, context):+ """Test query extraction from messages."""+ messages = [+ {"role": "system", "content": "System"},+ {"role": "user", "content": "First question"},+ {"role": "assistant", "content": "Answer"},+ {"role": "user", "content": "Second question"},+ ]++ # Store response+ layer.store_response(messages, {"text": "Response"}, context)++ # The query should be the last user message+ result = layer.process(messages, context)+ assert result.semantic_cache_hit is True++ def test_query_from_context(self, layer):+ """Test using query from context."""+ messages = [+ {"role": "user", "content": "Some message"},+ ]+ context = OptimizationContext(+ query="Specific query for caching",+ )++ layer.store_response(messages, {"text": "Response"}, context)+ result = layer.process(messages, context)++ assert result.semantic_cache_hit is True++ def test_provider_optimizer_fallback(self, layer, context):+ """Test that provider optimizer is used on cache miss."""+ messages = [+ {"role": "system", "content": "You are helpful. " * 500},+ {"role": "user", "content": "New uncached question"},+ ]++ result = layer.process(messages, context)++ # Should have used provider optimizer+ assert result.semantic_cache_hit is False+ # Provider optimizer should have processed+ assert result.metrics.stable_prefix_hash != ""++ def test_content_block_query_extraction(self, layer, context):+ """Test query extraction from content block format."""+ messages = [+ {"role": "system", "content": "System"},+ {+ "role": "user",+ "content": [{"type": "text", "text": "Block format question"}],+ },+ ]++ layer.store_response(messages, {"text": "Response"}, context)+ result = layer.process(messages, context)++ assert result.semantic_cache_hit is True