Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
Add adaptive compression sizing and fix cost tracking bugs
chopratejase8911efe9055
diff --git a/headroom/config.py b/headroom/config.pyindex c3fbc7534..86964733d 100644--- a/headroom/config.py+++ b/headroom/config.py@@ -348,24 +348,60 @@ class AnchorConfig:# Default tools to exclude from compression (local file/code tools)-# These tools return precise content (line numbers, paths, code) where-# exact fidelity matters. Web tools are NOT excluded - they benefit from-# compression and can use CCR for retrieval if needed.+# Read: Returns exact file content needed for Edit tool's old_string matching.+# Compressing would break the edit workflow.+# Glob: Returns compact file path lists used for navigation. Low token count,+# not worth compressing.+# Grep/Bash are NOT excluded - their outputs (search results, build logs,+# test output) are ideal compression targets for SearchCompressor/LogCompressor,+# and CCR provides safe retrieval if the LLM needs more detail.DEFAULT_EXCLUDE_TOOLS: frozenset[str] = frozenset({"Read","Glob",- "Grep",- "Bash",# Lowercase variants for case-insensitive matching"read","glob",- "grep",- "bash",})+@dataclass+class CompressionProfile:+ """Per-tool compression bias applied to statistically-determined K.++ Instead of hardcoding max_items=15, the adaptive sizer computes the optimal K+ via information saturation (Kneedle on unique bigram coverage). This profile+ applies a bias multiplier: >1 keeps more items (conservative), <1 keeps fewer+ (aggressive).+ """++ bias: float = 1.0 # 0.7=aggressive, 1.0=moderate, 1.5=conservative+ min_k: int = 3 # Never keep fewer than this+ max_k: int | None = None # Cap (None = no cap, let statistics decide)+++# Named presets for convenience+PROFILE_PRESETS: dict[str, CompressionProfile] = {+ "conservative": CompressionProfile(bias=1.5, min_k=5),+ "moderate": CompressionProfile(bias=1.0, min_k=3),+ "aggressive": CompressionProfile(bias=0.7, min_k=3),+}++# Default per-tool profiles: tools not listed here use moderate (bias=1.0)+DEFAULT_TOOL_PROFILES: dict[str, CompressionProfile] = {+ # Search results: keep more matches for accuracy+ "Grep": PROFILE_PRESETS["conservative"],+ "grep": PROFILE_PRESETS["conservative"],+ # Logs/output: balanced compression+ "Bash": PROFILE_PRESETS["moderate"],+ "bash": PROFILE_PRESETS["moderate"],+ # Web pages are verbose, compress aggressively+ "WebFetch": PROFILE_PRESETS["aggressive"],+ "webfetch": PROFILE_PRESETS["aggressive"],+}++@dataclassclass SmartCrusherConfig:"""Configuration for smart statistical crusher (DEFAULT).diff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex 037cd4cea..bfe83c39c 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -247,6 +247,9 @@ class ProxyConfig:# Code-aware compression (ON by default if installed)code_aware_enabled: bool = True # Enable AST-based code compression+ # Per-tool compression profiles (parsed from CLI/env)+ tool_profiles: dict[str, Any] | None = None+# Smart content routing (routes each message to optimal compressor)smart_routing: bool = True # Use ContentRouter for intelligent compression@@ -536,6 +539,39 @@ class CostTracker:self._total_savings_usd: float = 0self._last_prune_time: datetime = datetime.now()+ @staticmethod+ def _resolve_litellm_model(model: str) -> str:+ """Resolve model name to one LiteLLM recognizes, adding provider prefix if needed."""+ if not LITELLM_AVAILABLE:+ return model++ # Try as-is first+ try:+ litellm.cost_per_token(model=model, prompt_tokens=1, completion_tokens=0)+ return model+ except Exception:+ pass++ # Try with provider prefix+ prefixes = {+ "claude-": "anthropic/",+ "gpt-": "openai/",+ "o1-": "openai/",+ "o3-": "openai/",+ "o4-": "openai/",+ "gemini-": "google/",+ }+ for pattern, prefix in prefixes.items():+ if model.startswith(pattern):+ prefixed = f"{prefix}{model}"+ try:+ litellm.cost_per_token(model=prefixed, prompt_tokens=1, completion_tokens=0)+ return prefixed+ except Exception:+ break++ return model+def estimate_cost(self,model: str,@@ -558,6 +594,9 @@ class CostTracker:return Nonetry:+ # Resolve model name (adds provider prefix if needed, e.g. claude-opus-4-6 → anthropic/claude-opus-4-6)+ resolved_model = self._resolve_litellm_model(model)+# cost_per_token returns (total_input_cost, total_output_cost) for the given token counts# Despite the name, it returns total cost not per-token cost@@ -570,14 +609,14 @@ class CostTracker:# Get cost for regular (non-cached) input tokensinput_cost, _ = litellm.cost_per_token(- model=model,+ model=resolved_model,prompt_tokens=regular_input,completion_tokens=0,)# Get cost for output tokens_, output_cost = litellm.cost_per_token(- model=model,+ model=resolved_model,prompt_tokens=0,completion_tokens=output_tokens,)@@ -585,7 +624,7 @@ class CostTracker:# Get model info for cache pricingmodel_info: dict[str, Any] = {}try:- model_info = dict(litellm.get_model_info(model))+ model_info = dict(litellm.get_model_info(resolved_model))except Exception:pass@@ -598,7 +637,7 @@ class CostTracker:else:# Fallback: most providers charge ~10% of input price for cache readscache_read_full_cost, _ = litellm.cost_per_token(- model=model,+ model=resolved_model,prompt_tokens=cache_read_tokens,completion_tokens=0,)@@ -613,7 +652,7 @@ class CostTracker:else:# Fallback: most providers charge ~125% of input price for cache writescache_write_full_cost, _ = litellm.cost_per_token(- model=model,+ model=resolved_model,prompt_tokens=cache_write_tokens,completion_tokens=0,)@@ -999,6 +1038,7 @@ class HeadroomProxy:router_config = ContentRouterConfig(enable_llmlingua=config.llmlingua_enabled,enable_code_aware=config.code_aware_enabled,+ tool_profiles=config.tool_profiles,)transforms = [CacheAligner(CacheAlignerConfig(enabled=True)),@@ -2023,12 +2063,13 @@ class HeadroomProxy:cache_read_tokens=cache_read_tokens,cache_write_tokens=cache_write_tokens,)+ # original_cost: what it would have cost without compression+ # Use only original_tokens at regular input rate — no cache params,+ # since caching is orthogonal to compression savingsoriginal_cost = self.cost_tracker.estimate_cost(model,original_tokens,output_tokens,- cache_read_tokens=cache_read_tokens,- cache_write_tokens=cache_write_tokens,)if cost_usd and original_cost:savings_usd = original_cost - cost_usd@@ -4128,11 +4169,12 @@ class HeadroomProxy:output_tokens,cache_read_tokens=cache_read_tokens,)+ # original_cost: what it would have cost without compression+ # No cache params — caching is orthogonal to compression savingsoriginal_cost = self.cost_tracker.estimate_cost(model,original_tokens,output_tokens,- cache_read_tokens=cache_read_tokens,)if cost_usd and original_cost:savings_usd = original_cost - cost_usd@@ -5183,11 +5225,12 @@ class HeadroomProxy:output_tokens,cache_read_tokens=cache_read_tokens,)+ # original_cost: what it would have cost without compression+ # No cache params — caching is orthogonal to compression savingsoriginal_cost = self.cost_tracker.estimate_cost(model,original_tokens,output_tokens,- cache_read_tokens=cache_read_tokens,)if cost_usd and original_cost:savings_usd = original_cost - cost_usd@@ -6627,6 +6670,45 @@ def _get_env_str(name: str, default: str) -> str:return os.environ.get(name, default)+def _parse_tool_profiles(cli_profiles: list[str]) -> dict[str, Any]:+ """Parse tool profiles from CLI args and HEADROOM_TOOL_PROFILES env var.++ Format: ToolName:level (e.g., Grep:conservative, Bash:moderate)+ Env var format: comma-separated (e.g., "Grep:conservative,Bash:moderate")++ Returns:+ Dict mapping tool names to CompressionProfile instances.+ """+ from headroom.config import PROFILE_PRESETS, CompressionProfile++ profiles: dict[str, CompressionProfile] = {}+ raw_entries: list[str] = list(cli_profiles)++ # Also check env var+ env_val = os.environ.get("HEADROOM_TOOL_PROFILES", "")+ if env_val:+ raw_entries.extend(e.strip() for e in env_val.split(",") if e.strip())++ for entry in raw_entries:+ if ":" not in entry:+ logger.warning("Invalid tool profile format (expected ToolName:level): %s", entry)+ continue+ tool_name, level = entry.split(":", 1)+ tool_name = tool_name.strip()+ level = level.strip().lower()++ if level in PROFILE_PRESETS:+ profiles[tool_name] = PROFILE_PRESETS[level]+ else:+ logger.warning(+ "Unknown profile level '%s' for tool '%s'. Use: conservative, moderate, aggressive",+ level,+ tool_name,+ )++ return profiles++if __name__ == "__main__":parser = argparse.ArgumentParser(description="Headroom Proxy Server")@@ -6690,6 +6772,13 @@ if __name__ == "__main__":parser.add_argument("--no-optimize", action="store_true", help="Disable optimization")parser.add_argument("--min-tokens", type=int, default=500, help="Min tokens to crush")parser.add_argument("--max-items", type=int, default=50, help="Max items after crush")+ parser.add_argument(+ "--tool-profile",+ action="append",+ default=[],+ help="Per-tool compression profile: ToolName:level (e.g., Grep:conservative, Bash:moderate, WebFetch:aggressive). "+ "Can be specified multiple times. Also settable via HEADROOM_TOOL_PROFILES env var.",+ )# Cachingparser.add_argument("--no-cache", action="store_true", help="Disable caching")@@ -6783,6 +6872,9 @@ if __name__ == "__main__":if hasattr(args, "openrouter_api_key") and args.openrouter_api_key:os.environ["OPENROUTER_API_KEY"] = args.openrouter_api_key+ # Parse per-tool compression profiles from CLI and env var+ tool_profiles = _parse_tool_profiles(args.tool_profile)+config = ProxyConfig(host=_get_env_str("HEADROOM_HOST", args.host),port=_get_env_int("HEADROOM_PORT", args.port),@@ -6814,6 +6906,7 @@ if __name__ == "__main__":max_connections=_get_env_int("HEADROOM_MAX_CONNECTIONS", args.max_connections),max_keepalive_connections=_get_env_int("HEADROOM_MAX_KEEPALIVE", args.max_keepalive),http2=not args.no_http2 and _get_env_bool("HEADROOM_HTTP2", True),+ tool_profiles=tool_profiles if tool_profiles else None,)# Get worker and concurrency settingsdiff --git a/headroom/transforms/adaptive_sizer.py b/headroom/transforms/adaptive_sizer.pynew file mode 100644index 000000000..e61ce85a2--- /dev/null+++ b/headroom/transforms/adaptive_sizer.py@@ -0,0 +1,289 @@+"""Adaptive compression sizing via information saturation detection.++Instead of hardcoded max_items/max_matches, this module statistically determines+how many items to keep by finding the "knee point" — where adding more items+stops providing meaningful new information.++Algorithm: Track unique bigrams as items are added in importance order. Build a+cumulative coverage curve. Find the knee (Kneedle algorithm) where marginal+information gain drops sharply. That's the optimal K.++Per-tool profiles apply a bias multiplier on the statistically-determined K:+- conservative (bias=1.5): keep 50% more than mathematically needed+- moderate (bias=1.0): trust the statistics+- aggressive (bias=0.7): compress harder+"""++from __future__ import annotations++import hashlib+import logging+import zlib+from collections.abc import Sequence++logger = logging.getLogger(__name__)+++def compute_optimal_k(+ items: Sequence[str],+ bias: float = 1.0,+ min_k: int = 3,+ max_k: int | None = None,+) -> int:+ """Compute the optimal number of items to keep using information saturation.++ Three-tier decision system:+ Tier 1 (fast path): trivial cases, near-duplicate detection+ Tier 2 (standard): Kneedle on unique bigram coverage curve+ Tier 3 (validation): zlib compression ratio sanity check++ Args:+ items: Sequence of string representations of items (in importance order).+ bias: Multiplier on the knee point. >1 = keep more, <1 = keep fewer.+ min_k: Never return fewer than this.+ max_k: Never return more than this (None = no cap).++ Returns:+ Optimal number of items to keep.+ """+ n = len(items)+ effective_max = max_k if max_k is not None else n++ # Tier 1: Fast path+ if n <= 8:+ return n++ # Check for near-total redundancy+ unique_count = count_unique_simhash(items)+ if unique_count <= 3:+ k = max(min_k, unique_count)+ return min(k, effective_max)++ # Tier 2: Kneedle on unique bigram coverage+ curve = compute_unique_bigram_curve(items)+ knee = find_knee(curve)++ if knee is None:+ # No clear knee — content is uniformly diverse, keep ~30% as heuristic+ knee = max(min_k, int(n * 0.3))++ # Apply bias multiplier+ k = max(min_k, int(knee * bias))+ k = min(k, effective_max)++ # Tier 3: Validate with zlib compression ratio+ k = _validate_with_zlib(items, k, effective_max)++ k = max(min_k, min(k, effective_max))++ logger.debug(+ "adaptive_sizer: n=%d unique=%d knee=%s bias=%.1f → k=%d",+ n,+ unique_count,+ knee,+ bias,+ k,+ )+ return k+++def find_knee(curve: list[int]) -> int | None:+ """Find the knee point in a monotonically increasing curve.++ Uses the Kneedle algorithm: normalize to [0,1], compute the difference+ from the y=x diagonal, return the index of maximum difference.++ Args:+ curve: List of cumulative values (e.g., unique bigram counts).++ Returns:+ Index of the knee point, or None if no clear knee exists.+ """+ n = len(curve)+ if n < 3:+ return None++ # Normalize x and y to [0, 1]+ x_min, x_max = 0, n - 1+ y_min, y_max = curve[0], curve[-1]++ if y_max == y_min:+ # Flat curve — all items are identical+ return 1++ x_range = x_max - x_min+ y_range = y_max - y_min++ # Compute difference from the diagonal (y = x in normalized space)+ max_diff = -1.0+ knee_idx = None++ for i in range(n):+ x_norm = (i - x_min) / x_range+ y_norm = (curve[i] - y_min) / y_range+ diff = y_norm - x_norm # For concave curves, knee is where this is maximized+ if diff > max_diff:+ max_diff = diff+ knee_idx = i++ # Require a meaningful deviation from diagonal+ if max_diff < 0.05:+ return None++ # Knee is at knee_idx, but we want to include items up to and including the knee+ # Add 1 because we're converting from 0-indexed to count+ return knee_idx + 1 if knee_idx is not None else None+++def compute_unique_bigram_curve(items: Sequence[str]) -> list[int]:+ """Build cumulative unique bigram coverage curve.++ For each item (in order), extracts word-level bigrams, adds them to a+ running set, and records the total unique count.++ Args:+ items: Sequence of string items in importance order.++ Returns:+ List where curve[k] = number of unique bigrams after seeing items[0:k+1].+ """+ seen_bigrams: set[tuple[str, str]] = set()+ curve: list[int] = []++ for item in items:+ words = item.lower().split()+ if len(words) < 2:+ # For single-word items, use the word itself as a "unigram bigram"+ seen_bigrams.add((words[0] if words else "", ""))+ else:+ for j in range(len(words) - 1):+ seen_bigrams.add((words[j], words[j + 1]))+ curve.append(len(seen_bigrams))++ return curve+++def _simhash(text: str) -> int:+ """Compute a 64-bit SimHash fingerprint for a text string.++ Uses character 4-grams hashed to 64-bit values, then aggregates+ via weighted bit voting.++ Args:+ text: Input text.++ Returns:+ 64-bit integer fingerprint.+ """+ v = [0] * 64+ text_lower = text.lower()++ # Character 4-grams+ for i in range(max(1, len(text_lower) - 3)):+ gram = text_lower[i : i + 4]+ h = int(hashlib.md5(gram.encode(), usedforsecurity=False).hexdigest()[:16], 16)+ for j in range(64):+ if h & (1 << j):+ v[j] += 1+ else:+ v[j] -= 1++ fingerprint = 0+ for j in range(64):+ if v[j] > 0:+ fingerprint |= 1 << j+ return fingerprint+++def _hamming_distance(a: int, b: int) -> int:+ """Count differing bits between two 64-bit integers."""+ return bin(a ^ b).count("1")+++def count_unique_simhash(items: Sequence[str], threshold: int = 3) -> int:+ """Count items with distinct content using SimHash.++ Groups items by SimHash fingerprint similarity (Hamming distance <= threshold).+ Returns the number of distinct groups.++ Args:+ items: Sequence of string items.+ threshold: Max Hamming distance to consider items as duplicates.++ Returns:+ Number of unique content groups.+ """+ if not items:+ return 0++ # Compute fingerprints+ fingerprints = [_simhash(item) for item in items]++ # Greedy clustering: assign each item to the first matching cluster+ clusters: list[int] = [] # Representative fingerprint per cluster+ for fp in fingerprints:+ matched = False+ for rep in clusters:+ if _hamming_distance(fp, rep) <= threshold:+ matched = True+ break+ if not matched:+ clusters.append(fp)++ return len(clusters)+++def _validate_with_zlib(+ items: Sequence[str],+ k: int,+ max_k: int,+ tolerance: float = 0.15,+) -> int:+ """Validate K using zlib compression ratio comparison.++ If the compression ratio of the selected subset differs significantly+ from the full set, increase K.++ Args:+ items: All items.+ k: Currently proposed K.+ max_k: Maximum allowed K.+ tolerance: Max allowed ratio difference (default 15%).++ Returns:+ Adjusted K (may be increased if validation fails).+ """+ if k >= len(items) or k >= max_k:+ return k++ full_text = "\n".join(items).encode()+ subset_text = "\n".join(items[:k]).encode()++ # Skip validation for very small content (zlib overhead dominates)+ if len(full_text) < 200:+ return k++ full_compressed = len(zlib.compress(full_text, level=1))+ subset_compressed = len(zlib.compress(subset_text, level=1))++ full_ratio = full_compressed / len(full_text) if full_text else 1.0+ subset_ratio = subset_compressed / len(subset_text) if subset_text else 1.0++ # If subset compresses much better than full, it's missing diverse content+ # A lower ratio means more redundancy. If subset ratio is much lower,+ # it means the subset is more redundant than the full set — we're missing info.+ ratio_diff = abs(full_ratio - subset_ratio)++ if ratio_diff > tolerance:+ # Increase K by 20% to capture more diversity+ adjusted_k = min(int(k * 1.2), max_k)+ logger.debug(+ "zlib validation: ratio_diff=%.3f > %.3f, adjusting k=%d → %d",+ ratio_diff,+ tolerance,+ k,+ adjusted_k,+ )+ return adjusted_k++ return kdiff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.pyindex 61a8ad7d9..852a11822 100644--- a/headroom/transforms/content_router.py+++ b/headroom/transforms/content_router.py@@ -292,6 +292,10 @@ class ContentRouterConfig:# Set to None to use DEFAULT_EXCLUDE_TOOLS, or provide custom setexclude_tools: set[str] | None = None+ # Per-tool compression profiles (tool_name → CompressionProfile)+ # Set to None to use DEFAULT_TOOL_PROFILES from config+ tool_profiles: dict[str, Any] | None = None+# Patterns for detecting mixed content_CODE_FENCE_PATTERN = re.compile(r"^```(\w*)\s*$", re.MULTILINE)@@ -588,6 +592,7 @@ class ContentRouter(Transform):content: str,context: str = "",question: str | None = None,+ bias: float = 1.0,) -> RouterCompressionResult:"""Compress content using optimal strategy based on content detection.@@ -596,6 +601,7 @@ class ContentRouter(Transform):context: Optional context for relevance-aware compression.question: Optional question for QA-aware compression. When provided,tokens relevant to answering this question are preserved.+ bias: Compression bias multiplier (>1 = keep more, <1 = keep fewer).Returns:RouterCompressionResult with compressed content and routing metadata.@@ -612,9 +618,9 @@ class ContentRouter(Transform):strategy = self._determine_strategy(content)if strategy == CompressionStrategy.MIXED:- return self._compress_mixed(content, context, question)+ return self._compress_mixed(content, context, question, bias=bias)else:- return self._compress_pure(content, strategy, context, question)+ return self._compress_pure(content, strategy, context, question, bias=bias)def _determine_strategy(self, content: str) -> CompressionStrategy:"""Determine the compression strategy from content analysis.@@ -668,6 +674,7 @@ class ContentRouter(Transform):content: str,context: str,question: str | None = None,+ bias: float = 1.0,) -> RouterCompressionResult:"""Compress mixed content by splitting and routing sections.@@ -675,6 +682,7 @@ class ContentRouter(Transform):content: Mixed content to compress.context: User context for relevance.question: Optional question for QA-aware compression.+ bias: Compression bias multiplier.Returns:RouterCompressionResult with reassembled content.@@ -698,7 +706,12 @@ class ContentRouter(Transform):# Compress sectionoriginal_tokens = len(section.content.split())compressed_content, compressed_tokens = self._apply_strategy_to_content(- section.content, strategy, context, section.language, question+ section.content,+ strategy,+ context,+ section.language,+ question,+ bias=bias,)# Preserve code fence markers@@ -730,6 +743,7 @@ class ContentRouter(Transform):strategy: CompressionStrategy,context: str,question: str | None = None,+ bias: float = 1.0,) -> RouterCompressionResult:"""Compress pure (non-mixed) content.@@ -738,6 +752,7 @@ class ContentRouter(Transform):strategy: Selected strategy.context: User context.question: Optional question for QA-aware compression.+ bias: Compression bias multiplier.Returns:RouterCompressionResult.@@ -745,7 +760,7 @@ class ContentRouter(Transform):original_tokens = len(content.split())compressed, compressed_tokens = self._apply_strategy_to_content(- content, strategy, context, question=question+ content, strategy, context, question=question, bias=bias)return RouterCompressionResult(@@ -769,6 +784,7 @@ class ContentRouter(Transform):context: str,language: str | None = None,question: str | None = None,+ bias: float = 1.0,) -> tuple[str, int]:"""Apply a compression strategy to content.@@ -778,6 +794,7 @@ class ContentRouter(Transform):context: User context.language: Language hint for code.question: Optional question for QA-aware compression.+ bias: Compression bias multiplier (>1 = keep more, <1 = keep fewer).Returns:Tuple of (compressed_content, compressed_token_count).@@ -804,14 +821,14 @@ class ContentRouter(Transform):if self.config.enable_smart_crusher:crusher = self._get_smart_crusher()if crusher:- result = crusher.crush(content, query=context)+ result = crusher.crush(content, query=context, bias=bias)return result.compressed, len(result.compressed.split())elif strategy == CompressionStrategy.SEARCH:if self.config.enable_search_compressor:compressor = self._get_search_compressor()if compressor:- result = compressor.compress(content, context=context)+ result = compressor.compress(content, context=context, bias=bias)compressed, compressed_tokens = (result.compressed,len(result.compressed.split()),@@ -821,7 +838,7 @@ class ContentRouter(Transform):if self.config.enable_log_compressor:compressor = self._get_log_compressor()if compressor:- result = compressor.compress(content)+ result = compressor.compress(content, bias=bias)compressed, compressed_tokens = (result.compressed,result.compressed_line_count,@@ -1184,11 +1201,17 @@ class ContentRouter(Transform):for i, message in enumerate(messages):role = message.get("role", "")content = message.get("content", "")+ bias = 1.0 # Default bias, may be overridden for tool messages# Handle list content (Anthropic format with content blocks)if isinstance(content, list):transformed_message = self._process_content_blocks(- message, content, context, transforms_applied, excluded_tool_ids+ message,+ content,+ context,+ transforms_applied,+ excluded_tool_ids,+ tool_name_map=tool_name_map,)transformed_messages.append(transformed_message)continue@@ -1205,6 +1228,9 @@ class ContentRouter(Transform):transformed_messages.append(message)transforms_applied.append("router:excluded:tool")continue+ # Look up tool-specific compression bias for OpenAI tool messages+ tool_name = tool_name_map.get(tool_call_id, "")+ bias = self._get_tool_bias(tool_name) if tool_name else 1.0# Protection 1: Never compress user messagesif self.config.skip_user_messages and role == "user":@@ -1239,7 +1265,9 @@ class ContentRouter(Transform):continue# Route and compress based on content detection- result = self.compress(content, context=context)+ # Use tool-specific bias for tool messages, default 1.0 for others+ msg_bias = bias if role == "tool" else 1.0+ result = self.compress(content, context=context, bias=msg_bias)if result.compression_ratio < 0.9:transformed_messages.append({**message, "content": result.compressed})@@ -1261,6 +1289,27 @@ class ContentRouter(Transform):warnings=warnings,)+ def _get_tool_bias(self, tool_name: str) -> float:+ """Look up compression bias for a tool name.++ Checks user-configured profiles first, then DEFAULT_TOOL_PROFILES.+ Returns 1.0 (moderate) if no profile is configured.+ """+ from ..config import DEFAULT_TOOL_PROFILES++ # Check user-configured profiles+ if self.config.tool_profiles:+ profile = self.config.tool_profiles.get(tool_name)+ if profile:+ return float(profile.bias)++ # Check default profiles+ profile = DEFAULT_TOOL_PROFILES.get(tool_name)+ if profile:+ return profile.bias++ return 1.0 # Default: moderate+def _process_content_blocks(self,message: dict[str, Any],@@ -1268,6 +1317,7 @@ class ContentRouter(Transform):context: str,transforms_applied: list[str],excluded_tool_ids: set[str],+ tool_name_map: dict[str, str] | None = None,) -> dict[str, Any]:"""Process content blocks (Anthropic format) for tool_result compression.@@ -1279,6 +1329,8 @@ class ContentRouter(Transform):content_blocks: List of content blocks.context: Context for compression.transforms_applied: List to append transform names to.+ excluded_tool_ids: Tool IDs to skip compression for.+ tool_name_map: Mapping from tool_call_id to tool_name for profile lookup.Returns:Transformed message with compressed content blocks.@@ -1302,12 +1354,16 @@ class ContentRouter(Transform):transforms_applied.append("router:excluded:tool")continue+ # Look up tool-specific compression bias+ tool_name = (tool_name_map or {}).get(tool_use_id, "")+ bias = self._get_tool_bias(tool_name) if tool_name else 1.0+tool_content = block.get("content", "")# Only process string contentif isinstance(tool_content, str) and len(tool_content) > 500:# Compress using content detection (will auto-detect JSON arrays, etc.)- result = self.compress(tool_content, context=context)+ result = self.compress(tool_content, context=context, bias=bias)if result.compression_ratio < 0.9:new_blocks.append({**block, "content": result.compressed})transforms_applied.append(diff --git a/headroom/transforms/log_compressor.py b/headroom/transforms/log_compressor.pyindex b23563c37..31b52c722 100644--- a/headroom/transforms/log_compressor.py+++ b/headroom/transforms/log_compressor.py@@ -176,12 +176,13 @@ class LogCompressor:"""self.config = config or LogCompressorConfig()- def compress(self, content: str, context: str = "") -> LogCompressionResult:+ def compress(self, content: str, context: str = "", bias: float = 1.0) -> LogCompressionResult:"""Compress log output.Args:content: Raw log output.context: User query context (unused for now).+ bias: Compression bias multiplier (>1 = keep more, <1 = keep fewer).Returns:LogCompressionResult with compressed output and metadata.@@ -204,8 +205,8 @@ class LogCompressor:# Parse and categorize lineslog_lines = self._parse_lines(lines)- # Select important lines- selected = self._select_lines(log_lines)+ # Select important lines (with adaptive sizing)+ selected = self._select_lines(log_lines, bias=bias)# Format output with summariescompressed, stats = self._format_output(selected, log_lines)@@ -319,10 +320,22 @@ class LogCompressor:return min(1.0, score)- def _select_lines(self, log_lines: list[LogLine]) -> list[LogLine]:- """Select important lines to keep."""+ def _select_lines(self, log_lines: list[LogLine], bias: float = 1.0) -> list[LogLine]:+ """Select important lines to keep using adaptive sizing."""+ from .adaptive_sizer import compute_optimal_k+selected: list[LogLine] = []+ # Compute adaptive line budget from the FULL log output+ # This determines "how many lines of value exist" across the entire log+ all_line_strings = [line.content for line in log_lines]+ adaptive_max = compute_optimal_k(+ all_line_strings,+ bias=bias,+ min_k=10,+ max_k=self.config.max_total_lines,+ )+# Group by categoryerrors: list[LogLine] = []fails: list[LogLine] = []@@ -381,11 +394,11 @@ class LogCompressor:# Sort by line number and dedupeselected = sorted(set(selected), key=lambda x: x.line_number)- # Limit total lines- if len(selected) > self.config.max_total_lines:+ # Apply adaptive line limit+ if len(selected) > adaptive_max:# Keep most important linesselected = sorted(selected, key=lambda x: x.score, reverse=True)- selected = selected[: self.config.max_total_lines]+ selected = selected[:adaptive_max]selected = sorted(selected, key=lambda x: x.line_number)return selecteddiff --git a/headroom/transforms/search_compressor.py b/headroom/transforms/search_compressor.pyindex 19ddcd011..b80566714 100644--- a/headroom/transforms/search_compressor.py+++ b/headroom/transforms/search_compressor.py@@ -106,12 +106,14 @@ class SearchCompressor:self,content: str,context: str = "",+ bias: float = 1.0,) -> SearchCompressionResult:"""Compress search results.Args:content: Raw grep/ripgrep output.context: User query context for relevance scoring.+ bias: Compression bias multiplier (>1 = keep more, <1 = keep fewer).Returns:SearchCompressionResult with compressed output and metadata.@@ -135,8 +137,8 @@ class SearchCompressor:# Score matches by relevanceself._score_matches(file_matches, context)- # Select top matches per file- selected = self._select_matches(file_matches)+ # Select top matches per file (with adaptive sizing)+ selected = self._select_matches(file_matches, bias=bias)# Format compressed outputcompressed, summaries = self._format_output(selected, file_matches)@@ -235,8 +237,11 @@ class SearchCompressor:def _select_matches(self,file_matches: dict[str, FileMatches],+ bias: float = 1.0,) -> dict[str, FileMatches]:- """Select top matches per file and globally."""+ """Select top matches per file and globally using adaptive sizing."""+ from .adaptive_sizer import compute_optimal_k+selected: dict[str, FileMatches] = {}# Sort files by total match score (highest first)@@ -249,9 +254,22 @@ class SearchCompressor:# Limit number of filessorted_files = sorted_files[: self.config.max_files]+ # Compute adaptive total matches limit+ all_match_strings = [+ f"{file_path}:{m.line_number}:{m.content}"+ for file_path, fm in sorted_files+ for m in fm.matches+ ]+ adaptive_total = compute_optimal_k(+ all_match_strings,+ bias=bias,+ min_k=5,+ max_k=self.config.max_total_matches,+ )+total_selected = 0for file_path, fm in sorted_files:- if total_selected >= self.config.max_total_matches:+ if total_selected >= adaptive_total:break# Sort matches by score@@ -261,7 +279,7 @@ class SearchCompressor:file_selected: list[SearchMatch] = []remaining_slots = min(self.config.max_matches_per_file,- self.config.max_total_matches - total_selected,+ adaptive_total - total_selected,)# Always include first and last if configureddiff --git a/headroom/transforms/smart_crusher.py b/headroom/transforms/smart_crusher.pyindex 3c1f0c903..2af074eaa 100644--- a/headroom/transforms/smart_crusher.py+++ b/headroom/transforms/smart_crusher.py@@ -1440,7 +1440,7 @@ class SmartCrusher(Transform):CompressionStrategy.SMART_SAMPLE: AnchorDataPattern.GENERIC,}.get(strategy, AnchorDataPattern.GENERIC)- def crush(self, content: str, query: str = "") -> CrushResult:+ def crush(self, content: str, query: str = "", bias: float = 1.0) -> CrushResult:"""Crush content string directly (for use by ContentRouter).This is a simplified interface for compressing a single content string,@@ -1449,12 +1449,13 @@ class SmartCrusher(Transform):Args:content: JSON string content to compress.query: Query context for relevance-based compression.+ bias: Compression bias multiplier (>1 = keep more, <1 = keep fewer).Returns:CrushResult with compressed content and metadata."""compressed, was_modified, analysis_info = self._smart_crush_content(- content, query_context=query+ content, query_context=query, bias=bias)return CrushResult(compressed=compressed,@@ -2088,7 +2089,11 @@ class SmartCrusher(Transform):return " ".join(context_parts)def _smart_crush_content(- self, content: str, query_context: str = "", tool_name: str | None = None+ self,+ content: str,+ query_context: str = "",+ tool_name: str | None = None,+ bias: float = 1.0,) -> tuple[str, bool, str]:"""Apply smart crushing to content.@@ -2100,6 +2105,7 @@ class SmartCrusher(Transform):content: Content to crush (JSON or plain text).query_context: Context string from user messages for relevance scoring.tool_name: Name of the tool that produced this output.+ bias: Compression bias multiplier (>1 = keep more, <1 = keep fewer).Returns:Tuple of (crushed_content, was_modified, analysis_info).@@ -2113,7 +2119,7 @@ class SmartCrusher(Transform):# Recursively process and crush arrayscrushed, info, ccr_markers = self._process_value(- parsed, query_context=query_context, tool_name=tool_name+ parsed, query_context=query_context, tool_name=tool_name, bias=bias)result = safe_json_dumps(crushed, indent=None)@@ -2132,7 +2138,12 @@ class SmartCrusher(Transform):return result, was_modified, infodef _process_value(- self, value: Any, depth: int = 0, query_context: str = "", tool_name: str | None = None+ self,+ value: Any,+ depth: int = 0,+ query_context: str = "",+ tool_name: str | None = None,+ bias: float = 1.0,) -> tuple[Any, str, list[tuple[str, int, int]]]:"""Recursively process a value, crushing arrays where appropriate.@@ -2148,7 +2159,9 @@ class SmartCrusher(Transform):# Must have enough items AND all items must be dicts (not mixed types)all_dicts = value and all(isinstance(item, dict) for item in value)if len(value) >= self.config.min_items_to_analyze and all_dicts:- crushed, strategy, ccr_hash = self._crush_array(value, query_context, tool_name)+ crushed, strategy, ccr_hash = self._crush_array(+ value, query_context, tool_name, bias=bias+ )info_parts.append(f"{strategy}({len(value)}->{len(crushed)})")# Track CCR marker for later injection@@ -2161,7 +2174,7 @@ class SmartCrusher(Transform):processed = []for item in value:p_item, p_info, p_markers = self._process_value(- item, depth + 1, query_context, tool_name+ item, depth + 1, query_context, tool_name, bias=bias)processed.append(p_item)if p_info:@@ -2174,7 +2187,7 @@ class SmartCrusher(Transform):processed_dict: dict[str, Any] = {}for k, v in value.items():p_val, p_info, p_markers = self._process_value(- v, depth + 1, query_context, tool_name+ v, depth + 1, query_context, tool_name, bias=bias)processed_dict[k] = p_valif p_info:@@ -2186,7 +2199,11 @@ class SmartCrusher(Transform):return value, "", []def _crush_array(- self, items: list[dict], query_context: str = "", tool_name: str | None = None+ self,+ items: list[dict],+ query_context: str = "",+ tool_name: str | None = None,+ bias: float = 1.0,) -> tuple[list, str, str | None]:"""Crush an array using statistical analysis and relevance scoring.@@ -2199,17 +2216,34 @@ class SmartCrusher(Transform):Feedback-aware: Uses learned patterns to adjust compression aggressiveness.High retrieval rate for a tool → compress less aggressively.+ Args:+ items: List of dict items to compress.+ query_context: Context string from user messages for relevance scoring.+ tool_name: Name of the tool that produced this output.+ bias: Compression bias multiplier (>1 = keep more, <1 = keep fewer).+Returns:Tuple of (crushed_items, strategy_info, ccr_hash).ccr_hash is the hash for retrieval if CCR is enabled, None otherwise."""- # BOUNDARY CHECK: If already at or below max_items, no compression needed- if len(items) <= self.config.max_items_after_crush:- return items, "none:at_limit", None+ # BOUNDARY CHECK: Use adaptive sizing instead of hardcoded limit+ # compute_optimal_k handles trivial cases (n <= 8 → keep all)+ from .adaptive_sizer import compute_optimal_k++ item_strings = [json.dumps(item, default=str) for item in items]+ adaptive_k = compute_optimal_k(+ item_strings,+ bias=bias,+ min_k=3,+ max_k=self.config.max_items_after_crush if self.config.max_items_after_crush else None,+ )++ if len(items) <= adaptive_k:+ return items, "none:adaptive_at_limit", None# Get feedback hints if enabled# THREAD-SAFETY: Use a local effective_max_items instead of mutating shared config- effective_max_items = self.config.max_items_after_crush+ effective_max_items = adaptive_khints_applied = Falsetoin_hint_applied = False