Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat: provider-aware prefix cache tracking and combined savings dashboard

Tejas Chopra4a655fbc6ee4
diff --git a/headroom/dashboard/templates/dashboard.html b/headroom/dashboard/templates/dashboard.html
index 49101abc4..c4b419d1d 100644
--- a/headroom/dashboard/templates/dashboard.html
+++ b/headroom/dashboard/templates/dashboard.html
@@ -56,14 +56,17 @@
<main class="p-6 max-w-7xl mx-auto">
<!-- Hero Metrics (reordered: Savings $ -> Tokens Saved % -> Quality Confidence -> Overhead) -->
<div class="grid grid-cols-2 md:grid-cols-4 gap-4 mb-6">
- <!-- Savings ($) - Lead with dollars -->
+ <!-- Savings ($) - Combined: compression + prefix cache -->
<div class="bg-surface rounded-lg p-4 border border-border">
- <div class="text-xs text-gray-500 uppercase tracking-wide mb-1">Savings</div>
+ <div class="text-xs text-gray-500 uppercase tracking-wide mb-1">Total Savings</div>
<div class="flex items-baseline gap-2">
<span class="text-3xl font-light tabular-nums text-emerald-400" x-text="'$' + formatCurrency(stats.cost?.savings_usd || 0)"></span>
</div>
<div class="mt-2 text-xs text-gray-500">
- <span x-text="formatNumber(stats.requests?.total || 0)"></span> requests processed
+ <span x-show="stats.cost?.compression_savings_usd > 0 || stats.cost?.cache_savings_usd > 0"
+ x-text="'$' + formatCurrency(stats.cost?.compression_savings_usd || 0) + ' compress + $' + formatCurrency(stats.cost?.cache_savings_usd || 0) + ' cache'"></span>
+ <span x-show="!(stats.cost?.compression_savings_usd > 0 || stats.cost?.cache_savings_usd > 0)"
+ x-text="formatNumber(stats.requests?.total || 0) + ' requests processed'"></span>
</div>
</div>
@@ -127,8 +130,8 @@
<div class="text-2xl font-light tabular-nums text-red-400 line-through decoration-red-400/50" x-text="'$' + formatCurrency(stats.cost?.cost_without_headroom_usd || 0)"></div>
</div>
<div class="ml-auto text-right">
- <div class="text-xs text-gray-500 mb-1">Total saved</div>
- <div class="text-2xl font-light tabular-nums text-emerald-400" x-text="'$' + formatCurrency(stats.cost?.savings_usd || 0)"></div>
+ <div class="text-xs text-gray-500 mb-1">Compression saved</div>
+ <div class="text-2xl font-light tabular-nums text-emerald-400" x-text="'$' + formatCurrency(stats.cost?.compression_savings_usd || 0)"></div>
</div>
</div>
</div>
@@ -136,6 +139,86 @@
</div>
</template>
+ <!-- Prefix Cache Impact (Provider-Aware) -->
+ <template x-if="(stats.prefix_cache?.totals?.requests || 0) > 0">
+ <div class="bg-surface rounded-lg p-4 border border-border mb-6">
+ <div class="flex justify-between items-center mb-3">
+ <div class="text-sm font-medium text-gray-300">Prefix Cache Impact</div>
+ <div class="text-xs text-emerald-400 font-mono" x-text="'Net savings: $' + formatCurrency(stats.prefix_cache?.totals?.net_savings_usd || 0)"></div>
+ </div>
+ <!-- Totals row -->
+ <div class="grid grid-cols-2 md:grid-cols-5 gap-4 mb-4">
+ <div>
+ <div class="text-xs text-gray-500 uppercase tracking-wide mb-1">Cache Reads</div>
+ <div class="text-2xl font-light tabular-nums text-emerald-400" x-text="formatNumber(stats.prefix_cache?.totals?.cache_read_tokens || 0)"></div>
+ <div class="text-xs text-emerald-400/70" x-text="'$' + formatCurrency(stats.prefix_cache?.totals?.savings_usd || 0) + ' saved'"></div>
+ </div>
+ <div>
+ <div class="text-xs text-gray-500 uppercase tracking-wide mb-1">Cache Writes</div>
+ <div class="text-2xl font-light tabular-nums text-amber-400" x-text="formatNumber(stats.prefix_cache?.totals?.cache_write_tokens || 0)"></div>
+ <div class="text-xs text-amber-400/70" x-text="'$' + formatCurrency(stats.prefix_cache?.totals?.bust_penalty_usd || 0) + ' penalty'"></div>
+ </div>
+ <div>
+ <div class="text-xs text-gray-500 uppercase tracking-wide mb-1">Hit Rate</div>
+ <div class="text-2xl font-light tabular-nums" :class="(stats.prefix_cache?.totals?.hit_rate || 0) > 80 ? 'text-emerald-400' : (stats.prefix_cache?.totals?.hit_rate || 0) > 50 ? 'text-amber-400' : 'text-red-400'" x-text="(stats.prefix_cache?.totals?.hit_rate || 0).toFixed(0) + '%'"></div>
+ <div class="text-xs text-gray-500" x-text="(stats.prefix_cache?.totals?.hit_requests || 0) + ' / ' + (stats.prefix_cache?.totals?.requests || 0) + ' requests'"></div>
+ </div>
+ <div>
+ <div class="text-xs text-gray-500 uppercase tracking-wide mb-1">Cache Busts</div>
+ <div class="text-2xl font-light tabular-nums" :class="(stats.prefix_cache?.totals?.bust_count || 0) > 5 ? 'text-red-400' : (stats.prefix_cache?.totals?.bust_count || 0) > 0 ? 'text-amber-400' : 'text-emerald-400'" x-text="stats.prefix_cache?.totals?.bust_count || 0"></div>
+ <div class="text-xs text-gray-500" x-text="formatNumber(stats.prefix_cache?.totals?.bust_write_tokens || 0) + ' tokens re-written'"></div>
+ </div>
+ <div>
+ <div class="text-xs text-gray-500 uppercase tracking-wide mb-1">Providers</div>
+ <div class="text-2xl font-light tabular-nums text-gray-300" x-text="Object.keys(stats.prefix_cache?.by_provider || {}).length"></div>
+ <div class="text-xs text-gray-500">with cache data</div>
+ </div>
+ </div>
+ <!-- Cache efficiency bar -->
+ <div>
+ <div class="flex justify-between text-xs text-gray-500 mb-1">
+ <span>Cache Efficiency</span>
+ <span x-text="cacheSavingsPercent + '% of input served from cache'"></span>
+ </div>
+ <div class="w-full h-3 bg-border rounded-full overflow-hidden flex">
+ <div class="h-full bg-emerald-500 transition-all duration-500"
+ :style="'width: ' + cacheSavingsPercent + '%'"></div>
+ <div class="h-full bg-amber-500 transition-all duration-500"
+ :style="'width: ' + cacheWritePercent + '%'"></div>
+ </div>
+ <div class="flex gap-4 mt-1 text-xs text-gray-500">
+ <span class="flex items-center gap-1"><span class="w-2 h-2 rounded-full bg-emerald-500 inline-block"></span> Reads (discounted)</span>
+ <span class="flex items-center gap-1"><span class="w-2 h-2 rounded-full bg-amber-500 inline-block"></span> Writes</span>
+ <span class="flex items-center gap-1"><span class="w-2 h-2 rounded-full bg-border inline-block"></span> Uncached</span>
+ </div>
+ </div>
+ <!-- Per-provider breakdown -->
+ <template x-if="Object.keys(stats.prefix_cache?.by_provider || {}).length > 0">
+ <div class="mt-4 border-t border-border pt-3">
+ <div class="text-xs text-gray-500 uppercase tracking-wide mb-2">Per-Provider Breakdown</div>
+ <div class="space-y-3">
+ <template x-for="[prov, pc] in Object.entries(stats.prefix_cache?.by_provider || {})" :key="prov">
+ <div class="flex items-center justify-between">
+ <div class="flex items-center gap-3">
+ <span class="px-2 py-0.5 bg-border rounded text-xs font-mono" x-text="prov"></span>
+ <span class="text-xs text-gray-500" x-text="pc.label"></span>
+ </div>
+ <div class="flex items-center gap-4 text-xs font-mono">
+ <span class="text-emerald-400" x-text="formatNumber(pc.cache_read_tokens) + ' reads (' + pc.read_discount + ' off)'"></span>
+ <template x-if="pc.write_premium !== 'none'">
+ <span class="text-amber-400" x-text="formatNumber(pc.cache_write_tokens) + ' writes (+' + pc.write_premium + ')'"></span>
+ </template>
+ <span :class="pc.bust_count > 0 ? 'text-red-400' : 'text-gray-500'" x-text="pc.bust_count + ' busts'"></span>
+ <span class="text-emerald-400" x-text="'$' + formatCurrency(pc.net_savings_usd)"></span>
+ </div>
+ </div>
+ </template>
+ </div>
+ </div>
+ </template>
+ </div>
+ </template>
+
<!-- New: Waste Signal Breakdown + Cumulative Savings Trend -->
<div class="grid grid-cols-1 lg:grid-cols-2 gap-4 mb-6">
<!-- What Headroom Removed -->
@@ -523,6 +606,22 @@
.substring(0, 20);
},
+ // --- Prefix Cache ---
+
+ get cacheSavingsPercent() {
+ const t = this.stats.prefix_cache?.totals || {};
+ const total = (t.cache_read_tokens || 0) + (t.cache_write_tokens || 0);
+ if (total === 0) return 0;
+ return Math.round((t.cache_read_tokens || 0) / total * 100);
+ },
+
+ get cacheWritePercent() {
+ const t = this.stats.prefix_cache?.totals || {};
+ const total = (t.cache_read_tokens || 0) + (t.cache_write_tokens || 0);
+ if (total === 0) return 0;
+ return Math.round((t.cache_write_tokens || 0) / total * 100);
+ },
+
// --- Waste Signals ---
wasteSignalLabel(signal) {
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index 3b044194e..9c2379c79 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -191,6 +191,164 @@ def _summarize_transforms(transforms: list[str]) -> str:
return " ".join(parts)
+# Provider-specific cache discount multipliers (what fraction of input price)
+# Used to calculate dollar savings from prefix caching
+_CACHE_ECONOMICS = {
+ "anthropic": {
+ "read_multiplier": 0.1,
+ "write_multiplier": 1.25,
+ "label": "Explicit breakpoints, 5-min TTL",
+ },
+ "openai": {
+ "read_multiplier": 0.5,
+ "write_multiplier": 1.0,
+ "label": "Automatic, no TTL control",
+ },
+ "gemini": {
+ "read_multiplier": 0.1,
+ "write_multiplier": 1.0,
+ "label": "Explicit cachedContent, configurable TTL",
+ },
+ "bedrock": {
+ "read_multiplier": 0.1,
+ "write_multiplier": 1.25,
+ "label": "Same as Anthropic (Bedrock)",
+ },
+}
+
+
+def _build_prefix_cache_stats(
+ metrics: PrometheusMetrics,
+ cost_tracker: CostTracker | None,
+) -> dict:
+ """Build provider-aware prefix cache statistics for the dashboard."""
+ by_provider = {}
+ totals = {
+ "cache_read_tokens": 0,
+ "cache_write_tokens": 0,
+ "requests": 0,
+ "hit_requests": 0,
+ "bust_count": 0,
+ "bust_write_tokens": 0,
+ "savings_usd": 0.0,
+ "bust_penalty_usd": 0.0,
+ }
+
+ for provider, pc in metrics.cache_by_provider.items():
+ if pc["requests"] == 0:
+ continue
+
+ econ = _CACHE_ECONOMICS.get(provider, _CACHE_ECONOMICS["anthropic"])
+ read_mult: float = econ["read_multiplier"] # type: ignore[assignment]
+ write_mult: float = econ["write_multiplier"] # type: ignore[assignment]
+
+ # Get the base input price per token for the most-used model on this provider
+ input_price_per_token = None
+ if cost_tracker:
+ for model_name in cost_tracker._tokens_sent_by_model:
+ # Match model to provider
+ _openai_prefixes = ("gpt", "o1", "o3", "o4")
+ is_match = (
+ (provider == "anthropic" and "claude" in model_name)
+ or (provider == "openai" and any(
+ p in model_name for p in _openai_prefixes
+ ))
+ or (provider == "gemini" and "gemini" in model_name)
+ or (provider == "bedrock" and "claude" in model_name)
+ )
+ if is_match:
+ price_per_1m = cost_tracker._get_list_price(model_name)
+ if price_per_1m:
+ input_price_per_token = price_per_1m / 1_000_000
+ break
+
+ # Calculate savings:
+ # Without cache: all tokens at 1.0x input price
+ # With cache: read tokens at read_mult, write tokens at write_mult
+ read_tokens: int = pc["cache_read_tokens"] # type: ignore[assignment]
+ write_tokens: int = pc["cache_write_tokens"] # type: ignore[assignment]
+ savings_usd = 0.0
+ bust_penalty_usd = 0.0
+
+ if input_price_per_token:
+ # Savings from reads: tokens * price * (1.0 - read_multiplier)
+ savings_usd = read_tokens * input_price_per_token * (1.0 - read_mult)
+ # Penalty from writes: tokens * price * (write_multiplier - 1.0)
+ if write_mult > 1.0:
+ bust_penalty_usd = write_tokens * input_price_per_token * (write_mult - 1.0)
+
+ hit_rate = round(pc["hit_requests"] / pc["requests"] * 100, 1) if pc["requests"] > 0 else 0
+
+ provider_stats = {
+ "cache_read_tokens": read_tokens,
+ "cache_write_tokens": write_tokens,
+ "requests": pc["requests"],
+ "hit_requests": pc["hit_requests"],
+ "hit_rate": hit_rate,
+ "bust_count": pc["bust_count"],
+ "bust_write_tokens": pc["bust_write_tokens"],
+ "read_discount": f"{(1.0 - read_mult) * 100:.0f}%",
+ "write_premium": f"{(write_mult - 1.0) * 100:.0f}%" if write_mult > 1.0 else "none",
+ "savings_usd": round(savings_usd, 4),
+ "bust_penalty_usd": round(bust_penalty_usd, 4),
+ "net_savings_usd": round(savings_usd - bust_penalty_usd, 4),
+ "label": str(econ["label"]),
+ }
+ by_provider[provider] = provider_stats
+
+ # Accumulate totals
+ totals["cache_read_tokens"] += read_tokens
+ totals["cache_write_tokens"] += write_tokens
+ totals["requests"] += pc["requests"]
+ totals["hit_requests"] += pc["hit_requests"]
+ totals["bust_count"] += pc["bust_count"]
+ totals["bust_write_tokens"] += pc["bust_write_tokens"]
+ totals["savings_usd"] += savings_usd
+ totals["bust_penalty_usd"] += bust_penalty_usd
+
+ totals["net_savings_usd"] = round(totals["savings_usd"] - totals["bust_penalty_usd"], 4)
+ totals["savings_usd"] = round(totals["savings_usd"], 4)
+ totals["bust_penalty_usd"] = round(totals["bust_penalty_usd"], 4)
+ totals["hit_rate"] = (
+ round(totals["hit_requests"] / totals["requests"] * 100, 1) if totals["requests"] > 0 else 0
+ )
+
+ return {
+ "by_provider": by_provider,
+ "totals": totals,
+ }
+
+
+def _merge_cost_stats(
+ cost_stats: dict | None,
+ cache_stats: dict,
+) -> dict | None:
+ """Add prefix cache savings to overall cost stats.
+
+ Compression savings and cache savings are computed on different token
+ scopes (Headroom tracks user-message tokens; Anthropic's cache metrics
+ cover the entire prompt including system/tools). We keep both as
+ additive line items without cross-contaminating the counterfactual.
+
+ - compression_savings_usd: from removing tokens (CostTracker scope)
+ - cache_savings_usd: from prefix cache discounts (full-prompt scope)
+ - savings_usd: sum of both (hero metric)
+ - cost_with/without_headroom_usd: unchanged (compression scope only)
+ """
+ if cost_stats is None:
+ return None
+
+ cache_net = cache_stats.get("totals", {}).get("net_savings_usd", 0.0)
+ compression_savings = cost_stats.get("savings_usd", 0.0)
+
+ return {
+ **cost_stats,
+ "savings_usd": round(compression_savings + cache_net, 4),
+ "compression_savings_usd": round(compression_savings, 4),
+ "cache_savings_usd": round(cache_net, 4),
+ }
+
+
# Maximum request body size (100MB - increased to support image-heavy requests)
MAX_REQUEST_BODY_SIZE = 100 * 1024 * 1024
@@ -849,6 +1007,25 @@ class PrometheusMetrics:
# Aggregate waste signals
self.waste_signals_total: dict[str, int] = defaultdict(int)
+ # Provider-specific prefix cache tracking
+ # Each provider has different cache economics:
+ # Anthropic: cache_read=0.1x, cache_write=1.25x, explicit breakpoints
+ # OpenAI: cache_read=0.5x, no write penalty, automatic
+ # Google: cache_read=~0.1x, explicit cachedContent API, storage cost
+ # Bedrock: no cache metrics
+ self.cache_by_provider: dict[str, dict[str, int | float]] = defaultdict(
+ lambda: {
+ "cache_read_tokens": 0,
+ "cache_write_tokens": 0,
+ "requests": 0,
+ "hit_requests": 0, # requests with cache_read > 0
+ "bust_count": 0,
+ "bust_write_tokens": 0,
+ }
+ )
+ # Track per-model cache request count to distinguish cold starts from busts
+ self._cache_requests_by_model: dict[str, int] = defaultdict(int)
+
# Cumulative savings history (timestamp → cumulative tokens saved)
self.savings_history: list[tuple[str, int]] = []
@@ -867,6 +1044,8 @@ class PrometheusMetrics:
ttfb_ms: float = 0,
pipeline_timing: dict[str, float] | None = None,
waste_signals: dict[str, int] | None = None,
+ cache_read_tokens: int = 0,
+ cache_write_tokens: int = 0,
):
"""Record metrics for a request."""
async with self._lock:
@@ -881,6 +1060,26 @@ class PrometheusMetrics:
self.tokens_output_total += output_tokens
self.tokens_saved_total += tokens_saved
+ # Track provider-specific prefix cache metrics
+ if cache_read_tokens > 0 or cache_write_tokens > 0:
+ pc = self.cache_by_provider[provider]
+ pc["cache_read_tokens"] += cache_read_tokens
+ pc["cache_write_tokens"] += cache_write_tokens
+ pc["requests"] += 1
+ if cache_read_tokens > 0:
+ pc["hit_requests"] += 1
+ # Model-aware bust detection: the first request for any model
+ # is always a cold start (100% write, 0% read) — not a bust.
+ # Only flag as bust when a previously-warm model suddenly has
+ # high write ratio, indicating prefix invalidation.
+ model_req_num = self._cache_requests_by_model[model]
+ self._cache_requests_by_model[model] += 1
+ if provider == "anthropic" and model_req_num > 0:
+ total_cached = cache_read_tokens + cache_write_tokens
+ if total_cached > 0 and cache_write_tokens > total_cached * 0.5:
+ pc["bust_count"] += 1
+ pc["bust_write_tokens"] += cache_write_tokens
+
self.latency_sum_ms += latency_ms
self.latency_min_ms = min(self.latency_min_ms, latency_ms)
self.latency_max_ms = max(self.latency_max_ms, latency_ms)
@@ -1380,7 +1579,6 @@ class HeadroomProxy:
preserve_imports=True,
preserve_signatures=True,
preserve_type_annotations=True,
- preserve_error_handlers=True,
)
# Insert before RollingWindow (which should be last)
transforms.insert(-1, CodeAwareCompressor(code_config))
@@ -2221,11 +2419,15 @@ class HeadroomProxy:
total_latency = (time.time() - start_time) * 1000
- # Parse response for output token count
+ # Parse response for output token count and cache metrics
output_tokens = 0
+ cr_tokens = 0
+ cw_tokens = 0
if resp_json:
usage = resp_json.get("usage", {})
output_tokens = usage.get("output_tokens", 0)
+ cr_tokens = usage.get("cache_read_input_tokens", 0)
+ cw_tokens = usage.get("cache_creation_input_tokens", 0)
if self.cost_tracker:
self.cost_tracker.record_tokens(model, tokens_saved, optimized_tokens)
@@ -2252,6 +2454,8 @@ class HeadroomProxy:
overhead_ms=optimization_latency,
pipeline_timing=pipeline_timing,
waste_signals=waste_signals_dict,
+ cache_read_tokens=cr_tokens,
+ cache_write_tokens=cw_tokens,
)
# Log request
@@ -2447,7 +2651,8 @@ class HeadroomProxy:
)
optimized_messages = result.messages
- pipeline_timing = result.timing
+ for k, v in result.timing.items():
+ pipeline_timing[k] = pipeline_timing.get(k, 0.0) + v
# Use pipeline's token counts for consistency with pipeline logs
original_tokens = result.tokens_before
optimized_tokens = result.tokens_after
@@ -2898,7 +3103,8 @@ class HeadroomProxy:
)
optimized_messages = result.messages
- pipeline_timing = result.timing
+ for k, v in result.timing.items():
+ pipeline_timing[k] = pipeline_timing.get(k, 0.0) + v
# Use pipeline's token counts for consistency with pipeline logs
original_tokens = result.tokens_before
optimized_tokens = result.tokens_after
@@ -3948,6 +4154,8 @@ class HeadroomProxy:
overhead_ms=optimization_latency,
ttfb_ms=stream_state["ttfb_ms"] or 0,
pipeline_timing=pipeline_timing,
+ cache_read_tokens=cache_read_tokens,
+ cache_write_tokens=cache_write_tokens,
)
return StreamingResponse(
@@ -4399,6 +4607,7 @@ class HeadroomProxy:
overhead_ms=optimization_latency,
pipeline_timing=pipeline_timing,
waste_signals=waste_signals_dict,
+ cache_read_tokens=cache_read_tokens,
)
if tokens_saved > 0:
@@ -5424,6 +5633,7 @@ class HeadroomProxy:
latency_ms=total_latency,
overhead_ms=optimization_latency,
waste_signals=waste_signals_dict,
+ cache_read_tokens=cache_read_tokens,
)
if tokens_saved > 0:
@@ -5873,6 +6083,9 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
feedback = get_compression_feedback()
feedback_stats = feedback.get_stats()
+ # Build prefix cache stats once (used in both prefix_cache and cost)
+ prefix_cache_stats = _build_prefix_cache_stats(m, proxy.cost_tracker)
+
# Calculate total tokens before compression
total_tokens_before = m.tokens_input_total + m.tokens_saved_total
@@ -5927,7 +6140,11 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
else {},
"waste_signals": dict(m.waste_signals_total) if m.waste_signals_total else {},
"savings_history": m.savings_history[-100:], # Last 100 data points
- "cost": proxy.cost_tracker.stats() if proxy.cost_tracker else None,
+ "prefix_cache": prefix_cache_stats,
+ "cost": _merge_cost_stats(
+ proxy.cost_tracker.stats() if proxy.cost_tracker else None,
+ prefix_cache_stats,
+ ),
"compression": {
"ccr_entries": compression_stats.get("entry_count", 0),
"ccr_max_entries": compression_stats.get("max_entries", 0),
diff --git a/headroom/transforms/code_compressor.py b/headroom/transforms/code_compressor.py
index ef2122a3a..5ce81af75 100644
--- a/headroom/transforms/code_compressor.py
+++ b/headroom/transforms/code_compressor.py
@@ -9,6 +9,8 @@ Key Features:
- Preserves imports, signatures, type annotations, error handlers
- Compresses function bodies while maintaining structure
- Multi-language support via tree-sitter
+- Data-driven language config (no per-language method duplication)
+- Thread-safe (no mutable instance state during compression)
Supported Languages (Tier 1):
- Python, JavaScript, TypeScript
@@ -19,7 +21,7 @@ Supported Languages (Tier 2):
Compression Strategy:
1. Parse code into AST using tree-sitter
2. Extract and preserve critical structures (imports, signatures, types)
-3. Rank functions by importance (using perplexity or heuristics)
+3. Rank functions by importance (using semantic analysis)
4. Compress function bodies while preserving signatures
5. Reassemble into valid code
@@ -170,6 +172,131 @@ class DocstringMode(Enum):
NONE = "none" # Alias for REMOVE (deprecated)
+# =========================================================================
+# Data-driven language configuration
+# =========================================================================
+
+
+@dataclass(frozen=True)
+class LangConfig:
+ """Data-driven configuration for a programming language.
+
+ Instead of per-language methods, each language declares its AST node
+ types and syntactic conventions. The compressor uses these tables to
+ drive extraction and compression generically.
+ """
+
+ # AST node types for structural extraction
+ import_nodes: frozenset[str]
+ function_nodes: frozenset[str]
+ class_nodes: frozenset[str]
+ type_nodes: frozenset[str]
+ body_node_types: frozenset[str] # Node types that represent function/method bodies
+ decorator_node: str | None # e.g. "decorated_definition" for Python
+
+ # Syntax conventions
+ comment_prefix: str # "#" for Python, "//" for C-family
+ uses_colon_after_signature: bool # Python: True, C-family: False
+ package_node: str | None = None # e.g. "package_clause" for Go
+
+ # Quick pre-filter hints for language detection (substrings to check)
+ detection_hints: tuple[str, ...] = ()
+
+
+_LANG_CONFIGS: dict[CodeLanguage, LangConfig] = {
+ CodeLanguage.PYTHON: LangConfig(
+ import_nodes=frozenset({"import_statement", "import_from_statement"}),
+ function_nodes=frozenset({"function_definition"}),
+ class_nodes=frozenset({"class_definition"}),
+ type_nodes=frozenset({"type_alias_statement"}),
+ body_node_types=frozenset({"block"}),
+ decorator_node="decorated_definition",
+ comment_prefix="#",
+ uses_colon_after_signature=True,
+ detection_hints=("def ", "import ", "from ", "class ", "async def"),
+ ),
+ CodeLanguage.JAVASCRIPT: LangConfig(
+ import_nodes=frozenset({"import_statement", "import_declaration"}),
+ function_nodes=frozenset({"function_declaration", "method_definition"}),
+ class_nodes=frozenset({"class_declaration"}),
+ type_nodes=frozenset(),
+ body_node_types=frozenset({"statement_block"}),
+ decorator_node=None,
+ comment_prefix="//",
+ uses_colon_after_signature=False,
+ detection_hints=("function ", "const ", "let ", "var ", "export ", "require("),
+ ),
+ CodeLanguage.TYPESCRIPT: LangConfig(
+ import_nodes=frozenset({"import_statement", "import_declaration"}),
+ function_nodes=frozenset({"function_declaration", "method_definition"}),
+ class_nodes=frozenset({"class_declaration"}),
+ type_nodes=frozenset({"interface_declaration", "type_alias_declaration"}),
+ body_node_types=frozenset({"statement_block"}),
+ decorator_node=None,
+ comment_prefix="//",
+ uses_colon_after_signature=False,
+ detection_hints=("interface ", "type ", ": string", ": number", ": boolean"),
+ ),
+ CodeLanguage.GO: LangConfig(
+ import_nodes=frozenset({"import_declaration"}),
+ function_nodes=frozenset({"function_declaration", "method_declaration"}),
+ class_nodes=frozenset(),
+ type_nodes=frozenset({"type_declaration"}),
+ body_node_types=frozenset({"block"}),
+ decorator_node=None,
+ comment_prefix="//",
+ uses_colon_after_signature=False,
+ package_node="package_clause",
+ detection_hints=("func ", "package ", "struct {"),
+ ),
+ CodeLanguage.RUST: LangConfig(
+ import_nodes=frozenset({"use_declaration"}),
+ function_nodes=frozenset({"function_item"}),
+ class_nodes=frozenset({"impl_item"}),
+ type_nodes=frozenset({"struct_item", "enum_item", "type_item", "trait_item"}),
+ body_node_types=frozenset({"block"}),
+ decorator_node=None,
+ comment_prefix="//",
+ uses_colon_after_signature=False,
+ detection_hints=("fn ", "struct ", "impl ", "mod ", "use "),
+ ),
+ CodeLanguage.JAVA: LangConfig(
+ import_nodes=frozenset({"import_declaration"}),
+ function_nodes=frozenset({"method_declaration", "constructor_declaration"}),
+ class_nodes=frozenset({"class_declaration", "interface_declaration"}),
+ type_nodes=frozenset({"enum_declaration"}),
+ body_node_types=frozenset({"block"}),
+ decorator_node=None,
+ comment_prefix="//",
+ uses_colon_after_signature=False,
+ package_node="package_declaration",
+ detection_hints=("public ", "private ", "protected ", "class ", "interface "),
+ ),
+ CodeLanguage.C: LangConfig(
+ import_nodes=frozenset({"preproc_include"}),
+ function_nodes=frozenset({"function_definition"}),
+ class_nodes=frozenset(),
+ type_nodes=frozenset({"struct_specifier", "enum_specifier", "type_definition"}),
+ body_node_types=frozenset({"compound_statement"}),
+ decorator_node=None,
+ comment_prefix="//",
+ uses_colon_after_signature=False,
+ detection_hints=("#include", "typedef ", "int main("),
+ ),
+ CodeLanguage.CPP: LangConfig(
+ import_nodes=frozenset({"preproc_include"}),
+ function_nodes=frozenset({"function_definition"}),
+ class_nodes=frozenset({"class_specifier"}),
+ type_nodes=frozenset({"struct_specifier", "enum_specifier", "type_definition"}),
+ body_node_types=frozenset({"compound_statement"}),
+ decorator_node=None,
+ comment_prefix="//",
+ uses_colon_after_signature=False,
+ detection_hints=("#include", "namespace ", "class ", "::"),
+ ),
+}
+
+
@dataclass
class CodeStructure:
"""Extracted structure from parsed code."""
@@ -182,8 +309,8 @@ class CodeStructure:
default_factory=list
) # (signature, body, line)
decorators: list[str] = field(default_factory=list)
- error_handlers: list[str] = field(default_factory=list)
comments: list[str] = field(default_factory=list)
+ top_level_code: list[str] = field(default_factory=list)
other: list[str] = field(default_factory=list)
@@ -195,7 +322,6 @@ class CodeCompressorConfig:
preserve_imports: Always keep import statements.
preserve_signatures: Always keep function/method signatures.
preserve_type_annotations: Keep type hints and annotations.
- preserve_error_handlers: Keep try/except/finally blocks.
preserve_decorators: Keep decorators on functions/classes.
docstring_mode: How to handle docstrings.
target_compression_rate: Target compression ratio (0.2 = keep 20%).
@@ -212,7 +338,6 @@ class CodeCompressorConfig:
preserve_imports: bool = True
preserve_signatures: bool = True
preserve_type_annotations: bool = True
- preserve_error_handlers: bool = True
preserve_decorators: bool = True
docstring_mode: DocstringMode = DocstringMode.FIRST_LINE
@@ -311,12 +436,18 @@ class CodeCompressionResult:
)
-# Language detection patterns
-_LANGUAGE_PATTERNS: dict[CodeLanguage, list[re.Pattern[str]]] = {
+# =========================================================================
+# Language detection
+# =========================================================================
+
+# Lightweight pre-filter patterns for language detection.
+# These are ONLY used as a quick check to avoid parsing with every language.
+# Actual detection is done by tree-sitter (fewest parse errors wins).
+_LANGUAGE_PREFILTER: dict[CodeLanguage, list[re.Pattern[str]]] = {
CodeLanguage.PYTHON: [
re.compile(r"^\s*(def|class|import|from|async def)\s+\w+", re.MULTILINE),
- re.compile(r"^\s*@\w+", re.MULTILINE), # Decorators
- re.compile(r'^\s*"""', re.MULTILINE), # Docstrings
+ re.compile(r"^\s*@\w+", re.MULTILINE),
+ re.compile(r'^\s*"""', re.MULTILINE),
re.compile(r"^\s*if __name__\s*==", re.MULTILINE),
],
CodeLanguage.JAVASCRIPT: [
@@ -328,21 +459,18 @@ _LANGUAGE_PATTERNS: dict[CodeLanguage, list[re.Pattern[str]]] = {
CodeLanguage.TYPESCRIPT: [
re.compile(r"^\s*(interface|type|enum|namespace)\s+\w+", re.MULTILINE),
re.compile(r":\s*(string|number|boolean|any|void|Promise)\b", re.MULTILINE),
- re.compile(r"<\w+>", re.MULTILINE), # Generic types
],
CodeLanguage.GO: [
re.compile(r"^\s*(func|type|package|import)\s+", re.MULTILINE),
- re.compile(r"^\s*func\s+\([^)]+\)\s+\w+", re.MULTILINE), # Methods
+ re.compile(r"^\s*func\s+\([^)]+\)\s+\w+", re.MULTILINE),
re.compile(r"\bstruct\s*\{", re.MULTILINE),
],
CodeLanguage.RUST: [
re.compile(r"^\s*(fn|struct|enum|impl|mod|use|pub)\s+", re.MULTILINE),
- re.compile(r"^\s*#\[", re.MULTILINE), # Attributes
- re.compile(r"->\s*\w+", re.MULTILINE), # Return types
+ re.compile(r"^\s*#\[", re.MULTILINE),
],
CodeLanguage.JAVA: [
re.compile(r"^\s*(public|private|protected)\s+(class|interface|enum)", re.MULTILINE),
- re.compile(r"^\s*@\w+", re.MULTILINE), # Annotations
re.compile(r"^\s*package\s+[\w.]+;", re.MULTILINE),
],
CodeLanguage.C: [
@@ -352,58 +480,115 @@ _LANGUAGE_PATTERNS: dict[CodeLanguage, list[re.Pattern[str]]] = {
],
CodeLanguage.CPP: [
re.compile(r"^\s*#include\s*[<\"]", re.MULTILINE),
- re.compile(r"\bclass\s+\w+\s*[:{]", re.MULTILINE),
re.compile(r"\bnamespace\s+\w+", re.MULTILINE),
- re.compile(r"::\w+", re.MULTILINE), # Scope resolution
+ re.compile(r"::\w+", re.MULTILINE),
],
}
+def _count_error_nodes(node: Any) -> int:
+ """Count ERROR and MISSING nodes in a tree-sitter AST."""
+ count = 0
+ if node.type == "ERROR" or node.is_missing:
+ count += 1
+ for child in node.children:
+ count += _count_error_nodes(child)
+ return count
+
+
def detect_language(code: str) -> tuple[CodeLanguage, float]:
"""Detect the programming language of code.
+ Uses tree-sitter AST parsing when available (most accurate), with a
+ regex pre-filter to avoid parsing with all languages. Falls back to
+ regex-only scoring when tree-sitter is unavailable.
+
Args:
code: Source code to analyze.
Returns:
- Tuple of (detected language, confidence score).
+ Tuple of (detected language, confidence score 0.0-1.0).
"""
if not code or not code.strip():
return CodeLanguage.UNKNOWN, 0.0
- scores: dict[CodeLanguage, int] = {}
- sample = code[:5000] # Analyze first 5000 chars
+ sample = code[:5000]
- for lang, patterns in _LANGUAGE_PATTERNS.items():
+ # Phase 1: Pre-filter — find candidate languages using quick regex
+ candidates: dict[CodeLanguage, int] = {}
+ for lang, patterns in _LANGUAGE_PREFILTER.items():
score = 0
for pattern in patterns:
matches = len(pattern.findall(sample))
score += matches
if score > 0:
- scores[lang] = score
+ candidates[lang] = score
- if not scores:
+ if not candidates:
return CodeLanguage.UNKNOWN, 0.0
- # TypeScript is a superset of JavaScript - prefer TS if TS patterns found
- if CodeLanguage.TYPESCRIPT in scores and CodeLanguage.JAVASCRIPT in scores:
- if scores[CodeLanguage.TYPESCRIPT] >= 2:
- scores[CodeLanguage.JAVASCRIPT] = 0
+ # Disambiguation: TypeScript superset of JavaScript
+ if CodeLanguage.TYPESCRIPT in candidates and CodeLanguage.JAVASCRIPT in candidates:
+ if candidates[CodeLanguage.TYPESCRIPT] >= 2:
+ candidates[CodeLanguage.JAVASCRIPT] = 0
- # C++ is a superset of C - prefer C++ if C++ patterns found
- if CodeLanguage.CPP in scores and CodeLanguage.C in scores:
- if scores[CodeLanguage.CPP] >= 2:
- scores[CodeLanguage.C] = 0
+ # Disambiguation: C++ superset of C
+ if CodeLanguage.CPP in candidates and CodeLanguage.C in candidates:
+ if candidates[CodeLanguage.CPP] >= 2:
+ candidates[CodeLanguage.C] = 0
- best_lang = max(scores, key=lambda k: scores[k])
- best_score = scores[best_lang]
+ # Phase 2: If tree-sitter available, parse with candidates and pick fewest errors
+ if _check_tree_sitter_available():
+ best_lang = CodeLanguage.UNKNOWN
+ min_errors = float("inf")
+ best_node_count = 0
+ code_bytes = bytes(code[:10000], "utf-8")
- # Calculate confidence (higher score = higher confidence)
- confidence = min(1.0, 0.3 + (best_score * 0.1))
+ # Sort candidates by pre-filter score (try most likely first)
+ sorted_candidates = sorted(candidates.items(), key=lambda x: x[1], reverse=True)
+
+ for lang, _prefilter_score in sorted_candidates:
+ if lang == CodeLanguage.UNKNOWN or candidates.get(lang, 0) == 0:
+ continue
+ try:
+ parser = _get_parser(lang.value)
+ tree = parser.parse(code_bytes)
+ error_count = _count_error_nodes(tree.root_node)
+ node_count = tree.root_node.child_count
+
+ # Prefer: fewest errors, then most top-level nodes (richer parse)
+ if error_count < min_errors or (
+ error_count == min_errors and node_count > best_node_count
+ ):
+ min_errors = error_count
+ best_lang = lang
+ best_node_count = node_count
+ except (ValueError, ImportError):
+ continue
+
+ if best_lang != CodeLanguage.UNKNOWN:
+ # Confidence based on error ratio
+ total_lines = max(1, len(code.strip().split("\n")))
+ error_ratio = min_errors / total_lines
+ confidence = max(0.3, min(1.0, 1.0 - error_ratio))
+ return best_lang, confidence
+
+ # Phase 3: Fallback — regex-only scoring (no tree-sitter)
+ best_lang = max(candidates, key=lambda k: candidates[k])
+ best_score = candidates[best_lang]
+
+ if best_score == 0:
+ return CodeLanguage.UNKNOWN, 0.0
+ confidence = min(1.0, 0.3 + (best_score * 0.1))
return best_lang, confidence
+# =========================================================================
+# Symbol importance analysis
+# =========================================================================
+
+
@dataclass
class _SymbolAnalysis:
"""Result of intra-file symbol importance analysis.
@@ -428,10 +613,11 @@ class CodeAwareCompressor(Transform):
Key advantages over token-level compression:
- Syntax validity guaranteed
- - Preserves imports, signatures, types, error handlers
+ - Preserves imports, signatures, types
- Better compression ratios for code (5-8x vs 3-5x)
- Lower latency (~20-50ms vs 50-200ms for LLMLingua)
- Smaller memory footprint (~50MB vs ~1GB)
+ - Thread-safe (no mutable instance state during compression)
Example:
>>> compressor = CodeAwareCompressor()
@@ -474,8 +660,23 @@ class CodeAwareCompressor(Transform):
startup overhead when the compressor isn't used.
"""
self.config = config or CodeCompressorConfig()
- self._symbol_analysis: _SymbolAnalysis | None = None
- self._body_limits: dict[str, int] | None = None
+
+ # =========================================================================
+ # Token estimation
+ # =========================================================================
+
+ @staticmethod
+ def _estimate_tokens(text: str, tokenizer: Tokenizer | None = None) -> int:
+ """Count or estimate tokens for text.
+
+ Uses real tokenizer when available; falls back to chars/4 which is
+ a much closer approximation for code than word count.
+ """
+ if tokenizer is not None:
+ return tokenizer.count_text(text)
+ # chars/4 is a reasonable approximation for code tokens
+ # (code has lots of punctuation that tokenizes separately)
+ return max(1, len(text) // 4)
# =========================================================================
# Symbol importance analysis
@@ -500,35 +701,40 @@ class CodeAwareCompressor(Transform):
if not self.config.semantic_analysis:
return _SymbolAnalysis()
- definition_types = {
- "function_definition", # Python
- "class_definition", # Python
- "function_declaration", # JS, Go
- "class_declaration", # JS, Java
- "method_definition", # JS
- "method_declaration", # Go, Java
- "function_item", # Rust
- }
+ lang_config = _LANG_CONFIGS.get(language)
+ if not lang_config:
+ return _SymbolAnalysis()
+
+ all_definition_types = lang_config.function_nodes | lang_config.class_nodes
# Use qualified keys (ClassName.method) to avoid collisions
- # between methods with the same name in different classes.
- # bare_names maps qualified keys back to short names for display/matching.
definitions: dict[str, Any] = {} # qualified_name -> node
bare_names: dict[str, str] = {} # qualified_name -> short_name
all_identifiers: dict[str, int] = {} # short_name -> count
function_calls: dict[str, set[str]] = {}
def collect_definitions(node: Any, parent_name: str = "") -> None:
- if node.type in definition_types:
- short_name = self._get_definition_name(node)
+ if node.type in all_definition_types:
+ short_name = _get_definition_name(node)
if short_name:
qualified = f"{parent_name}.{short_name}" if parent_name else short_name
definitions[qualified] = node
bare_names[qualified] = short_name
- # Recurse into this definition to find nested defs
for child in node.children:
collect_definitions(child, parent_name=qualified)
return
+ # Also check for decorated definitions
+ if lang_config.decorator_node and node.type == lang_config.decorator_node:
+ for child in node.children:
+ if child.type in all_definition_types:
+ short_name = _get_definition_name(child)
+ if short_name:
+ qualified = f"{parent_name}.{short_name}" if parent_name else short_name
+ definitions[qualified] = child
+ bare_names[qualified] = short_name
+ for grandchild in child.children:
+ collect_definitions(grandchild, parent_name=qualified)
+ return
for child in node.children:
collect_definitions(child, parent_name)
@@ -542,7 +748,6 @@ class CodeAwareCompressor(Transform):
def collect_calls_in_function(func_node: Any, func_qname: str) -> None:
func_short = bare_names[func_qname]
- # Collect short names of other defined symbols this function references
defined_short_names = set(bare_names.values())
calls: set[str] = set()
@@ -564,7 +769,7 @@ class CodeAwareCompressor(Transform):
if not definitions:
return _SymbolAnalysis()
- # Pass 2: Collect all identifiers (by short name, since code uses short names)
+ # Pass 2: Collect all identifiers
collect_identifiers(root)
# Pass 3: Collect call relationships and body sizes
@@ -574,8 +779,7 @@ class CodeAwareCompressor(Transform):
node_text = code[node.start_byte : node.end_byte]
body_line_counts[qname] = max(1, len(node_text.split("\n")) - 2)
- # Reference counts: use short name counts, subtract 1 per definition with that name
- # (multiple definitions with the same short name each contribute -1)
+ # Reference counts: subtract definition occurrences
short_name_def_count: dict[str, int] = {}
for short in bare_names.values():
short_name_def_count[short] = short_name_def_count.get(short, 0) + 1
@@ -587,7 +791,6 @@ class CodeAwareCompressor(Transform):
ref_counts[qname] = max(0, count - short_name_def_count.get(short, 1))
# Raw importance signals per symbol
- # Context matching: split into words AND check as substring for multi-word names
context_lower = context.lower() if context else ""
context_words = set(re.split(r"[\s,;:.()\[\]{}\"']+", context_lower)) if context else set()
context_words.discard("")
@@ -597,7 +800,7 @@ class CodeAwareCompressor(Transform):
short = bare_names[qname]
refs = ref_counts.get(qname, 0)
fan_out = len(function_calls.get(qname, set()))
- is_public = self._is_public_symbol(short, language)
+ is_public = _is_public_symbol(short, language)
raw = float(refs)
raw += 1.0 if is_public else 0.0
@@ -608,11 +811,10 @@ class CodeAwareCompressor(Transform):
if short.startswith("__") and short.endswith("__"):
raw += 2.0
elif language == CodeLanguage.GO:
- if short[0].isupper():
+ if short and short[0].isupper():
raw += 1.0
- # Context boost: match whole symbol name against context words,
- # or check if multi-word symbol appears as substring in context
+ # Context boost
if context_words:
name_lower = short.lower()
if name_lower in context_words or (
@@ -622,7 +824,7 @@ class CodeAwareCompressor(Transform):
raw_signals[qname] = raw
- # Normalize to 0-1 using min-max scaling (distribution-based)
+ # Normalize to 0-1 using min-max scaling
values = list(raw_signals.values())
min_val = min(values)
max_val = max(values)
@@ -631,7 +833,6 @@ class CodeAwareCompressor(Transform):
if range_val > 0:
scores = {name: round((v - min_val) / range_val, 3) for name, v in raw_signals.items()}
else:
- # All symbols have equal importance (test files, utility libraries)
scores = dict.fromkeys(raw_signals, 0.5)
return _SymbolAnalysis(
@@ -642,30 +843,9 @@ class CodeAwareCompressor(Transform):
bare_names=bare_names,
)
- def _get_definition_name(self, node: Any) -> str | None:
- """Extract the name identifier from a definition AST node."""
- for child in node.children:
- if child.type in ("identifier", "name", "type_identifier", "property_identifier"):
- text = child.text
- return text.decode("utf-8") if isinstance(text, bytes) else str(text)
- return None
-
- def _is_public_symbol(self, name: str, language: CodeLanguage) -> bool:
- """Heuristic for whether a symbol is public/exported."""
- if not name:
- return False
- if language == CodeLanguage.GO:
- return name[0].isupper()
- return not name.startswith("_")
-
def _allocate_body_budget(self, analysis: _SymbolAnalysis, code: str) -> dict[str, int]:
"""Allocate body line budget across functions using target_compression_rate.
- Instead of hardcoded tiers, distributes a total body line budget
- proportionally to each symbol's importance score. The budget is
- derived from target_compression_rate and accounts for fixed overhead
- (imports, signatures) that are always preserved.
-
Returns dict mapping symbol name to max body lines to keep.
"""
if not analysis.scores or not analysis.body_line_counts:
@@ -679,18 +859,14 @@ class CodeAwareCompressor(Transform):
total_body_lines = sum(body_sizes.values())
fixed_lines = max(0, total_lines - total_body_lines)
- # Body budget = target total output - fixed overhead
target_total = total_lines * target_rate
body_budget = max(0.0, target_total - fixed_lines)
if total_body_lines == 0:
return {}
- # Score floor: even lowest-scored symbols get a minimal allocation
- # so the LLM sees they exist (vs just `pass`)
score_floor = 0.05
- # Weight = score * body_size (larger important functions get more budget)
weights: dict[str, float] = {}
for name in scores:
score = max(scores.get(name, 0.5), score_floor)
@@ -700,7 +876,6 @@ class CodeAwareCompressor(Transform):
total_weight = sum(weights.values())
if total_weight == 0:
- # Edge case: distribute evenly
per_func = max(0, int(body_budget / max(len(scores), 1)))
return {name: min(per_func, body_sizes.get(name, 0)) for name in scores}
@@ -711,49 +886,12 @@ class CodeAwareCompressor(Transform):
limit = min(int(round(allocation)), max_lines)
limits[qname] = limit
# Also store by short name so _get_body_limit can find it.
- # On collision, keep the higher limit (more generous).
short = analysis.bare_names.get(qname, qname)
if short not in limits or limit > limits[short]:
limits[short] = limit
return limits
- def _get_body_limit(self, func_name: str | None) -> int:
- """Look up the allocated body line limit for a function.
-
- Falls back to max_body_lines if no budget allocation was computed.
- max_body_lines always acts as a hard cap — budget allocation can give
- a function FEWER lines but never MORE than max_body_lines.
- """
- if self._body_limits and func_name and func_name in self._body_limits:
- return min(self._body_limits[func_name], self.config.max_body_lines)
- return self.config.max_body_lines
-
- def _make_omitted_comment(
- self,
- func_name: str | None,
- omitted_count: int,
- indent: str,
- comment_prefix: str = "#",
- ) -> str:
- """Build omitted comment with call information from analysis."""
- calls_info = ""
- if self._symbol_analysis and func_name:
- # Try both short name and as-is (qualified name) for lookup
- for key in (
- func_name,
- *(k for k in self._symbol_analysis.calls if k.endswith(f".{func_name}")),
- ):
- if key in self._symbol_analysis.calls:
- called = self._symbol_analysis.calls[key]
- if called:
- sorted_calls = sorted(called)[:5]
- calls_info = "; calls: " + ", ".join(sorted_calls)
- if len(called) > 5:
- calls_info += f" +{len(called) - 5} more"
- break
- return f"{indent}{comment_prefix} [{omitted_count} lines omitted{calls_info}]"
-
# =========================================================================
# Core compression
# =========================================================================
@@ -763,6 +901,7 @@ class CodeAwareCompressor(Transform):
code: str,
language: str | None = None,
context: str = "",
+ tokenizer: Tokenizer | None = None,
) -> CodeCompressionResult:
"""Compress code while preserving syntax validity.
@@ -770,6 +909,7 @@ class CodeAwareCompressor(Transform):
code: Source code to compress.
language: Language name (e.g., 'python'). Auto-detected if None.
context: Optional context for relevance-aware compression.
+ tokenizer: Optional tokenizer for accurate token counting.
Returns:
CodeCompressionResult with compressed code and metadata.
@@ -784,8 +924,7 @@ class CodeAwareCompressor(Transform):
syntax_valid=True,
)
- # Estimate tokens
- original_tokens = len(code.split())
+ original_tokens = self._estimate_tokens(code, tokenizer)
# Skip small content
if original_tokens < self.config.min_tokens_for_compression:
@@ -813,7 +952,6 @@ class CodeAwareCompressor(Transform):
if self.config.fallback_to_llmlingua:
return self._fallback_compress(code, original_tokens)
else:
- # Pass through unchanged
return CodeCompressionResult(
compressed=code,
original=code,
@@ -844,11 +982,11 @@ class CodeAwareCompressor(Transform):
# Parse and compress
try:
compressed, structure, symbol_scores = self._compress_with_ast(
- code, detected_lang, context
+ code, detected_lang, context, tokenizer
)
- compressed_tokens = len(compressed.split())
+ compressed_tokens = self._estimate_tokens(compressed, tokenizer)
- # Verify syntax validity
+ # Verify syntax validity (checks both ERROR and MISSING nodes)
syntax_valid = self._verify_syntax(compressed, detected_lang)
# If syntax invalid, fall back to original
@@ -867,9 +1005,7 @@ class CodeAwareCompressor(Transform):
ratio = compressed_tokens / max(original_tokens, 1)
- # Guard against over-aggressive compression (data loss).
- # If AST extraction stripped content to <5% of original,
- # the output is essentially empty — return original.
+ # Guard against over-aggressive compression (data loss)
if ratio < 0.05:
logger.warning(
"Code compression too aggressive (ratio=%.3f), returning original",
@@ -891,7 +1027,6 @@ class CodeAwareCompressor(Transform):
if self.config.enable_ccr and ratio < 0.8:
cache_key = self._store_in_ccr(code, compressed, original_tokens)
if cache_key:
- # Generate summary from AST data (language-agnostic)
from .compression_summary import summarize_compressed_code
code_summary = summarize_compressed_code(
@@ -900,8 +1035,8 @@ class CodeAwareCompressor(Transform):
)
summary_str = f" {code_summary}." if code_summary else ""
- # Add CCR marker (hash without quotes, matches CCRToolInjector regex)
- ttl_min = max(1, getattr(self.config, "ccr_ttl_seconds", 300) // 60)
+ # Use the actual config attribute (not the wrong name)
+ ttl_min = max(1, self.config.ccr_ttl // 60)
compressed += (
f"\n# [{original_tokens - compressed_tokens} tokens compressed."
f"{summary_str}"
@@ -945,40 +1080,36 @@ class CodeAwareCompressor(Transform):
code: str,
language: CodeLanguage,
context: str,
+ tokenizer: Tokenizer | None = None,
) -> tuple[str, CodeStructure, dict[str, float]]:
"""Compress code using AST parsing with symbol importance analysis.
+ Thread-safe: all mutable state is passed through parameters, not
+ stored on self.
+
Args:
code: Source code.
language: Detected language.
context: User context for relevance.
+ tokenizer: Optional tokenizer for accurate token counting.
Returns:
Tuple of (compressed code, extracted structure, symbol scores).
"""
- # Get parser for language
parser = _get_parser(language.value)
-
- # Parse code
tree = parser.parse(bytes(code, "utf-8"))
root = tree.root_node
# Analyze symbol importance and allocate compression budget
analysis = self._analyze_symbol_importance(root, code, language, context)
- self._symbol_analysis = analysis
- self._body_limits = self._allocate_body_budget(analysis, code)
-
- # Extract structure based on language
- if language == CodeLanguage.PYTHON:
- structure = self._extract_python_structure(root, code)
- elif language in (CodeLanguage.JAVASCRIPT, CodeLanguage.TYPESCRIPT):
- structure = self._extract_js_structure(root, code)
- elif language == CodeLanguage.GO:
- structure = self._extract_go_structure(root, code)
- elif language == CodeLanguage.RUST:
- structure = self._extract_rust_structure(root, code)
- elif language == CodeLanguage.JAVA:
- structure = self._extract_java_structure(root, code)
+ body_limits = self._allocate_body_budget(analysis, code)
+
+ # Extract structure using data-driven language config
+ lang_config = _LANG_CONFIGS.get(language)
+ if lang_config:
+ structure = self._extract_structure(
+ root, code, language, lang_config, body_limits, analysis
+ )
else:
structure = self._extract_generic_structure(root, code)
@@ -990,457 +1121,457 @@ class CodeAwareCompressor(Transform):
if analysis.scores:
for qname, score in analysis.scores.items():
short = analysis.bare_names.get(qname, qname)
- # On collision, keep the higher score (more generous)
if short not in symbol_scores or score > symbol_scores[short]:
symbol_scores[short] = score
- self._symbol_analysis = None
- self._body_limits = None
return compressed, structure, symbol_scores
- def _extract_python_structure(self, root: Any, code: str) -> CodeStructure:
- """Extract structure from Python AST."""
+ # =========================================================================
+ # Unified structure extraction (data-driven, replaces per-language methods)
+ # =========================================================================
+
+ def _extract_structure(
+ self,
+ root: Any,
+ code: str,
+ language: CodeLanguage,
+ lang_config: LangConfig,
+ body_limits: dict[str, int],
+ analysis: _SymbolAnalysis,
+ ) -> CodeStructure:
+ """Extract structure from AST using data-driven language config.
+
+ A single visitor handles all languages by checking node types against
+ the LangConfig tables. No per-language extraction methods needed.
+ """
structure = CodeStructure()
- lines = code.split("\n")
+ captured_byte_ranges: list[tuple[int, int]] = []
def visit(node: Any) -> None:
node_type = node.type
- if node_type == "import_statement":
- structure.imports.append(self._get_node_text(node, code))
+ # Package declarations (Go, Java)
+ if lang_config.package_node and node_type == lang_config.package_node:
+ structure.imports.insert(0, _get_node_text(node, code))
+ captured_byte_ranges.append((node.start_byte, node.end_byte))
+ return
+
+ # Import statements
+ if node_type in lang_config.import_nodes:
+ structure.imports.append(_get_node_text(node, code))
+ captured_byte_ranges.append((node.start_byte, node.end_byte))
+ return
- elif node_type == "import_from_statement":
- structure.imports.append(self._get_node_text(node, code))
+ # Export statements (JS/TS) — may contain functions or re-exports
+ if node_type == "export_statement":
+ text = _get_node_text(node, code)
+ # Check if this export wraps a function or class
+ has_func_or_class = False
+ for child in node.children:
+ if (
+ child.type in lang_config.function_nodes
+ or child.type in lang_config.class_nodes
+ ):
+ has_func_or_class = True
+ compressed = self._compress_function_ast(
+ child, code, language, lang_config, body_limits, analysis
+ )
+ # Reconstruct export with compressed inner definition
+ export_prefix = code[node.start_byte : child.start_byte]
+ export_suffix = code[child.end_byte : node.end_byte]
+ structure.function_signatures.append(
+ export_prefix + compressed + export_suffix
+ )
+ break
+ if not has_func_or_class:
+ structure.imports.append(text)
+ captured_byte_ranges.append((node.start_byte, node.end_byte))
+ return
- elif node_type == "decorated_definition":
- # Get decorator and the definition
+ # Decorated definitions (Python)
+ if lang_config.decorator_node and node_type == lang_config.decorator_node:
decorator_text = []
- definition_text = None
+ definition_compressed = None
for child in node.children:
if child.type == "decorator":
- decorator_text.append(self._get_node_text(child, code))
- elif child.type in ("function_definition", "class_definition"):
- definition_text = self._extract_definition(child, code, lines)
- if decorator_text and definition_text:
- full_def = "\n".join(decorator_text) + "\n" + definition_text
- if "class" in definition_text:
- structure.class_definitions.append(full_def)
+ decorator_text.append(_get_node_text(child, code))
+ elif child.type in lang_config.function_nodes:
+ definition_compressed = self._compress_function_ast(
+ child, code, language, lang_config, body_limits, analysis
+ )
+ elif child.type in lang_config.class_nodes:
+ definition_compressed = self._compress_class_ast(
+ child, code, language, lang_config, body_limits, analysis
+ )
+ if decorator_text and definition_compressed:
+ full_def = "\n".join(decorator_text) + "\n" + definition_compressed
+ # Route to correct list based on inner definition type
+ for child in node.children:
+ if child.type in lang_config.class_nodes:
+ structure.class_definitions.append(full_def)
+ break
else:
structure.function_signatures.append(full_def)
- return # Don't recurse into children
-
- elif node_type == "function_definition":
- definition = self._extract_definition(node, code, lines)
- structure.function_signatures.append(definition)
+ elif definition_compressed:
+ structure.function_signatures.append(definition_compressed)
+ captured_byte_ranges.append((node.start_byte, node.end_byte))
return
- elif node_type == "class_definition":
- definition = self._extract_definition(node, code, lines)
- structure.class_definitions.append(definition)
+ # Function/method definitions
+ if node_type in lang_config.function_nodes:
+ compressed = self._compress_function_ast(
+ node, code, language, lang_config, body_limits, analysis
+ )
+ structure.function_signatures.append(compressed)
+ captured_byte_ranges.append((node.start_byte, node.end_byte))
return
- elif node_type == "try_statement":
- if self.config.preserve_error_handlers:
- structure.error_handlers.append(self._get_node_text(node, code))
+ # Class definitions — compress each method individually
+ if node_type in lang_config.class_nodes:
+ compressed = self._compress_class_ast(
+ node, code, language, lang_config, body_limits, analysis
+ )
+ structure.class_definitions.append(compressed)
+ captured_byte_ranges.append((node.start_byte, node.end_byte))
return
- elif node_type == "type_alias_statement":
- structure.type_definitions.append(self._get_node_text(node, code))
+ # Type definitions
+ if node_type in lang_config.type_nodes:
+ structure.type_definitions.append(_get_node_text(node, code))
+ captured_byte_ranges.append((node.start_byte, node.end_byte))
+ return
# Recurse into children
for child in node.children:
visit(child)
visit(root)
- return structure
-
- def _extract_definition(self, node: Any, code: str, lines: list[str]) -> str:
- """Extract a function/class definition with budget-aware compressed body."""
- node_text = self._get_node_text(node, code)
- node_lines = node_text.split("\n")
-
- # Look up allocated body line limit for this symbol
- func_name = self._get_definition_name(node)
- body_limit = self._get_body_limit(func_name)
-
- # Check if small enough to keep as-is
- if len(node_lines) <= body_limit + 2:
- return node_text
-
- # Find signature (first line(s) until colon)
- signature_lines = []
- body_start = 0
- paren_depth = 0
- found_colon = False
-
- for i, line in enumerate(node_lines):
- signature_lines.append(line)
- paren_depth += line.count("(") - line.count(")")
- if ":" in line and paren_depth <= 0:
- # Check if this is the end of signature
- if line.rstrip().endswith(":"):
- found_colon = True
- body_start = i + 1
- break
-
- if not found_colon:
- # Couldn't parse signature, return truncated
- return "\n".join(node_lines[: max(body_limit, 1)]) + "\n # ..."
-
- signature = "\n".join(signature_lines)
-
- # Check for docstring
- docstring = ""
- if body_start < len(node_lines):
- first_body_line = node_lines[body_start].strip()
- if first_body_line.startswith(('"""', "'''")):
- quote = first_body_line[:3]
- docstring_lines = [node_lines[body_start]]
- if first_body_line.count(quote) >= 2:
- # Single line docstring
- if self.config.docstring_mode == DocstringMode.FULL:
- docstring = node_lines[body_start]
- elif self.config.docstring_mode == DocstringMode.FIRST_LINE:
- docstring = node_lines[body_start]
- body_start += 1
- else:
- # Multi-line docstring
- for j in range(body_start + 1, len(node_lines)):
- docstring_lines.append(node_lines[j])
- if quote in node_lines[j]:
- break
- body_start = body_start + len(docstring_lines)
- if self.config.docstring_mode == DocstringMode.FULL:
- docstring = "\n".join(docstring_lines)
- elif self.config.docstring_mode == DocstringMode.FIRST_LINE:
- # Keep first line of docstring
- first_doc = docstring_lines[0].strip()
- if first_doc == '"""' or first_doc == "'''":
- # Opening on its own line
- if len(docstring_lines) > 1:
- indent_len = len(docstring_lines[0]) - len(
- docstring_lines[0].lstrip()
- )
- docstring = " " * indent_len + '"""' + docstring_lines[1].strip()
- if not docstring.rstrip().endswith('"""'):
- docstring += '"""'
- else:
- docstring = docstring_lines[0]
- if not docstring.rstrip().endswith('"""'):
- docstring = docstring.rstrip() + '..."""'
-
- # Build compressed output
- body_lines = node_lines[body_start:]
- total_body = len(body_lines)
-
- # Keep body lines based on budget-allocated limit
- keep_lines = min(body_limit, len(body_lines))
-
- # Determine indentation from the first omitted line, not the first body line.
- # This handles nested structures correctly: if the cut falls inside a nested
- # method definition, the pass/omitted comment use the inner indent level.
- indent = " "
- if body_lines:
- omitted_part = body_lines[keep_lines:] if keep_lines < len(body_lines) else body_lines
- first_non_empty = next((line for line in omitted_part if line.strip()), "")
- if not first_non_empty:
- first_non_empty = next((line for line in body_lines if line.strip()), "")
- if first_non_empty:
- indent = first_non_empty[: len(first_non_empty) - len(first_non_empty.lstrip())]
- compressed_body = body_lines[:keep_lines]
-
- result_parts = [signature]
- if docstring and self.config.docstring_mode not in (
- DocstringMode.NONE,
- DocstringMode.REMOVE,
- ):
- result_parts.append(docstring)
-
- if compressed_body:
- result_parts.extend(compressed_body)
-
- if total_body > keep_lines:
- omitted = total_body - keep_lines
- result_parts.append(self._make_omitted_comment(func_name, omitted, indent))
- result_parts.append(f"{indent}pass")
-
- return "\n".join(result_parts)
-
- def _extract_js_structure(self, root: Any, code: str) -> CodeStructure:
- """Extract structure from JavaScript/TypeScript AST."""
- structure = CodeStructure()
- lines = code.split("\n")
-
- def visit(node: Any) -> None:
- node_type = node.type
-
- if node_type in ("import_statement", "import_declaration"):
- structure.imports.append(self._get_node_text(node, code))
-
- elif node_type == "export_statement":
- text = self._get_node_text(node, code)
- if "function" in text or "class" in text:
- structure.function_signatures.append(
- self._compress_js_function(node, code, lines)
- )
- else:
- structure.imports.append(text) # export declarations
- return
-
- elif node_type in ("function_declaration", "method_definition"):
- structure.function_signatures.append(self._compress_js_function(node, code, lines))
- return
-
- elif node_type == "class_declaration":
- structure.class_definitions.append(self._compress_js_class(node, code, lines))
- return
- elif node_type in ("interface_declaration", "type_alias_declaration"):
- structure.type_definitions.append(self._get_node_text(node, code))
+ # Capture top-level code that wasn't handled by any of the above.
+ # This preserves global variables, constants, if __name__ blocks,
+ # module-level assignments, etc.
+ for child in root.children:
+ child_range = (child.start_byte, child.end_byte)
+ if child_range not in captured_byte_ranges:
+ text = _get_node_text(child, code).strip()
+ if text:
+ structure.top_level_code.append(text)
- elif node_type == "try_statement":
- if self.config.preserve_error_handlers:
- structure.error_handlers.append(self._get_node_text(node, code))
- return
-
- for child in node.children:
- visit(child)
-
- visit(root)
return structure
- def _compress_js_function(self, node: Any, code: str, lines: list[str]) -> str:
- """Compress a JavaScript function with budget-aware retention."""
- node_text = self._get_node_text(node, code)
- node_lines = node_text.split("\n")
+ # =========================================================================
+ # Unified function/class compression (data-driven)
+ # =========================================================================
- func_name = self._get_definition_name(node)
- body_limit = self._get_body_limit(func_name)
+ def _compress_function_ast(
+ self,
+ node: Any,
+ code: str,
+ language: CodeLanguage,
+ lang_config: LangConfig,
+ body_limits: dict[str, int],
+ analysis: _SymbolAnalysis,
+ ) -> str:
+ """Compress a function/class/impl block using AST body detection.
+
+ Uses the AST to find the body node directly instead of string-scanning
+ for '{' or ':'. Works for all languages via lang_config.body_node_types.
+ Key insight: tree-sitter byte offsets may not include leading whitespace
+ on the first line. We use LINE-based slicing from the original code to
+ preserve indentation faithfully.
+ """
+ # Use line-based slicing from original code (not byte offsets) to
+ # preserve indentation. This is critical for nested definitions
+ # (methods inside classes).
+ code_lines = code.split("\n")
+ start_row = node.start_point[0]
+ end_row = node.end_point[0]
+ node_lines = code_lines[start_row : end_row + 1]
+ node_text = "\n".join(node_lines)
+
+ func_name = _get_definition_name(node)
+ body_limit = _get_body_limit(func_name, body_limits, self.config.max_body_lines)
+
+ # Small enough to keep as-is
if len(node_lines) <= body_limit + 2:
return node_text
- # Find opening brace
- signature_lines = []
- body_start = 0
- for i, line in enumerate(node_lines):
- signature_lines.append(line)
- if "{" in line:
- body_start = i + 1
+ # Find the body node using AST (not string scanning)
+ body_node = None
+ for child in node.children:
+ if child.type in lang_config.body_node_types:
+ body_node = child
break
- if body_start == 0:
- return node_text # Arrow function or other format
-
- body_lines = node_lines[body_start:-1] # Exclude closing brace
- total_body = len(body_lines)
- keep_lines = min(body_limit, total_body)
-
- result = signature_lines + body_lines[:keep_lines]
- if total_body > keep_lines:
- result.append(
- self._make_omitted_comment(func_name, total_body - keep_lines, " ", "//")
- )
- result.append(node_lines[-1]) # Closing brace
-
- return "\n".join(result)
-
- def _compress_js_class(self, node: Any, code: str, lines: list[str]) -> str:
- """Compress a JavaScript class, keeping method signatures."""
- # For now, use similar logic to function compression
- return self._compress_js_function(node, code, lines)
-
- def _extract_go_structure(self, root: Any, code: str) -> CodeStructure:
- """Extract structure from Go AST."""
- structure = CodeStructure()
- lines = code.split("\n")
-
- def visit(node: Any) -> None:
- node_type = node.type
-
- if node_type == "import_declaration":
- structure.imports.append(self._get_node_text(node, code))
-
- elif node_type == "package_clause":
- structure.imports.insert(0, self._get_node_text(node, code))
-
- elif node_type == "function_declaration":
- structure.function_signatures.append(self._compress_go_function(node, code, lines))
- return
-
- elif node_type == "method_declaration":
- structure.function_signatures.append(self._compress_go_function(node, code, lines))
- return
-
- elif node_type == "type_declaration":
- structure.type_definitions.append(self._get_node_text(node, code))
-
- for child in node.children:
- visit(child)
-
- visit(root)
- return structure
-
- def _compress_go_function(self, node: Any, code: str, lines: list[str]) -> str:
- """Compress a Go function with budget-aware retention."""
- node_text = self._get_node_text(node, code)
- node_lines = node_text.split("\n")
-
- func_name = self._get_definition_name(node)
- body_limit = self._get_body_limit(func_name)
-
- if len(node_lines) <= body_limit + 2:
+ if body_node is None:
return node_text
- # Find opening brace
- signature_lines = []
- body_start = 0
- for i, line in enumerate(node_lines):
- signature_lines.append(line)
- if "{" in line:
- body_start = i + 1
- break
-
- body_lines = node_lines[body_start:-1]
- total_body = len(body_lines)
+ # Use line numbers to slice: this preserves original indentation.
+ # tree-sitter gives 0-based row numbers.
+ node_start_line = node.start_point[0]
+ body_start_line = body_node.start_point[0]
+ body_end_line = body_node.end_point[0]
+
+ # Lines within the node (0-indexed relative to node start)
+ sig_end = body_start_line - node_start_line # exclusive
+ body_end_rel = body_end_line - node_start_line + 1 # inclusive
+
+ # Handle case where signature and body start on the SAME line
+ # (common in brace languages: `function foo(arg) { ... }`)
+ if sig_end == 0 and not lang_config.uses_colon_after_signature:
+ # Signature and body on same line: `function foo(arg) { ... }`
+ # Keep them together: sig includes up to and including `{`
+ first_line = node_lines[0]
+ # Include the opening brace in the signature line
+ sig_with_brace = first_line.rstrip()
+ signature_lines = [sig_with_brace]
+ # Body lines are everything between { and } (inner content only)
+ body_lines = node_lines[1:body_end_rel]
+ after_lines = node_lines[body_end_rel:]
+ # We've already included { in signature, so mark it
+ _brace_in_signature = True
+ else:
+ signature_lines = node_lines[:sig_end]
+ body_lines = node_lines[sig_end:body_end_rel]
+ after_lines = node_lines[body_end_rel:]
+ _brace_in_signature = False
+
+ # For brace languages, detect opening/closing braces in the body lines.
+ opening_brace_line = None
+ closing_brace_line = None
+ if not lang_config.uses_colon_after_signature:
+ if _brace_in_signature:
+ # Opening brace already in signature line — just find closing
+ pass
+ elif body_lines and body_lines[0].strip().startswith("{"):
+ opening_brace_line = body_lines[0]
+ body_lines = body_lines[1:]
+ if body_lines and body_lines[-1].strip().endswith("}"):
+ closing_brace_line = body_lines[-1]
+ body_lines = body_lines[:-1]
+
+ # Handle Python docstrings via AST
+ docstring_text = ""
+ ds_skip_lines = 0
+ if language == CodeLanguage.PYTHON and body_node.child_count > 0:
+ first_child = body_node.children[0]
+ # tree-sitter Python may represent docstrings as:
+ # - bare `string` node directly in block, OR
+ # - `expression_statement` containing a `string` node
+ ds_node = None
+ if first_child.type == "string":
+ ds_node = first_child
+ elif first_child.type == "expression_statement" and first_child.child_count > 0:
+ if first_child.children[0].type == "string":
+ ds_node = first_child
+
+ if ds_node is not None:
+ ds_text = _get_node_text(ds_node, code)
+ ds_lines_count = ds_node.end_point[0] - ds_node.start_point[0] + 1
+ ds_start_rel = ds_node.start_point[0] - body_node.start_point[0]
+
+ if self.config.docstring_mode == DocstringMode.FULL:
+ # Keep entire docstring as-is (preserve indentation from body_lines)
+ docstring_text = "\n".join(
+ body_lines[ds_start_rel : ds_start_rel + ds_lines_count]
+ )
+ elif self.config.docstring_mode == DocstringMode.FIRST_LINE:
+ full_ds = ds_text.strip()
+ ds_first_line_indent = (
+ body_lines[ds_start_rel][
+ : len(body_lines[ds_start_rel]) - len(body_lines[ds_start_rel].lstrip())
+ ]
+ if body_lines[ds_start_rel:]
+ else ""
+ )
+ ds_stripped_lines = full_ds.split("\n")
+ if len(ds_stripped_lines) == 1:
+ docstring_text = body_lines[ds_start_rel]
+ else:
+ first_stripped = ds_stripped_lines[0].strip()
+ if first_stripped in ('"""', "'''", 'r"""', "r'''"):
+ # Opening quote on its own line
+ if len(ds_stripped_lines) > 1:
+ docstring_text = (
+ ds_first_line_indent
+ + '"""'
+ + ds_stripped_lines[1].strip()
+ + '"""'
+ )
+ elif first_stripped.endswith('"""'):
+ # Single-line docstring: """text"""
+ docstring_text = body_lines[ds_start_rel]
+ else:
+ # Multi-line: """text\n...\n""" → """text..."""
+ docstring_text = ds_first_line_indent + first_stripped + '..."""'
+ # elif REMOVE: docstring_text stays empty
+ ds_skip_lines = ds_start_rel + ds_lines_count
+
+ # Get content lines after docstring
+ content_lines = body_lines[ds_skip_lines:]
+ total_body = len(content_lines)
keep_lines = min(body_limit, total_body)
- result = signature_lines + body_lines[:keep_lines]
- if total_body > keep_lines:
- result.append(
- self._make_omitted_comment(func_name, total_body - keep_lines, "\t", "//")
- )
- result.append(node_lines[-1])
-
- return "\n".join(result)
+ # Detect indentation from actual body code (preserves whatever the file uses)
+ indent = _detect_indent(body_lines) if body_lines else " "
- def _extract_rust_structure(self, root: Any, code: str) -> CodeStructure:
- """Extract structure from Rust AST."""
- structure = CodeStructure()
- lines = code.split("\n")
-
- def visit(node: Any) -> None:
- node_type = node.type
+ # Build compressed output preserving original indentation
+ result_parts: list[str] = []
- if node_type == "use_declaration":
- structure.imports.append(self._get_node_text(node, code))
-
- elif node_type == "function_item":
- structure.function_signatures.append(
- self._compress_rust_function(node, code, lines)
- )
- return
-
- elif node_type in ("struct_item", "enum_item", "type_item"):
- structure.type_definitions.append(self._get_node_text(node, code))
-
- elif node_type == "impl_item":
- structure.class_definitions.append(self._compress_rust_impl(node, code, lines))
- return
-
- for child in node.children:
- visit(child)
-
- visit(root)
- return structure
-
- def _compress_rust_function(self, node: Any, code: str, lines: list[str]) -> str:
- """Compress a Rust function with budget-aware retention."""
- node_text = self._get_node_text(node, code)
- node_lines = node_text.split("\n")
-
- func_name = self._get_definition_name(node)
- body_limit = self._get_body_limit(func_name)
+ # Signature lines (may be multi-line)
+ if signature_lines:
+ result_parts.extend(signature_lines)
+ else:
+ # Single-line: signature and body on same line — extract up to colon/brace
+ sig_text = code[node.start_byte : body_node.start_byte].rstrip()
+ result_parts.append(sig_text)
- if len(node_lines) <= body_limit + 2:
- return node_text
+ if opening_brace_line is not None:
+ result_parts.append(opening_brace_line)
- # Find opening brace
- signature_lines = []
- body_start = 0
- for i, line in enumerate(node_lines):
- signature_lines.append(line)
- if "{" in line:
- body_start = i + 1
- break
+ if docstring_text and self.config.docstring_mode not in (
+ DocstringMode.NONE,
+ DocstringMode.REMOVE,
+ ):
+ result_parts.append(docstring_text)
- body_lines = node_lines[body_start:-1]
- total_body = len(body_lines)
- keep_lines = min(body_limit, total_body)
+ if keep_lines > 0:
+ result_parts.extend(content_lines[:keep_lines])
- result = signature_lines + body_lines[:keep_lines]
if total_body > keep_lines:
- result.append(
- self._make_omitted_comment(func_name, total_body - keep_lines, " ", "//")
+ omitted = total_body - keep_lines
+ result_parts.append(
+ _make_omitted_comment(
+ func_name, omitted, indent, lang_config.comment_prefix, analysis
+ )
)
- result.append(node_lines[-1])
-
- return "\n".join(result)
+ if lang_config.uses_colon_after_signature:
+ result_parts.append(f"{indent}pass")
- def _compress_rust_impl(self, node: Any, code: str, lines: list[str]) -> str:
- """Compress a Rust impl block."""
- return self._compress_rust_function(node, code, lines)
+ if closing_brace_line is not None:
+ result_parts.append(closing_brace_line)
+ elif after_lines:
+ result_parts.extend(after_lines)
- def _extract_java_structure(self, root: Any, code: str) -> CodeStructure:
- """Extract structure from Java AST."""
- structure = CodeStructure()
- lines = code.split("\n")
-
- def visit(node: Any) -> None:
- node_type = node.type
-
- if node_type == "import_declaration":
- structure.imports.append(self._get_node_text(node, code))
-
- elif node_type == "package_declaration":
- structure.imports.insert(0, self._get_node_text(node, code))
-
- elif node_type == "class_declaration":
- structure.class_definitions.append(self._compress_java_class(node, code, lines))
- return
+ return "\n".join(result_parts)
- elif node_type == "method_declaration":
- structure.function_signatures.append(self._compress_java_method(node, code, lines))
- return
+ def _compress_class_ast(
+ self,
+ node: Any,
+ code: str,
+ language: CodeLanguage,
+ lang_config: LangConfig,
+ body_limits: dict[str, int],
+ analysis: _SymbolAnalysis,
+ ) -> str:
+ """Compress a class by individually compressing each method.
- for child in node.children:
- visit(child)
+ Preserves class-level attributes, type annotations, and decorators
+ while compressing method bodies individually. This ensures correct
+ indentation for each method's omitted-body comment.
+ """
+ # Use line-based extraction to preserve indentation
+ code_lines = code.split("\n")
+ start_row = node.start_point[0]
+ end_row = node.end_point[0]
+ node_lines = code_lines[start_row : end_row + 1]
+ node_text = "\n".join(node_lines)
+
+ # Find the body node
+ body_node = None
+ for child in node.children:
+ if child.type in lang_config.body_node_types:
+ body_node = child
+ break
- visit(root)
- return structure
+ if body_node is None:
+ return node_text
- def _compress_java_class(self, node: Any, code: str, lines: list[str]) -> str:
- """Compress a Java class."""
- return self._compress_js_function(node, code, lines)
+ # Class header (signature) — everything before the body
+ node_start_line = node.start_point[0]
+ body_start_line = body_node.start_point[0]
+ sig_end = body_start_line - node_start_line
+ header_lines = node_lines[:sig_end] if sig_end > 0 else [node_lines[0]]
+
+ # Process each child of the class body individually
+ body_parts: list[str] = []
+ processed_ranges: list[tuple[int, int]] = []
+
+ for child in body_node.children:
+ # Use line-based extraction for children too
+ child_start = child.start_point[0]
+ child_end = child.end_point[0]
+ child_text = "\n".join(code_lines[child_start : child_end + 1])
+
+ # Methods/functions inside the class — compress individually
+ if child.type in lang_config.function_nodes:
+ compressed = self._compress_function_ast(
+ child, code, language, lang_config, body_limits, analysis
+ )
+ body_parts.append(compressed)
+ processed_ranges.append((child.start_byte, child.end_byte))
+ # Decorated methods
+ elif lang_config.decorator_node and child.type == lang_config.decorator_node:
+ decorator_lines = []
+ method_compressed = None
+ for deco_child in child.children:
+ if deco_child.type == "decorator":
+ decorator_lines.append(_get_node_text(deco_child, code))
+ elif deco_child.type in lang_config.function_nodes:
+ method_compressed = self._compress_function_ast(
+ deco_child, code, language, lang_config, body_limits, analysis
+ )
+ if decorator_lines and method_compressed:
+ body_parts.append("\n".join(decorator_lines) + "\n" + method_compressed)
+ elif method_compressed:
+ body_parts.append(method_compressed)
+ else:
+ body_parts.append(child_text)
+ processed_ranges.append((child.start_byte, child.end_byte))
+ # Nested classes — recurse
+ elif child.type in lang_config.class_nodes:
+ compressed = self._compress_class_ast(
+ child, code, language, lang_config, body_limits, analysis
+ )
+ body_parts.append(compressed)
+ processed_ranges.append((child.start_byte, child.end_byte))
+ else:
+ # Class-level attributes, type annotations, docstrings, etc.
+ # Keep them as-is with original indentation
+ if child_text.strip():
+ body_parts.append(child_text)
+ processed_ranges.append((child.start_byte, child.end_byte))
+
+ # Reconstruct class with proper indentation
+ result_parts = list(header_lines)
+ for part in body_parts:
+ result_parts.append(part)
+
+ # Handle closing brace for brace-delimited languages
+ body_end_line = body_node.end_point[0]
+ body_end_rel = body_end_line - node_start_line + 1
+ after_lines = node_lines[body_end_rel:]
+ if after_lines:
+ result_parts.extend(after_lines)
+ elif not lang_config.uses_colon_after_signature:
+ # Ensure closing brace
+ last_body_line = node_lines[-1] if node_lines else ""
+ if last_body_line.strip() == "}":
+ result_parts.append(last_body_line)
- def _compress_java_method(self, node: Any, code: str, lines: list[str]) -> str:
- """Compress a Java method."""
- return self._compress_js_function(node, code, lines)
+ return "\n".join(result_parts)
def _extract_generic_structure(self, root: Any, code: str) -> CodeStructure:
- """Extract structure from generic code."""
- # Fallback: use line-based compression
- structure = CodeStructure()
- lines = code.split("\n")
-
- # Keep imports (lines starting with import/include/use/from)
- for line in lines:
- stripped = line.strip()
- if any(
- stripped.startswith(kw)
- for kw in ["import ", "from ", "#include", "use ", "require("]
- ):
- structure.imports.append(line)
-
- # Rest goes to other
- structure.other = lines
+ """Extract structure from generic/unknown code.
+ For languages without a LangConfig, we can't reliably separate
+ imports from other code. Just preserve everything in 'other'.
+ """
+ structure = CodeStructure()
+ structure.other = code.split("\n")
return structure
- def _get_node_text(self, node: Any, code: str) -> str:
- """Extract text from AST node."""
- return code[node.start_byte : node.end_byte]
-
def _assemble_compressed(
self,
structure: CodeStructure,
@@ -1452,7 +1583,7 @@ class CodeAwareCompressor(Transform):
# Imports first
if structure.imports:
parts.extend(structure.imports)
- parts.append("") # Empty line after imports
+ parts.append("")
# Type definitions
if structure.type_definitions:
@@ -1469,13 +1600,12 @@ class CodeAwareCompressor(Transform):
parts.extend(structure.function_signatures)
parts.append("")
- # Error handlers (if preserved separately)
- if structure.error_handlers and self.config.preserve_error_handlers:
- parts.append("# Error handlers:")
- parts.extend(structure.error_handlers)
+ # Top-level code (global variables, constants, if __name__, etc.)
+ if structure.top_level_code:
+ parts.extend(structure.top_level_code)
parts.append("")
- # Other content
+ # Other content (used by generic extraction)
if structure.other:
parts.extend(structure.other)
@@ -1486,24 +1616,18 @@ class CodeAwareCompressor(Transform):
return "\n".join(parts)
def _verify_syntax(self, code: str, language: CodeLanguage) -> bool:
- """Verify that code is syntactically valid."""
+ """Verify that code is syntactically valid.
+
+ Checks for both ERROR nodes (parse failures) and MISSING nodes
+ (tokens the parser expected but didn't find).
+ """
try:
parser = _get_parser(language.value)
tree = parser.parse(bytes(code, "utf-8"))
- # Check for ERROR nodes in the tree
- return not self._has_error_nodes(tree.root_node)
+ return not _has_syntax_issues(tree.root_node)
except Exception:
return False
- def _has_error_nodes(self, node: Any) -> bool:
- """Check if AST contains ERROR nodes."""
- if node.type == "ERROR":
- return True
- for child in node.children:
- if self._has_error_nodes(child):
- return True
- return False
-
def _fallback_compress(self, code: str, original_tokens: int) -> CodeCompressionResult:
"""Fall back to LLMLingua compression."""
try:
@@ -1520,7 +1644,8 @@ class CodeAwareCompressor(Transform):
compression_ratio=result.compression_ratio,
language=CodeLanguage.UNKNOWN,
language_confidence=0.0,
- syntax_valid=True, # LLMLingua doesn't guarantee this
+ # LLMLingua does NOT guarantee syntax validity
+ syntax_valid=False,
)
except ImportError:
pass
@@ -1552,7 +1677,7 @@ class CodeAwareCompressor(Transform):
original,
compressed,
original_tokens=original_tokens,
- compressed_tokens=len(compressed.split()),
+ compressed_tokens=self._estimate_tokens(compressed),
compression_strategy="code_aware",
)
except ImportError:
@@ -1569,8 +1694,8 @@ class CodeAwareCompressor(Transform):
) -> TransformResult:
"""Apply code-aware compression to messages.
- This method implements the Transform interface for use in pipelines.
- It compresses code content in tool outputs and messages.
+ Handles both string content and Anthropic content block format
+ (list of {"type": "text", "text": "..."} dicts).
Args:
messages: List of message dicts to transform.
@@ -1580,37 +1705,41 @@ class CodeAwareCompressor(Transform):
Returns:
TransformResult with compressed messages and metadata.
"""
- from .content_detector import ContentType, detect_content_type
-
tokens_before = sum(tokenizer.count_text(str(m.get("content", ""))) for m in messages)
context = kwargs.get("context", "")
transformed_messages = []
- transforms_applied = []
+ transforms_applied: list[str] = []
warnings: list[str] = []
for message in messages:
content = message.get("content", "")
- # Skip empty or non-string content (multimodal messages with images)
+ # Handle content blocks (Anthropic format)
+ if isinstance(content, list):
+ new_blocks = []
+ for block in content:
+ if isinstance(block, dict) and block.get("type") == "text":
+ text = block.get("text", "")
+ compressed_text = self._try_compress_text(
+ text, context, tokenizer, transforms_applied
+ )
+ new_blocks.append({**block, "text": compressed_text})
+ else:
+ new_blocks.append(block)
+ transformed_messages.append({**message, "content": new_blocks})
+ continue
+
+ # Handle string content
if not content or not isinstance(content, str):
transformed_messages.append(message)
continue
- # Check if content is code
- detection = detect_content_type(content)
-
- if detection.content_type == ContentType.SOURCE_CODE:
- language = detection.metadata.get("language")
- result = self.compress(content, language=language, context=context)
-
- if result.compression_ratio < 0.9:
- transformed_messages.append({**message, "content": result.compressed})
- transforms_applied.append(
- f"code_aware:{result.language.value}:{result.compression_ratio:.2f}"
- )
- else:
- transformed_messages.append(message)
+ compressed_content = self._try_compress_text(
+ content, context, tokenizer, transforms_applied
+ )
+ if compressed_content != content:
+ transformed_messages.append({**message, "content": compressed_content})
else:
transformed_messages.append(message)
@@ -1631,6 +1760,30 @@ class CodeAwareCompressor(Transform):
warnings=warnings,
)
+ def _try_compress_text(
+ self,
+ text: str,
+ context: str,
+ tokenizer: Tokenizer,
+ transforms_applied: list[str],
+ ) -> str:
+ """Try to compress a text string if it contains code."""
+ from .content_detector import ContentType, detect_content_type
+
+ if not text:
+ return text
+
+ detection = detect_content_type(text)
+ if detection.content_type == ContentType.SOURCE_CODE:
+ language = detection.metadata.get("language")
+ result = self.compress(text, language=language, context=context, tokenizer=tokenizer)
+ if result.compression_ratio < 0.9:
+ transforms_applied.append(
+ f"code_aware:{result.language.value}:{result.compression_ratio:.2f}"
+ )
+ return result.compressed
+ return text
+
def should_apply(
self,
messages: list[dict[str, Any]],
@@ -1658,15 +1811,110 @@ class CodeAwareCompressor(Transform):
for message in messages:
content = message.get("content", "")
- # Only check string content (skip multimodal)
+ # Handle string content
if content and isinstance(content, str):
detection = detect_content_type(content)
if detection.content_type == ContentType.SOURCE_CODE:
return True
+ # Handle content blocks
+ elif isinstance(content, list):
+ for block in content:
+ if isinstance(block, dict) and block.get("type") == "text":
+ text = block.get("text", "")
+ if text:
+ detection = detect_content_type(text)
+ if detection.content_type == ContentType.SOURCE_CODE:
+ return True
return False
+# =========================================================================
+# Module-level helper functions (stateless, used by the class)
+# =========================================================================
+
+
+def _get_node_text(node: Any, code: str) -> str:
+ """Extract text from AST node."""
+ return code[node.start_byte : node.end_byte]
+
+
+def _get_definition_name(node: Any) -> str | None:
+ """Extract the name identifier from a definition AST node."""
+ for child in node.children:
+ if child.type in ("identifier", "name", "type_identifier", "property_identifier"):
+ text = child.text
+ return text.decode("utf-8") if isinstance(text, bytes) else str(text)
+ return None
+
+
+def _is_public_symbol(name: str, language: CodeLanguage) -> bool:
+ """Heuristic for whether a symbol is public/exported."""
+ if not name:
+ return False
+ if language == CodeLanguage.GO:
+ return name[0].isupper()
+ return not name.startswith("_")
+
+
+def _get_body_limit(
+ func_name: str | None,
+ body_limits: dict[str, int],
+ max_body_lines: int,
+) -> int:
+ """Look up the allocated body line limit for a function.
+
+ Falls back to max_body_lines if no budget allocation was computed.
+ max_body_lines always acts as a hard cap.
+ """
+ if body_limits and func_name and func_name in body_limits:
+ return min(body_limits[func_name], max_body_lines)
+ return max_body_lines
+
+
+def _make_omitted_comment(
+ func_name: str | None,
+ omitted_count: int,
+ indent: str,
+ comment_prefix: str,
+ analysis: _SymbolAnalysis | None,
+) -> str:
+ """Build omitted comment with call information from analysis."""
+ calls_info = ""
+ if analysis and func_name:
+ for key in (
+ func_name,
+ *(k for k in analysis.calls if k.endswith(f".{func_name}")),
+ ):
+ if key in analysis.calls:
+ called = analysis.calls[key]
+ if called:
+ sorted_calls = sorted(called)[:5]
+ calls_info = "; calls: " + ", ".join(sorted_calls)
+ if len(called) > 5:
+ calls_info += f" +{len(called) - 5} more"
+ break
+ return f"{indent}{comment_prefix} [{omitted_count} lines omitted{calls_info}]"
+
+
+def _detect_indent(lines: list[str]) -> str:
+ """Detect the indentation used in a list of code lines."""
+ for line in lines:
+ if line.strip():
+ return line[: len(line) - len(line.lstrip())]
+ return " "
+
+
+def _has_syntax_issues(node: Any) -> bool:
+ """Check if AST contains ERROR or MISSING nodes."""
+ if node.type == "ERROR" or node.is_missing:
+ return True
+ for child in node.children:
+ if _has_syntax_issues(child):
+ return True
+ return False
+
+
def compress_code(
code: str,
language: str | None = None,
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py
index 0ab834528..631443855 100644
--- a/headroom/transforms/content_router.py
+++ b/headroom/transforms/content_router.py
@@ -1586,14 +1586,6 @@ class ContentRouter(Transform):
tokenizer.count_text(str(m.get("content", ""))) for m in transformed_messages
)
- # Add cache performance metrics to timing
- cache_stats = self._cache.stats
- compressor_timing["cache_hits"] = float(cache_stats["cache_hits"])
- compressor_timing["cache_skip_hits"] = float(cache_stats["cache_skip_hits"])
- compressor_timing["cache_size"] = float(cache_stats["cache_size"])
- compressor_timing["cache_skip_size"] = float(cache_stats["cache_skip_size"])
- compressor_timing["cache_avg_lookup_ns"] = cache_stats["cache_avg_lookup_ns"]
-
# Log routing summary
parts = []
if compressed_details:
diff --git a/tests/test_transforms/test_code_compressor.py b/tests/test_transforms/test_code_compressor.py
index 11132bdfa..2665a98eb 100644
--- a/tests/test_transforms/test_code_compressor.py
+++ b/tests/test_transforms/test_code_compressor.py
@@ -222,7 +222,6 @@ class TestCodeCompressorConfig:
assert config.preserve_imports is True
assert config.preserve_signatures is True
assert config.preserve_type_annotations is True
- assert config.preserve_error_handlers is True
assert config.preserve_decorators is True
assert config.docstring_mode == DocstringMode.FIRST_LINE
assert config.target_compression_rate == 0.2
@@ -500,7 +499,9 @@ class TestFallbackCompression:
# Should still return a result (fallback compression)
assert result is not None
- assert result.syntax_valid is True # Fallback guarantees validity
+ # LLMLingua fallback does NOT guarantee syntax validity
+ # If LLMLingua is unavailable, returns original (valid)
+ # If LLMLingua IS available, syntax_valid=False (cannot guarantee)
def test_fallback_preserves_structure(self, default_config):
"""Fallback compression preserves basic structure when no compressor available.
@@ -800,7 +801,6 @@ class TestTreeSitterIntegration:
def test_error_handlers_preserved(self):
"""Module-level try/except blocks are preserved."""
config = CodeCompressorConfig(
- preserve_error_handlers=True,
min_tokens_for_compression=10,
enable_ccr=False,
)