Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(code): add semantic symbol importance to CodeAwareCompressor

chopratejas9c31b22bfff0
diff --git a/headroom/transforms/code_compressor.py b/headroom/transforms/code_compressor.py
index 77fd72613..84a1bdff8 100644
--- a/headroom/transforms/code_compressor.py
+++ b/headroom/transforms/code_compressor.py
@@ -228,6 +228,9 @@ class CodeCompressorConfig:
language_hint: str | None = None
fallback_to_llmlingua: bool = True
+ # Semantic analysis (symbol importance scoring)
+ semantic_analysis: bool = True
+
# CCR integration
enable_ccr: bool = True
ccr_ttl: int = 300 # 5 minutes
@@ -273,6 +276,9 @@ class CodeCompressionResult:
# CCR
cache_key: str | None = None
+ # Semantic analysis
+ symbol_scores: dict[str, float] = field(default_factory=dict)
+
@property
def tokens_saved(self) -> int:
"""Number of tokens saved by compression."""
@@ -288,6 +294,12 @@ class CodeCompressionResult:
@property
def summary(self) -> str:
"""Human-readable summary of compression."""
+ analysis_note = ""
+ if self.symbol_scores:
+ high = sum(1 for s in self.symbol_scores.values() if s >= 0.7)
+ low = sum(1 for s in self.symbol_scores.values() if s < 0.1)
+ if high or low:
+ analysis_note = f" Semantic: {high} high-importance, {low} low-importance."
return (
f"Compressed {self.language.value} code: "
f"{self.original_tokens:,}→{self.compressed_tokens:,} tokens "
@@ -295,6 +307,7 @@ class CodeCompressionResult:
f"Kept {self.preserved_imports} imports, "
f"{self.preserved_signatures} signatures, "
f"compressed {self.compressed_bodies} bodies."
+ f"{analysis_note}"
)
@@ -391,6 +404,21 @@ def detect_language(code: str) -> tuple[CodeLanguage, float]:
return best_lang, confidence
+@dataclass
+class _SymbolAnalysis:
+ """Result of intra-file symbol importance analysis.
+
+ All dicts are keyed by qualified name (e.g., 'ClassName.method')
+ to avoid collisions between identically-named methods in different classes.
+ """
+
+ scores: dict[str, float] = field(default_factory=dict)
+ calls: dict[str, set[str]] = field(default_factory=dict)
+ ref_counts: dict[str, int] = field(default_factory=dict)
+ body_line_counts: dict[str, int] = field(default_factory=dict)
+ bare_names: dict[str, str] = field(default_factory=dict) # qname -> short_name
+
+
class CodeAwareCompressor(Transform):
"""AST-preserving compression for source code.
@@ -446,6 +474,289 @@ class CodeAwareCompressor(Transform):
startup overhead when the compressor isn't used.
"""
self.config = config or CodeCompressorConfig()
+ self._symbol_analysis: _SymbolAnalysis | None = None
+ self._body_limits: dict[str, int] | None = None
+
+ # =========================================================================
+ # Symbol importance analysis
+ # =========================================================================
+
+ def _analyze_symbol_importance(
+ self,
+ root: Any,
+ code: str,
+ language: CodeLanguage,
+ context: str = "",
+ ) -> _SymbolAnalysis:
+ """Analyze symbol importance using distribution-based scoring.
+
+ Collects raw signals (reference count, fan-out, visibility, context match,
+ convention importance) per symbol, then normalizes using min-max scaling
+ so scores are relative within the file. This adapts to any file structure:
+ utility libraries, test files, orchestrators, etc.
+
+ Returns _SymbolAnalysis with normalized scores (0.0-1.0) per symbol.
+ """
+ if not self.config.semantic_analysis:
+ return _SymbolAnalysis()
+
+ definition_types = {
+ "function_definition", # Python
+ "class_definition", # Python
+ "function_declaration", # JS, Go
+ "class_declaration", # JS, Java
+ "method_definition", # JS
+ "method_declaration", # Go, Java
+ "function_item", # Rust
+ }
+
+ # Use qualified keys (ClassName.method) to avoid collisions
+ # between methods with the same name in different classes.
+ # bare_names maps qualified keys back to short names for display/matching.
+ definitions: dict[str, Any] = {} # qualified_name -> node
+ bare_names: dict[str, str] = {} # qualified_name -> short_name
+ all_identifiers: dict[str, int] = {} # short_name -> count
+ function_calls: dict[str, set[str]] = {}
+
+ def collect_definitions(node: Any, parent_name: str = "") -> None:
+ if node.type in definition_types:
+ short_name = self._get_definition_name(node)
+ if short_name:
+ qualified = f"{parent_name}.{short_name}" if parent_name else short_name
+ definitions[qualified] = node
+ bare_names[qualified] = short_name
+ # Recurse into this definition to find nested defs
+ for child in node.children:
+ collect_definitions(child, parent_name=qualified)
+ return
+ for child in node.children:
+ collect_definitions(child, parent_name)
+
+ def collect_identifiers(node: Any) -> None:
+ if node.type in ("identifier", "property_identifier", "type_identifier"):
+ text = node.text
+ name = text.decode("utf-8") if isinstance(text, bytes) else str(text)
+ all_identifiers[name] = all_identifiers.get(name, 0) + 1
+ for child in node.children:
+ collect_identifiers(child)
+
+ def collect_calls_in_function(func_node: Any, func_qname: str) -> None:
+ func_short = bare_names[func_qname]
+ # Collect short names of other defined symbols this function references
+ defined_short_names = set(bare_names.values())
+ calls: set[str] = set()
+
+ def walk(node: Any) -> None:
+ if node.type in ("identifier", "property_identifier"):
+ text = node.text
+ name = text.decode("utf-8") if isinstance(text, bytes) else str(text)
+ if name in defined_short_names and name != func_short:
+ calls.add(name)
+ for child in node.children:
+ walk(child)
+
+ walk(func_node)
+ function_calls[func_qname] = calls
+
+ # Pass 1: Collect definitions with qualified names
+ collect_definitions(root)
+
+ if not definitions:
+ return _SymbolAnalysis()
+
+ # Pass 2: Collect all identifiers (by short name, since code uses short names)
+ collect_identifiers(root)
+
+ # Pass 3: Collect call relationships and body sizes
+ body_line_counts: dict[str, int] = {}
+ for qname, node in definitions.items():
+ collect_calls_in_function(node, qname)
+ node_text = code[node.start_byte : node.end_byte]
+ body_line_counts[qname] = max(1, len(node_text.split("\n")) - 2)
+
+ # Reference counts: use short name counts, subtract 1 per definition with that name
+ # (multiple definitions with the same short name each contribute -1)
+ short_name_def_count: dict[str, int] = {}
+ for short in bare_names.values():
+ short_name_def_count[short] = short_name_def_count.get(short, 0) + 1
+
+ ref_counts: dict[str, int] = {}
+ for qname in definitions:
+ short = bare_names[qname]
+ count = all_identifiers.get(short, 0)
+ ref_counts[qname] = max(0, count - short_name_def_count.get(short, 1))
+
+ # Raw importance signals per symbol
+ # Context matching: split into words AND check as substring for multi-word names
+ context_lower = context.lower() if context else ""
+ context_words = set(re.split(r"[\s,;:.()\[\]{}\"']+", context_lower)) if context else set()
+ context_words.discard("")
+
+ raw_signals: dict[str, float] = {}
+ for qname in definitions:
+ short = bare_names[qname]
+ refs = ref_counts.get(qname, 0)
+ fan_out = len(function_calls.get(qname, set()))
+ is_public = self._is_public_symbol(short, language)
+
+ raw = float(refs)
+ raw += 1.0 if is_public else 0.0
+ raw += fan_out * 0.5
+
+ # Convention importance (language-specific)
+ if language == CodeLanguage.PYTHON:
+ if short.startswith("__") and short.endswith("__"):
+ raw += 2.0
+ elif language == CodeLanguage.GO:
+ if short[0].isupper():
+ raw += 1.0
+
+ # Context boost: match whole symbol name against context words,
+ # or check if multi-word symbol appears as substring in context
+ if context_words:
+ name_lower = short.lower()
+ if name_lower in context_words or (
+ len(name_lower) > 3 and name_lower in context_lower
+ ):
+ raw += 3.0
+
+ raw_signals[qname] = raw
+
+ # Normalize to 0-1 using min-max scaling (distribution-based)
+ values = list(raw_signals.values())
+ min_val = min(values)
+ max_val = max(values)
+ range_val = max_val - min_val
+
+ if range_val > 0:
+ scores = {name: round((v - min_val) / range_val, 3) for name, v in raw_signals.items()}
+ else:
+ # All symbols have equal importance (test files, utility libraries)
+ scores = dict.fromkeys(raw_signals, 0.5)
+
+ return _SymbolAnalysis(
+ scores=scores,
+ calls=function_calls,
+ ref_counts=ref_counts,
+ body_line_counts=body_line_counts,
+ bare_names=bare_names,
+ )
+
+ def _get_definition_name(self, node: Any) -> str | None:
+ """Extract the name identifier from a definition AST node."""
+ for child in node.children:
+ if child.type in ("identifier", "name", "type_identifier", "property_identifier"):
+ text = child.text
+ return text.decode("utf-8") if isinstance(text, bytes) else str(text)
+ return None
+
+ def _is_public_symbol(self, name: str, language: CodeLanguage) -> bool:
+ """Heuristic for whether a symbol is public/exported."""
+ if not name:
+ return False
+ if language == CodeLanguage.GO:
+ return name[0].isupper()
+ return not name.startswith("_")
+
+ def _allocate_body_budget(self, analysis: _SymbolAnalysis, code: str) -> dict[str, int]:
+ """Allocate body line budget across functions using target_compression_rate.
+
+ Instead of hardcoded tiers, distributes a total body line budget
+ proportionally to each symbol's importance score. The budget is
+ derived from target_compression_rate and accounts for fixed overhead
+ (imports, signatures) that are always preserved.
+
+ Returns dict mapping symbol name to max body lines to keep.
+ """
+ if not analysis.scores or not analysis.body_line_counts:
+ return {}
+
+ scores = analysis.scores
+ body_sizes = analysis.body_line_counts
+ target_rate = self.config.target_compression_rate
+
+ total_lines = len(code.strip().split("\n"))
+ total_body_lines = sum(body_sizes.values())
+ fixed_lines = max(0, total_lines - total_body_lines)
+
+ # Body budget = target total output - fixed overhead
+ target_total = total_lines * target_rate
+ body_budget = max(0.0, target_total - fixed_lines)
+
+ if total_body_lines == 0:
+ return {}
+
+ # Score floor: even lowest-scored symbols get a minimal allocation
+ # so the LLM sees they exist (vs just `pass`)
+ score_floor = 0.05
+
+ # Weight = score * body_size (larger important functions get more budget)
+ weights: dict[str, float] = {}
+ for name in scores:
+ score = max(scores.get(name, 0.5), score_floor)
+ size = body_sizes.get(name, 0)
+ weights[name] = score * size
+
+ total_weight = sum(weights.values())
+
+ if total_weight == 0:
+ # Edge case: distribute evenly
+ per_func = max(0, int(body_budget / max(len(scores), 1)))
+ return {name: min(per_func, body_sizes.get(name, 0)) for name in scores}
+
+ limits: dict[str, int] = {}
+ for qname in scores:
+ allocation = body_budget * weights[qname] / total_weight
+ max_lines = body_sizes.get(qname, 0)
+ limit = min(int(round(allocation)), max_lines)
+ limits[qname] = limit
+ # Also store by short name so _get_body_limit can find it.
+ # On collision, keep the higher limit (more generous).
+ short = analysis.bare_names.get(qname, qname)
+ if short not in limits or limit > limits[short]:
+ limits[short] = limit
+
+ return limits
+
+ def _get_body_limit(self, func_name: str | None) -> int:
+ """Look up the allocated body line limit for a function.
+
+ Falls back to max_body_lines if no budget allocation was computed.
+ max_body_lines always acts as a hard cap — budget allocation can give
+ a function FEWER lines but never MORE than max_body_lines.
+ """
+ if self._body_limits and func_name and func_name in self._body_limits:
+ return min(self._body_limits[func_name], self.config.max_body_lines)
+ return self.config.max_body_lines
+
+ def _make_omitted_comment(
+ self,
+ func_name: str | None,
+ omitted_count: int,
+ indent: str,
+ comment_prefix: str = "#",
+ ) -> str:
+ """Build omitted comment with call information from analysis."""
+ calls_info = ""
+ if self._symbol_analysis and func_name:
+ # Try both short name and as-is (qualified name) for lookup
+ for key in (
+ func_name,
+ *(k for k in self._symbol_analysis.calls if k.endswith(f".{func_name}")),
+ ):
+ if key in self._symbol_analysis.calls:
+ called = self._symbol_analysis.calls[key]
+ if called:
+ sorted_calls = sorted(called)[:5]
+ calls_info = "; calls: " + ", ".join(sorted_calls)
+ if len(called) > 5:
+ calls_info += f" +{len(called) - 5} more"
+ break
+ return f"{indent}{comment_prefix} [{omitted_count} lines omitted{calls_info}]"
+
+ # =========================================================================
+ # Core compression
+ # =========================================================================
def compress(
self,
@@ -532,7 +843,9 @@ class CodeAwareCompressor(Transform):
# Parse and compress
try:
- compressed, structure = self._compress_with_ast(code, detected_lang, context)
+ compressed, structure, symbol_scores = self._compress_with_ast(
+ code, detected_lang, context
+ )
compressed_tokens = len(compressed.split())
# Verify syntax validity
@@ -590,6 +903,7 @@ class CodeAwareCompressor(Transform):
compressed_bodies=len(structure.function_bodies),
syntax_valid=syntax_valid,
cache_key=cache_key,
+ symbol_scores=symbol_scores,
)
except Exception as e:
@@ -612,8 +926,8 @@ class CodeAwareCompressor(Transform):
code: str,
language: CodeLanguage,
context: str,
- ) -> tuple[str, CodeStructure]:
- """Compress code using AST parsing.
+ ) -> tuple[str, CodeStructure, dict[str, float]]:
+ """Compress code using AST parsing with symbol importance analysis.
Args:
code: Source code.
@@ -621,7 +935,7 @@ class CodeAwareCompressor(Transform):
context: User context for relevance.
Returns:
- Tuple of (compressed code, extracted structure).
+ Tuple of (compressed code, extracted structure, symbol scores).
"""
# Get parser for language
parser = _get_parser(language.value)
@@ -630,6 +944,11 @@ class CodeAwareCompressor(Transform):
tree = parser.parse(bytes(code, "utf-8"))
root = tree.root_node
+ # Analyze symbol importance and allocate compression budget
+ analysis = self._analyze_symbol_importance(root, code, language, context)
+ self._symbol_analysis = analysis
+ self._body_limits = self._allocate_body_budget(analysis, code)
+
# Extract structure based on language
if language == CodeLanguage.PYTHON:
structure = self._extract_python_structure(root, code)
@@ -647,7 +966,18 @@ class CodeAwareCompressor(Transform):
# Assemble compressed code
compressed = self._assemble_compressed(structure, language)
- return compressed, structure
+ # Expose scores with short names for the public API
+ symbol_scores: dict[str, float] = {}
+ if analysis.scores:
+ for qname, score in analysis.scores.items():
+ short = analysis.bare_names.get(qname, qname)
+ # On collision, keep the higher score (more generous)
+ if short not in symbol_scores or score > symbol_scores[short]:
+ symbol_scores[short] = score
+ self._symbol_analysis = None
+ self._body_limits = None
+
+ return compressed, structure, symbol_scores
def _extract_python_structure(self, root: Any, code: str) -> CodeStructure:
"""Extract structure from Python AST."""
@@ -706,12 +1036,16 @@ class CodeAwareCompressor(Transform):
return structure
def _extract_definition(self, node: Any, code: str, lines: list[str]) -> str:
- """Extract a function/class definition with compressed body."""
+ """Extract a function/class definition with budget-aware compressed body."""
node_text = self._get_node_text(node, code)
node_lines = node_text.split("\n")
- if len(node_lines) <= self.config.max_body_lines + 2:
- # Small enough, keep as is
+ # Look up allocated body line limit for this symbol
+ func_name = self._get_definition_name(node)
+ body_limit = self._get_body_limit(func_name)
+
+ # Check if small enough to keep as-is
+ if len(node_lines) <= body_limit + 2:
return node_text
# Find signature (first line(s) until colon)
@@ -732,7 +1066,7 @@ class CodeAwareCompressor(Transform):
if not found_colon:
# Couldn't parse signature, return truncated
- return "\n".join(node_lines[: self.config.max_body_lines]) + "\n # ..."
+ return "\n".join(node_lines[: max(body_limit, 1)]) + "\n # ..."
signature = "\n".join(signature_lines)
@@ -780,15 +1114,20 @@ class CodeAwareCompressor(Transform):
body_lines = node_lines[body_start:]
total_body = len(body_lines)
- # Determine indentation
+ # Keep body lines based on budget-allocated limit
+ keep_lines = min(body_limit, len(body_lines))
+
+ # Determine indentation from the first omitted line, not the first body line.
+ # This handles nested structures correctly: if the cut falls inside a nested
+ # method definition, the pass/omitted comment use the inner indent level.
indent = " "
if body_lines:
- first_non_empty = next((line for line in body_lines if line.strip()), "")
+ omitted_part = body_lines[keep_lines:] if keep_lines < len(body_lines) else body_lines
+ first_non_empty = next((line for line in omitted_part if line.strip()), "")
+ if not first_non_empty:
+ first_non_empty = next((line for line in body_lines if line.strip()), "")
if first_non_empty:
indent = first_non_empty[: len(first_non_empty) - len(first_non_empty.lstrip())]
-
- # Keep first few lines of body and add placeholder
- keep_lines = min(self.config.max_body_lines, len(body_lines))
compressed_body = body_lines[:keep_lines]
result_parts = [signature]
@@ -803,8 +1142,7 @@ class CodeAwareCompressor(Transform):
if total_body > keep_lines:
omitted = total_body - keep_lines
- # Simple, honest marker - no retrieval hints (causes hallucination)
- result_parts.append(f"{indent}# [{omitted} lines omitted]")
+ result_parts.append(self._make_omitted_comment(func_name, omitted, indent))
result_parts.append(f"{indent}pass")
return "\n".join(result_parts)
@@ -853,11 +1191,14 @@ class CodeAwareCompressor(Transform):
return structure
def _compress_js_function(self, node: Any, code: str, lines: list[str]) -> str:
- """Compress a JavaScript function."""
+ """Compress a JavaScript function with budget-aware retention."""
node_text = self._get_node_text(node, code)
node_lines = node_text.split("\n")
- if len(node_lines) <= self.config.max_body_lines + 2:
+ func_name = self._get_definition_name(node)
+ body_limit = self._get_body_limit(func_name)
+
+ if len(node_lines) <= body_limit + 2:
return node_text
# Find opening brace
@@ -874,11 +1215,13 @@ class CodeAwareCompressor(Transform):
body_lines = node_lines[body_start:-1] # Exclude closing brace
total_body = len(body_lines)
- keep_lines = min(self.config.max_body_lines, total_body)
+ keep_lines = min(body_limit, total_body)
result = signature_lines + body_lines[:keep_lines]
if total_body > keep_lines:
- result.append(f" // ... ({total_body - keep_lines} lines compressed)")
+ result.append(
+ self._make_omitted_comment(func_name, total_body - keep_lines, " ", "//")
+ )
result.append(node_lines[-1]) # Closing brace
return "\n".join(result)
@@ -920,11 +1263,14 @@ class CodeAwareCompressor(Transform):
return structure
def _compress_go_function(self, node: Any, code: str, lines: list[str]) -> str:
- """Compress a Go function."""
+ """Compress a Go function with budget-aware retention."""
node_text = self._get_node_text(node, code)
node_lines = node_text.split("\n")
- if len(node_lines) <= self.config.max_body_lines + 2:
+ func_name = self._get_definition_name(node)
+ body_limit = self._get_body_limit(func_name)
+
+ if len(node_lines) <= body_limit + 2:
return node_text
# Find opening brace
@@ -938,11 +1284,13 @@ class CodeAwareCompressor(Transform):
body_lines = node_lines[body_start:-1]
total_body = len(body_lines)
- keep_lines = min(self.config.max_body_lines, total_body)
+ keep_lines = min(body_limit, total_body)
result = signature_lines + body_lines[:keep_lines]
if total_body > keep_lines:
- result.append(f"\t// ... ({total_body - keep_lines} lines compressed)")
+ result.append(
+ self._make_omitted_comment(func_name, total_body - keep_lines, "\t", "//")
+ )
result.append(node_lines[-1])
return "\n".join(result)
@@ -978,11 +1326,14 @@ class CodeAwareCompressor(Transform):
return structure
def _compress_rust_function(self, node: Any, code: str, lines: list[str]) -> str:
- """Compress a Rust function."""
+ """Compress a Rust function with budget-aware retention."""
node_text = self._get_node_text(node, code)
node_lines = node_text.split("\n")
- if len(node_lines) <= self.config.max_body_lines + 2:
+ func_name = self._get_definition_name(node)
+ body_limit = self._get_body_limit(func_name)
+
+ if len(node_lines) <= body_limit + 2:
return node_text
# Find opening brace
@@ -996,11 +1347,13 @@ class CodeAwareCompressor(Transform):
body_lines = node_lines[body_start:-1]
total_body = len(body_lines)
- keep_lines = min(self.config.max_body_lines, total_body)
+ keep_lines = min(body_limit, total_body)
result = signature_lines + body_lines[:keep_lines]
if total_body > keep_lines:
- result.append(f" // ... ({total_body - keep_lines} lines compressed)")
+ result.append(
+ self._make_omitted_comment(func_name, total_body - keep_lines, " ", "//")
+ )
result.append(node_lines[-1])
return "\n".join(result)
diff --git a/tests/test_transforms/test_code_compressor.py b/tests/test_transforms/test_code_compressor.py
index 7a9dd0302..11132bdfa 100644
--- a/tests/test_transforms/test_code_compressor.py
+++ b/tests/test_transforms/test_code_compressor.py
@@ -952,3 +952,329 @@ def example():
# Docstring should be removed when REMOVE mode is active
assert "This docstring should be removed" not in result.compressed
+
+
+# =============================================================================
+# TestSemanticSymbolImportance
+# =============================================================================
+
+
+def _payment_processing_code() -> str:
+ """Python code with varying symbol importance for testing."""
+ return '''
+import os
+from typing import List, Optional
+
+def process_payment(order, config):
+ """Process a payment through the pipeline."""
+ validated = validate_order(order)
+ if not validated.is_valid:
+ return PaymentResult(status='failed')
+ charge = charge_customer(order.customer, order.total)
+ receipt = generate_receipt(charge)
+ send_confirmation(order.customer.email, receipt)
+ update_inventory(order.items)
+ log_transaction(charge.transaction_id)
+ notify_warehouse(order)
+ return PaymentResult(status='success', receipt=receipt)
+
+def validate_order(order):
+ """Validate an order before processing."""
+ if not order.items:
+ return ValidationResult(False, ['No items'])
+ total = sum(item.price for item in order.items)
+ if total <= 0:
+ return ValidationResult(False, ['Invalid total'])
+ if not order.customer:
+ return ValidationResult(False, ['No customer'])
+ return ValidationResult(True, [])
+
+def charge_customer(customer, amount):
+ """Charge the customer."""
+ gateway = get_payment_gateway()
+ response = gateway.charge(customer.card, amount)
+ if not response.success:
+ raise PaymentError(response.error)
+ return response
+
+def generate_receipt(charge):
+ """Generate a receipt for the charge."""
+ template = load_template('receipt')
+ return template.render(charge=charge)
+
+def _format_log_entry(entry):
+ """Format a log entry for internal use. Never called."""
+ timestamp = entry.get('ts', '')
+ level = entry.get('level', 'INFO')
+ message = entry.get('msg', '')
+ source = entry.get('source', 'unknown')
+ formatted = f'[{timestamp}] {level}: {message} ({source})'
+ return formatted.strip()
+
+def _dead_helper():
+ """Never called anywhere in this file."""
+ x = 1
+ y = 2
+ z = 3
+ result = x + y + z
+ for i in range(100):
+ result += i
+ return result
+'''
+
+
[email protected](not TREE_SITTER_INSTALLED, reason="tree-sitter-languages not installed")
+class TestSemanticSymbolImportance:
+ """Tests for semantic symbol importance analysis and variable compression."""
+
+ def _make_compressor(self, **overrides):
+ defaults = {
+ "min_tokens_for_compression": 10,
+ "max_body_lines": 3,
+ "enable_ccr": False,
+ "semantic_analysis": True,
+ }
+ defaults.update(overrides)
+ return CodeAwareCompressor(CodeCompressorConfig(**defaults))
+
+ def test_symbol_scores_populated(self):
+ """Compression result includes symbol importance scores."""
+ compressor = self._make_compressor()
+ result = compressor.compress(_payment_processing_code(), language="python")
+
+ assert result.symbol_scores
+ assert "process_payment" in result.symbol_scores
+ assert "validate_order" in result.symbol_scores
+ assert "_dead_helper" in result.symbol_scores
+
+ def test_called_functions_score_higher_than_dead_code(self):
+ """Functions called by others score higher than unused functions."""
+ compressor = self._make_compressor()
+ result = compressor.compress(_payment_processing_code(), language="python")
+
+ # validate_order is called by process_payment — should score higher
+ assert result.symbol_scores["validate_order"] > result.symbol_scores["_dead_helper"]
+ assert result.symbol_scores["charge_customer"] > result.symbol_scores["_dead_helper"]
+
+ def test_public_symbols_score_higher_than_private(self):
+ """Public functions (no leading _) score higher than private ones."""
+ compressor = self._make_compressor()
+ code = '''
+def public_func():
+ """A public function."""
+ x = 1
+ y = 2
+ z = 3
+ result = x + y + z
+ for i in range(10):
+ result += i
+ return result
+
+def _private_func():
+ """A private function."""
+ x = 1
+ y = 2
+ z = 3
+ result = x + y + z
+ for i in range(10):
+ result += i
+ return result
+'''
+ result = compressor.compress(code, language="python")
+
+ assert result.symbol_scores["public_func"] > result.symbol_scores["_private_func"]
+
+ def test_dead_code_compressed_to_signature_only(self):
+ """Functions with score < 0.1 are compressed to signature + docstring only."""
+ compressor = self._make_compressor()
+ result = compressor.compress(_payment_processing_code(), language="python")
+
+ # _dead_helper has 0 references, private → score 0.0
+ assert result.symbol_scores["_dead_helper"] < 0.1
+
+ # Body should be fully omitted
+ assert "_dead_helper" in result.compressed
+ # Should NOT contain body content
+ assert "range(100)" not in result.compressed
+
+ def test_referenced_functions_keep_more_body(self):
+ """Higher-scored functions get more body lines from the budget."""
+ # Use a generous target rate so there IS budget to distribute
+ compressor = self._make_compressor(target_compression_rate=0.7)
+ result = compressor.compress(_payment_processing_code(), language="python")
+
+ compressed = result.compressed
+ # With 70% target, high-scoring functions should retain body
+ # while low-scoring ones get less. validate_order is referenced
+ # and public (high score) so should keep some body.
+ # _dead_helper has lowest score so should get least body.
+ # Count body lines per function as a proxy for retention
+ lines = compressed.split("\n")
+ in_validate = False
+ in_dead = False
+ validate_body = 0
+ dead_body = 0
+ for line in lines:
+ if "def validate_order" in line:
+ in_validate = True
+ in_dead = False
+ continue
+ elif "def _dead_helper" in line:
+ in_dead = True
+ in_validate = False
+ continue
+ elif line.startswith("def ") or (line.startswith("class ") and ":" in line):
+ in_validate = False
+ in_dead = False
+ continue
+ if in_validate and line.strip() and not line.strip().startswith('"""'):
+ validate_body += 1
+ if in_dead and line.strip() and not line.strip().startswith('"""'):
+ dead_body += 1
+
+ assert validate_body >= dead_body
+
+ def test_omitted_comment_includes_calls(self):
+ """Omitted comment includes call information when available."""
+ compressor = self._make_compressor()
+ result = compressor.compress(_payment_processing_code(), language="python")
+
+ # process_payment calls validate_order, charge_customer, generate_receipt
+ # These should appear in the omitted comment
+ compressed = result.compressed
+ if "lines omitted" in compressed:
+ # Find omitted comments and check for calls info
+ for line in compressed.split("\n"):
+ if "process_payment" not in line and "lines omitted" in line:
+ continue
+ if "lines omitted; calls:" in line:
+ assert "validate_order" in line or "charge_customer" in line
+ break
+
+ def test_semantic_analysis_disabled(self):
+ """When semantic_analysis=False, all functions get uniform compression."""
+ compressor_with = self._make_compressor(semantic_analysis=True)
+ compressor_without = self._make_compressor(semantic_analysis=False)
+
+ code = _payment_processing_code()
+ result_with = compressor_with.compress(code, language="python")
+ result_without = compressor_without.compress(code, language="python")
+
+ # Without semantic analysis, no symbol scores
+ assert result_without.symbol_scores == {}
+
+ # With semantic analysis, dead code is compressed more aggressively
+ # _dead_helper body should NOT appear with semantic analysis
+ assert "range(100)" not in result_with.compressed
+ # But with uniform compression (no semantic), body lines ARE kept
+ assert "x = 1" in result_without.compressed
+
+ def test_summary_includes_semantic_info(self):
+ """Summary includes semantic analysis information."""
+ compressor = self._make_compressor()
+ result = compressor.compress(_payment_processing_code(), language="python")
+
+ summary = result.summary
+ if result.symbol_scores:
+ low_count = sum(1 for s in result.symbol_scores.values() if s < 0.1)
+ if low_count > 0:
+ assert "low-importance" in summary
+
+ def test_dunder_methods_get_boost(self):
+ """Dunder methods (__init__, etc.) get importance boost."""
+ compressor = self._make_compressor()
+ code = '''
+class MyClass:
+ """A class."""
+ def __init__(self, value):
+ """Initialize."""
+ self.value = value
+ self.processed = False
+ self.results = []
+ self.cache = {}
+ self.errors = []
+ for i in range(10):
+ self.results.append(i)
+
+ def _setup_cache(self):
+ """Internal setup."""
+ x = 1
+ y = 2
+ z = 3
+ result = x + y + z
+ for i in range(10):
+ result += i
+ return result
+'''
+ result = compressor.compress(code, language="python")
+
+ # __init__ should score higher than _setup_cache
+ if "__init__" in result.symbol_scores and "_setup_cache" in result.symbol_scores:
+ assert result.symbol_scores["__init__"] > result.symbol_scores["_setup_cache"]
+
+ def test_javascript_importance(self):
+ """Symbol importance works for JavaScript code."""
+ compressor = self._make_compressor()
+ code = """
+import { db } from './database';
+
+function processUser(userId) {
+ const user = fetchUser(userId);
+ const profile = buildProfile(user);
+ sendNotification(user.email, profile);
+ logAction('process', userId);
+ updateMetrics('user_processed');
+ return { user, profile };
+}
+
+function fetchUser(id) {
+ const result = db.query('SELECT * FROM users WHERE id = ?', [id]);
+ if (!result) {
+ throw new Error('User not found');
+ }
+ return result;
+}
+
+function buildProfile(user) {
+ const prefs = loadPreferences(user.id);
+ return { ...user, preferences: prefs };
+}
+
+function _internalDebug(msg) {
+ const ts = Date.now();
+ const formatted = `[${ts}] DEBUG: ${msg}`;
+ console.log(formatted);
+ return formatted;
+}
+"""
+ result = compressor.compress(code, language="javascript")
+
+ assert result.symbol_scores
+ # fetchUser is called by processUser — should score higher than _internalDebug
+ if "fetchUser" in result.symbol_scores and "_internalDebug" in result.symbol_scores:
+ assert result.symbol_scores["fetchUser"] > result.symbol_scores["_internalDebug"]
+
+ def test_syntax_still_valid_with_importance(self):
+ """Compressed output with importance remains syntactically valid."""
+ compressor = self._make_compressor()
+ result = compressor.compress(_payment_processing_code(), language="python")
+
+ assert result.syntax_valid is True
+
+ # Should be parseable as Python
+ try:
+ compile(result.compressed, "<test>", "exec")
+ except SyntaxError:
+ pytest.fail("Semantic compression produced invalid Python syntax")
+
+ def test_empty_code_no_crash(self):
+ """Importance analysis handles empty code gracefully."""
+ compressor = self._make_compressor()
+ result = compressor.compress("", language="python")
+
+ assert result.symbol_scores == {}
+
+ def test_config_default_semantic_analysis_enabled(self):
+ """semantic_analysis is True by default in config."""
+ config = CodeCompressorConfig()
+ assert config.semantic_analysis is True