Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
Add opt-in text compression utilities for coding tasks
chopratejaseac2890bce04
diff --git a/README.md b/README.mdindex ef2c28b82..dfb30d7e2 100644--- a/README.md+++ b/README.md@@ -39,9 +39,10 @@ Headroom is a **smart compression layer** that sits between your app and LLM pro| Transform | What It Does | Savings ||-----------|--------------|---------|-| **SmartCrusher** | Compresses tool outputs statistically (keeps errors, anomalies, relevant items) | 70-90% |+| **SmartCrusher** | Compresses JSON tool outputs statistically (keeps errors, anomalies, relevant items) | 70-90% || **CacheAligner** | Stabilizes prefixes so provider caching works | Up to 10x || **RollingWindow** | Manages context within limits without breaking tool calls | Prevents failures |+| **Text Utilities** | Opt-in compression for search results, build logs, plain text | 50-90% |**Zero accuracy loss** - we keep what matters: errors, anomalies, relevant items.@@ -376,6 +377,112 @@ except HeadroomError as e:---+## Text Compression Utilities (Opt-In)++For coding tasks, Headroom provides **standalone text compression utilities** that applications can use explicitly. These are **opt-in** - they're not applied automatically, giving you full control over when and how to compress text content.++> **Design Philosophy**: SmartCrusher compresses JSON automatically because it's structure-preserving and safe. Text compression is lossy and context-dependent, so applications should decide when to use it.++### Available Utilities++| Utility | Input Type | Use Case |+|---------|------------|----------|+| `SearchCompressor` | grep/ripgrep output | Search results with `file:line:content` format |+| `LogCompressor` | Build/test logs | pytest, npm, cargo, make output |+| `TextCompressor` | Generic text | Any plain text with anchor preservation |+| `detect_content_type` | Any content | Detect content type for routing decisions |++### Example: Compressing Search Results++```python+from headroom.transforms import SearchCompressor++# Your grep/ripgrep output (could be 1000s of lines)+search_results = """+src/utils.py:42:def process_data(items):+src/utils.py:43: \"\"\"Process items.\"\"\"+src/models.py:15:class DataProcessor:+src/models.py:89: def process(self, items):+... hundreds more matches ...+"""++# Explicitly compress when you decide it's appropriate+compressor = SearchCompressor()+result = compressor.compress(search_results, context="find process")++print(f"Compressed {result.original_match_count} matches to {result.compressed_match_count}")+print(result.compressed)+```++### Example: Compressing Build Logs++```python+from headroom.transforms import LogCompressor++# pytest output with 1000s of lines+build_output = """+===== test session starts =====+collected 500 items+tests/test_foo.py::test_1 PASSED+... hundreds of passed tests ...+tests/test_bar.py::test_fail FAILED+AssertionError: expected 5, got 3+===== 1 failed, 499 passed =====+"""++# Compress logs, preserving errors and stack traces+compressor = LogCompressor()+result = compressor.compress(build_output)++# Errors, stack traces, and summary are preserved+print(result.compressed)+print(f"Compression ratio: {result.compression_ratio:.1%}")+```++### Example: Content Type Detection++```python+from headroom.transforms import detect_content_type, ContentType++content = "src/main.py:42:def process():"++detection = detect_content_type(content)+if detection.content_type == ContentType.SEARCH_RESULTS:+ # Route to SearchCompressor+ pass+elif detection.content_type == ContentType.BUILD_OUTPUT:+ # Route to LogCompressor+ pass+```++### Integration Pattern++```python+from headroom.transforms import (+ detect_content_type, ContentType,+ SearchCompressor, LogCompressor, TextCompressor+)++def compress_tool_output(content: str, context: str = "") -> str:+ """Application-level compression with explicit control."""+ detection = detect_content_type(content)++ if detection.content_type == ContentType.SEARCH_RESULTS:+ result = SearchCompressor().compress(content, context)+ return result.compressed+ elif detection.content_type == ContentType.BUILD_OUTPUT:+ result = LogCompressor().compress(content)+ return result.compressed+ elif detection.content_type == ContentType.PLAIN_TEXT:+ result = TextCompressor().compress(content, context)+ return result.compressed+ else:+ # JSON or other - let SmartCrusher handle it automatically+ return content+```++---+## Metrics & Monitoring### Prometheus Metrics (Proxy)diff --git a/headroom/transforms/__init__.py b/headroom/transforms/__init__.pyindex 24f624e01..98f4bb586 100644--- a/headroom/transforms/__init__.py+++ b/headroom/transforms/__init__.py@@ -2,17 +2,41 @@from .base import Transformfrom .cache_aligner import CacheAligner+from .content_detector import ContentType, DetectionResult, detect_content_type+from .log_compressor import LogCompressor, LogCompressorConfig, LogCompressionResultfrom .pipeline import TransformPipelinefrom .rolling_window import RollingWindow+from .search_compressor import (+ SearchCompressor,+ SearchCompressorConfig,+ SearchCompressionResult,+)from .smart_crusher import SmartCrusher, SmartCrusherConfig+from .text_compressor import TextCompressor, TextCompressorConfig, TextCompressionResultfrom .tool_crusher import ToolCrusher__all__ = [+ # Base"Transform",+ "TransformPipeline",+ # JSON compression"ToolCrusher","SmartCrusher","SmartCrusherConfig",+ # Text compression (coding tasks)+ "ContentType",+ "DetectionResult",+ "detect_content_type",+ "SearchCompressor",+ "SearchCompressorConfig",+ "SearchCompressionResult",+ "LogCompressor",+ "LogCompressorConfig",+ "LogCompressionResult",+ "TextCompressor",+ "TextCompressorConfig",+ "TextCompressionResult",+ # Other transforms"CacheAligner","RollingWindow",- "TransformPipeline",]diff --git a/headroom/transforms/content_detector.py b/headroom/transforms/content_detector.pynew file mode 100644index 000000000..0779247be--- /dev/null+++ b/headroom/transforms/content_detector.py@@ -0,0 +1,338 @@+"""Content type detection for multi-format compression.++This module detects the type of tool output content to route it to the+appropriate compressor. SmartCrusher handles JSON arrays, but coding tasks+produce many other formats that need specialized handling.++Supported content types:+- JSON_ARRAY: Structured JSON data (existing SmartCrusher)+- SOURCE_CODE: Python, JavaScript, TypeScript, Go, etc.+- SEARCH_RESULTS: grep/ripgrep output (file:line:content)+- BUILD_OUTPUT: Compiler, test, lint logs+- GIT_DIFF: Unified diff format+- PLAIN_TEXT: Generic text (fallback)+"""++from __future__ import annotations++import json+import re+from dataclasses import dataclass+from enum import Enum+++class ContentType(Enum):+ """Types of content that can be compressed."""++ JSON_ARRAY = "json_array" # Existing SmartCrusher handles this+ SOURCE_CODE = "source_code" # Python, JS, TS, Go, Rust, etc.+ SEARCH_RESULTS = "search" # grep/ripgrep output+ BUILD_OUTPUT = "build" # Compiler, test, lint logs+ GIT_DIFF = "diff" # Unified diff format+ PLAIN_TEXT = "text" # Fallback+++@dataclass+class DetectionResult:+ """Result of content type detection."""++ content_type: ContentType+ confidence: float # 0.0 to 1.0+ metadata: dict # Type-specific metadata (e.g., language for code)+++# Patterns for detection+_SEARCH_RESULT_PATTERN = re.compile(+ r"^[^\s:]+:\d+:" # file:line: format (grep -n style)+)++_DIFF_HEADER_PATTERN = re.compile(+ r"^(diff --git|--- a/|@@\s+-\d+,\d+\s+\+\d+,\d+\s+@@)"+)++_DIFF_CHANGE_PATTERN = re.compile(r"^[+-][^+-]")++# Code patterns by language+_CODE_PATTERNS = {+ "python": [+ re.compile(r"^\s*(def|class|import|from|async def)\s+\w+"),+ re.compile(r"^\s*@\w+"), # decorators+ re.compile(r'^\s*"""'), # docstrings+ re.compile(r"^\s*if __name__\s*=="),+ ],+ "javascript": [+ re.compile(r"^\s*(function|const|let|var|class|import|export)\s+"),+ re.compile(r"^\s*(async\s+function|=>\s*\{)"),+ re.compile(r"^\s*module\.exports"),+ ],+ "typescript": [+ re.compile(r"^\s*(interface|type|enum|namespace)\s+\w+"),+ re.compile(r":\s*(string|number|boolean|any|void)\b"),+ ],+ "go": [+ re.compile(r"^\s*(func|type|package|import)\s+"),+ re.compile(r"^\s*func\s+\([^)]+\)\s+\w+"), # method+ ],+ "rust": [+ re.compile(r"^\s*(fn|struct|enum|impl|mod|use|pub)\s+"),+ re.compile(r"^\s*#\["), # attributes+ ],+ "java": [+ re.compile(r"^\s*(public|private|protected)\s+(class|interface|enum)"),+ re.compile(r"^\s*@\w+"), # annotations+ re.compile(r"^\s*package\s+[\w.]+;"),+ ],+}++# Log/build output patterns+_LOG_PATTERNS = [+ re.compile(r"\b(ERROR|FAIL|FAILED|FATAL|CRITICAL)\b", re.IGNORECASE),+ re.compile(r"\b(WARN|WARNING)\b", re.IGNORECASE),+ re.compile(r"\b(INFO|DEBUG|TRACE)\b", re.IGNORECASE),+ re.compile(r"^\s*\d{4}-\d{2}-\d{2}"), # timestamp+ re.compile(r"^\s*\[\d{2}:\d{2}:\d{2}\]"), # time format+ re.compile(r"^={3,}|^-{3,}"), # separators+ re.compile(r"^\s*PASSED|^\s*FAILED|^\s*SKIPPED"), # test results+ re.compile(r"^npm ERR!|^yarn error|^cargo error"), # build tools+ re.compile(r"Traceback \(most recent call last\)"), # Python traceback+ re.compile(r"^\s*at\s+[\w.$]+\("), # JS/Java stack trace+]+++def detect_content_type(content: str) -> DetectionResult:+ """Detect the type of content for appropriate compression.++ Args:+ content: The content to analyze.++ Returns:+ DetectionResult with type, confidence, and metadata.++ Examples:+ >>> result = detect_content_type('[{"id": 1}, {"id": 2}]')+ >>> result.content_type+ ContentType.JSON_ARRAY++ >>> result = detect_content_type('src/main.py:42:def process():')+ >>> result.content_type+ ContentType.SEARCH_RESULTS+ """+ if not content or not content.strip():+ return DetectionResult(ContentType.PLAIN_TEXT, 0.0, {})++ # 1. Try JSON first (highest priority for SmartCrusher compatibility)+ json_result = _try_detect_json(content)+ if json_result:+ return json_result++ # 2. Check for diff (very distinctive patterns)+ diff_result = _try_detect_diff(content)+ if diff_result and diff_result.confidence >= 0.7:+ return diff_result++ # 3. Check for search results (file:line: format)+ search_result = _try_detect_search(content)+ if search_result and search_result.confidence >= 0.6:+ return search_result++ # 4. Check for build/log output+ log_result = _try_detect_log(content)+ if log_result and log_result.confidence >= 0.5:+ return log_result++ # 5. Check for source code+ code_result = _try_detect_code(content)+ if code_result and code_result.confidence >= 0.5:+ return code_result++ # 6. Fallback to plain text+ return DetectionResult(ContentType.PLAIN_TEXT, 0.5, {})+++def _try_detect_json(content: str) -> DetectionResult | None:+ """Try to detect JSON array content."""+ content = content.strip()++ # Quick check: must start with [ for array+ if not content.startswith("["):+ return None++ try:+ parsed = json.loads(content)+ if isinstance(parsed, list):+ # Check if it's a list of dicts (SmartCrusher compatible)+ if parsed and all(isinstance(item, dict) for item in parsed):+ return DetectionResult(+ ContentType.JSON_ARRAY,+ 1.0,+ {"item_count": len(parsed), "is_dict_array": True},+ )+ # It's a list but not of dicts+ return DetectionResult(+ ContentType.JSON_ARRAY,+ 0.8,+ {"item_count": len(parsed), "is_dict_array": False},+ )+ except json.JSONDecodeError:+ pass++ return None+++def _try_detect_diff(content: str) -> DetectionResult | None:+ """Try to detect git diff format."""+ lines = content.split("\n")[:50] # Check first 50 lines++ header_matches = 0+ change_matches = 0++ for line in lines:+ if _DIFF_HEADER_PATTERN.match(line):+ header_matches += 1+ if _DIFF_CHANGE_PATTERN.match(line):+ change_matches += 1++ if header_matches == 0:+ return None++ # High confidence if we see diff headers+ confidence = min(1.0, 0.5 + (header_matches * 0.2) + (change_matches * 0.05))++ return DetectionResult(+ ContentType.GIT_DIFF,+ confidence,+ {"header_matches": header_matches, "change_lines": change_matches},+ )+++def _try_detect_search(content: str) -> DetectionResult | None:+ """Try to detect grep/ripgrep search results."""+ lines = content.split("\n")[:100] # Check first 100 lines+ if not lines:+ return None++ matching_lines = 0+ for line in lines:+ if line.strip() and _SEARCH_RESULT_PATTERN.match(line):+ matching_lines += 1++ if matching_lines == 0:+ return None++ # Calculate confidence based on proportion of matching lines+ non_empty_lines = sum(1 for line in lines if line.strip())+ if non_empty_lines == 0:+ return None++ ratio = matching_lines / non_empty_lines++ # Need at least 30% of lines to match the pattern+ if ratio < 0.3:+ return None++ confidence = min(1.0, 0.4 + (ratio * 0.6))++ return DetectionResult(+ ContentType.SEARCH_RESULTS,+ confidence,+ {"matching_lines": matching_lines, "total_lines": non_empty_lines},+ )+++def _try_detect_log(content: str) -> DetectionResult | None:+ """Try to detect build/log output."""+ lines = content.split("\n")[:200] # Check first 200 lines+ if not lines:+ return None++ pattern_matches = 0+ error_matches = 0++ for line in lines:+ for i, pattern in enumerate(_LOG_PATTERNS):+ if pattern.search(line):+ pattern_matches += 1+ if i < 2: # ERROR or WARN patterns+ error_matches += 1+ break # One pattern per line is enough++ if pattern_matches == 0:+ return None++ non_empty_lines = sum(1 for line in lines if line.strip())+ if non_empty_lines == 0:+ return None++ ratio = pattern_matches / non_empty_lines++ # Need at least 10% of lines to match log patterns+ if ratio < 0.1:+ return None++ confidence = min(1.0, 0.3 + (ratio * 0.5) + (error_matches * 0.05))++ return DetectionResult(+ ContentType.BUILD_OUTPUT,+ confidence,+ {+ "pattern_matches": pattern_matches,+ "error_matches": error_matches,+ "total_lines": non_empty_lines,+ },+ )+++def _try_detect_code(content: str) -> DetectionResult | None:+ """Try to detect source code and identify language."""+ lines = content.split("\n")[:100] # Check first 100 lines+ if not lines:+ return None++ language_scores: dict[str, int] = {}++ for line in lines:+ for lang, patterns in _CODE_PATTERNS.items():+ for pattern in patterns:+ if pattern.match(line):+ language_scores[lang] = language_scores.get(lang, 0) + 1+ break # One pattern per language per line++ if not language_scores:+ return None++ # Find best matching language+ best_lang = max(language_scores, key=lambda k: language_scores[k])+ best_score = language_scores[best_lang]++ # Need at least 3 pattern matches to be confident+ if best_score < 3:+ return None++ non_empty_lines = sum(1 for line in lines if line.strip())+ ratio = best_score / max(non_empty_lines, 1)++ confidence = min(1.0, 0.4 + (ratio * 0.4) + (best_score * 0.02))++ return DetectionResult(+ ContentType.SOURCE_CODE,+ confidence,+ {"language": best_lang, "pattern_matches": best_score},+ )+++def is_json_array_of_dicts(content: str) -> bool:+ """Quick check if content is a JSON array of dictionaries.++ This is the format SmartCrusher can handle natively.++ Args:+ content: The content to check.++ Returns:+ True if content is a JSON array where all items are dicts.+ """+ result = detect_content_type(content)+ return (+ result.content_type == ContentType.JSON_ARRAY+ and result.metadata.get("is_dict_array", False)+ )diff --git a/headroom/transforms/log_compressor.py b/headroom/transforms/log_compressor.pynew file mode 100644index 000000000..0a150c47e--- /dev/null+++ b/headroom/transforms/log_compressor.py@@ -0,0 +1,542 @@+"""Log/build output compressor for test and compiler output.++This module compresses build and test output which can be 10,000+ lines+with only 5-10 actual errors. Typical compression: 10-50x.++Supported formats:+- pytest output+- npm/yarn output+- cargo/rustc output+- make/gcc output+- generic log format (ERROR, WARN, INFO)++Compression Strategy:+1. Detect log format (pytest, npm, cargo, etc.)+2. Extract all ERROR/FAIL lines with context+3. Extract first stack trace completely+4. Deduplicate repeated warnings+5. Summarize: [247 INFO lines, 12 WARN lines omitted]++Key Patterns to Preserve:+- First error (often root cause)+- Last error (sometimes the real failure)+- Stack traces+- Exit codes+- Test summary lines+"""++from __future__ import annotations++import re+from dataclasses import dataclass, field+from enum import Enum+++class LogFormat(Enum):+ """Detected log format."""++ PYTEST = "pytest"+ NPM = "npm"+ CARGO = "cargo"+ MAKE = "make"+ JEST = "jest"+ GENERIC = "generic"+++class LogLevel(Enum):+ """Log level for categorization."""++ ERROR = "error"+ FAIL = "fail"+ WARN = "warn"+ INFO = "info"+ DEBUG = "debug"+ TRACE = "trace"+ UNKNOWN = "unknown"+++@dataclass(eq=False)+class LogLine:+ """A single log line with metadata."""++ line_number: int+ content: str+ level: LogLevel = LogLevel.UNKNOWN+ is_stack_trace: bool = False+ is_summary: bool = False+ score: float = 0.0++ def __eq__(self, other: object) -> bool:+ if not isinstance(other, LogLine):+ return NotImplemented+ return self.line_number == other.line_number++ def __hash__(self) -> int:+ return hash(self.line_number)+++@dataclass+class LogCompressorConfig:+ """Configuration for log compression."""++ # Error handling+ max_errors: int = 10+ error_context_lines: int = 3+ keep_first_error: bool = True+ keep_last_error: bool = True++ # Stack trace handling+ max_stack_traces: int = 3+ stack_trace_max_lines: int = 20++ # Warning handling+ max_warnings: int = 5+ dedupe_warnings: bool = True++ # Summary handling+ keep_summary_lines: bool = True++ # Global limits+ max_total_lines: int = 100++ # CCR integration+ enable_ccr: bool = True+ min_lines_for_ccr: int = 50+++class LogCompressor:+ """Compresses build/test log output.++ Example:+ >>> compressor = LogCompressor()+ >>> result = compressor.compress(pytest_output)+ >>> print(result.compressed) # Just errors + summary+ """++ # Format detection patterns+ _FORMAT_PATTERNS = {+ LogFormat.PYTEST: [+ re.compile(r"^={3,} (FAILURES|ERRORS|test session|short test summary)"),+ re.compile(r"^(PASSED|FAILED|ERROR|SKIPPED)\s+\["),+ re.compile(r"^collected \d+ items?"),+ ],+ LogFormat.NPM: [+ re.compile(r"^npm (ERR!|WARN|info|http)"),+ re.compile(r"^(>|added|removed) .+ packages?"),+ ],+ LogFormat.CARGO: [+ re.compile(r"^\s*(Compiling|Finished|Running|error\[E\d+\])"),+ re.compile(r"^warning: .+"),+ ],+ LogFormat.JEST: [+ re.compile(r"^(PASS|FAIL)\s+.+\.test\.(js|ts)"),+ re.compile(r"^Test Suites:"),+ ],+ LogFormat.MAKE: [+ re.compile(r"^make(\[\d+\])?: "),+ re.compile(r"^(gcc|g\+\+|clang).*-o "),+ ],+ }++ # Level detection patterns+ _LEVEL_PATTERNS = {+ LogLevel.ERROR: re.compile(+ r"\b(ERROR|error|Error|FATAL|fatal|Fatal|CRITICAL|critical)\b"+ ),+ LogLevel.FAIL: re.compile(r"\b(FAIL|FAILED|fail|failed|Fail|Failed)\b"),+ LogLevel.WARN: re.compile(r"\b(WARN|WARNING|warn|warning|Warn|Warning)\b"),+ LogLevel.INFO: re.compile(r"\b(INFO|info|Info)\b"),+ LogLevel.DEBUG: re.compile(r"\b(DEBUG|debug|Debug)\b"),+ LogLevel.TRACE: re.compile(r"\b(TRACE|trace|Trace)\b"),+ }++ # Stack trace patterns+ _STACK_TRACE_PATTERNS = [+ re.compile(r"^\s*Traceback \(most recent call last\)"),+ re.compile(r'^\s*File ".+", line \d+'),+ re.compile(r"^\s*at .+\(.+:\d+:\d+\)"), # JS stack trace+ re.compile(r"^\s+at [\w.$]+\("), # Java stack trace+ re.compile(r"^\s*--> .+:\d+:\d+"), # Rust error+ re.compile(r"^\s*\d+:\s+0x[0-9a-f]+"), # Go stack trace+ ]++ # Summary line patterns+ _SUMMARY_PATTERNS = [+ re.compile(r"^={3,}"), # pytest separators+ re.compile(r"^-{3,}"),+ re.compile(r"^\d+ (passed|failed|skipped|error|warning)"),+ re.compile(r"^(Tests?|Suites?):?\s+\d+"),+ re.compile(r"^(TOTAL|Total|Summary)"),+ re.compile(r"^(Build|Compile|Test).*(succeeded|failed|complete)"),+ ]++ def __init__(self, config: LogCompressorConfig | None = None):+ """Initialize log compressor.++ Args:+ config: Compression configuration.+ """+ self.config = config or LogCompressorConfig()++ def compress(self, content: str, context: str = "") -> "LogCompressionResult":+ """Compress log output.++ Args:+ content: Raw log output.+ context: User query context (unused for now).++ Returns:+ LogCompressionResult with compressed output and metadata.+ """+ lines = content.split("\n")++ if len(lines) < self.config.min_lines_for_ccr:+ return LogCompressionResult(+ compressed=content,+ original=content,+ original_line_count=len(lines),+ compressed_line_count=len(lines),+ format_detected=LogFormat.GENERIC,+ compression_ratio=1.0,+ )++ # Detect format+ log_format = self._detect_format(lines)++ # Parse and categorize lines+ log_lines = self._parse_lines(lines)++ # Select important lines+ selected = self._select_lines(log_lines)++ # Format output with summaries+ compressed, stats = self._format_output(selected, log_lines)++ ratio = len(compressed) / max(len(content), 1)++ # Store in CCR if significant compression+ cache_key = None+ if self.config.enable_ccr and ratio < 0.5:+ cache_key = self._store_in_ccr(content, compressed, len(lines))+ if cache_key:+ compressed += f"\n[{len(lines)} lines compressed. hash={cache_key}]"++ return LogCompressionResult(+ compressed=compressed,+ original=content,+ original_line_count=len(lines),+ compressed_line_count=len(selected),+ format_detected=log_format,+ compression_ratio=ratio,+ cache_key=cache_key,+ stats=stats,+ )++ def _detect_format(self, lines: list[str]) -> LogFormat:+ """Detect the log format."""+ sample = lines[:100] # Check first 100 lines++ format_scores: dict[LogFormat, int] = {}+ for log_format, patterns in self._FORMAT_PATTERNS.items():+ score = 0+ for line in sample:+ for pattern in patterns:+ if pattern.search(line):+ score += 1+ break+ if score > 0:+ format_scores[log_format] = score++ if not format_scores:+ return LogFormat.GENERIC++ return max(format_scores, key=lambda k: format_scores[k])++ def _parse_lines(self, lines: list[str]) -> list[LogLine]:+ """Parse lines and categorize by level."""+ log_lines: list[LogLine] = []+ in_stack_trace = False+ stack_trace_lines = 0++ for i, line in enumerate(lines):+ log_line = LogLine(line_number=i, content=line)++ # Detect level+ for level, pattern in self._LEVEL_PATTERNS.items():+ if pattern.search(line):+ log_line.level = level+ break++ # Detect stack trace+ for pattern in self._STACK_TRACE_PATTERNS:+ if pattern.search(line):+ in_stack_trace = True+ stack_trace_lines = 0+ break++ if in_stack_trace:+ log_line.is_stack_trace = True+ stack_trace_lines += 1+ # End stack trace after max lines or empty line+ if stack_trace_lines > self.config.stack_trace_max_lines or not line.strip():+ in_stack_trace = False++ # Detect summary lines+ for pattern in self._SUMMARY_PATTERNS:+ if pattern.search(line):+ log_line.is_summary = True+ break++ # Score line by importance+ log_line.score = self._score_line(log_line)++ log_lines.append(log_line)++ return log_lines++ def _score_line(self, log_line: LogLine) -> float:+ """Score a line by importance."""+ score = 0.0++ # Level-based scoring+ level_scores = {+ LogLevel.ERROR: 1.0,+ LogLevel.FAIL: 1.0,+ LogLevel.WARN: 0.5,+ LogLevel.INFO: 0.1,+ LogLevel.DEBUG: 0.05,+ LogLevel.TRACE: 0.02,+ LogLevel.UNKNOWN: 0.1,+ }+ score += level_scores.get(log_line.level, 0.1)++ # Boost stack traces+ if log_line.is_stack_trace:+ score += 0.3++ # Boost summary lines+ if log_line.is_summary:+ score += 0.4++ return min(1.0, score)++ def _select_lines(self, log_lines: list[LogLine]) -> list[LogLine]:+ """Select important lines to keep."""+ selected: list[LogLine] = []++ # Group by category+ errors: list[LogLine] = []+ fails: list[LogLine] = []+ warnings: list[LogLine] = []+ stack_traces: list[list[LogLine]] = []+ summaries: list[LogLine] = []+ current_stack: list[LogLine] = []++ for log_line in log_lines:+ if log_line.level == LogLevel.ERROR:+ errors.append(log_line)+ elif log_line.level == LogLevel.FAIL:+ fails.append(log_line)+ elif log_line.level == LogLevel.WARN:+ warnings.append(log_line)++ if log_line.is_stack_trace:+ current_stack.append(log_line)+ elif current_stack:+ stack_traces.append(current_stack)+ current_stack = []++ if log_line.is_summary:+ summaries.append(log_line)++ if current_stack:+ stack_traces.append(current_stack)++ # Select errors (first, last, highest scoring)+ if errors:+ selected_errors = self._select_with_first_last(+ errors, self.config.max_errors+ )+ selected.extend(selected_errors)++ # Select fails+ if fails:+ selected_fails = self._select_with_first_last(fails, self.config.max_errors)+ selected.extend(selected_fails)++ # Select warnings (dedupe if configured)+ if warnings:+ if self.config.dedupe_warnings:+ warnings = self._dedupe_similar(warnings)+ selected.extend(warnings[: self.config.max_warnings])++ # Select stack traces+ for i, stack in enumerate(stack_traces[: self.config.max_stack_traces]):+ selected.extend(stack[: self.config.stack_trace_max_lines])++ # Always include summary lines+ if self.config.keep_summary_lines:+ selected.extend(summaries)++ # Add context lines around errors+ selected = self._add_context(log_lines, selected)++ # Sort by line number and dedupe+ selected = sorted(set(selected), key=lambda x: x.line_number)++ # Limit total lines+ if len(selected) > self.config.max_total_lines:+ # Keep most important lines+ selected = sorted(selected, key=lambda x: x.score, reverse=True)+ selected = selected[: self.config.max_total_lines]+ selected = sorted(selected, key=lambda x: x.line_number)++ return selected++ def _select_with_first_last(+ self, lines: list[LogLine], max_count: int+ ) -> list[LogLine]:+ """Select lines keeping first and last."""+ if len(lines) <= max_count:+ return lines++ selected: list[LogLine] = []++ if self.config.keep_first_error and lines:+ selected.append(lines[0])++ if self.config.keep_last_error and lines and lines[-1] not in selected:+ selected.append(lines[-1])++ # Fill remaining with highest scoring+ remaining = max_count - len(selected)+ if remaining > 0:+ candidates = [l for l in lines if l not in selected]+ candidates = sorted(candidates, key=lambda x: x.score, reverse=True)+ selected.extend(candidates[:remaining])++ return selected++ def _dedupe_similar(self, lines: list[LogLine]) -> list[LogLine]:+ """Remove duplicate/similar lines."""+ seen_patterns: set[str] = set()+ deduped: list[LogLine] = []++ for line in lines:+ # Normalize: remove numbers, paths for comparison+ normalized = re.sub(r"\d+", "N", line.content)+ normalized = re.sub(r"/[\w/]+/", "/PATH/", normalized)+ normalized = re.sub(r"0x[0-9a-f]+", "ADDR", normalized)++ if normalized not in seen_patterns:+ seen_patterns.add(normalized)+ deduped.append(line)++ return deduped++ def _add_context(+ self, all_lines: list[LogLine], selected: list[LogLine]+ ) -> list[LogLine]:+ """Add context lines around selected lines."""+ selected_indices = {l.line_number for l in selected}+ context_indices: set[int] = set()++ for idx in selected_indices:+ # Add lines before+ for i in range(+ max(0, idx - self.config.error_context_lines), idx+ ):+ context_indices.add(i)+ # Add lines after+ for i in range(+ idx + 1,+ min(len(all_lines), idx + self.config.error_context_lines + 1),+ ):+ context_indices.add(i)++ # Add context lines to selected+ for idx in context_indices:+ if idx not in selected_indices and idx < len(all_lines):+ selected.append(all_lines[idx])++ return selected++ def _format_output(+ self, selected: list[LogLine], all_lines: list[LogLine]+ ) -> tuple[str, dict[str, int]]:+ """Format selected lines with summary stats."""+ # Count categories+ stats: dict[str, int] = {+ "errors": sum(1 for l in all_lines if l.level == LogLevel.ERROR),+ "fails": sum(1 for l in all_lines if l.level == LogLevel.FAIL),+ "warnings": sum(1 for l in all_lines if l.level == LogLevel.WARN),+ "info": sum(1 for l in all_lines if l.level == LogLevel.INFO),+ "total": len(all_lines),+ "selected": len(selected),+ }++ # Build output+ output_lines = [line.content for line in selected]++ # Add summary of omitted lines+ omitted = len(all_lines) - len(selected)+ if omitted > 0:+ summary_parts = []+ for level_name, count in [+ ("ERROR", stats["errors"]),+ ("FAIL", stats["fails"]),+ ("WARN", stats["warnings"]),+ ("INFO", stats["info"]),+ ]:+ if count > 0:+ summary_parts.append(f"{count} {level_name}")++ if summary_parts:+ summary = f"[{omitted} lines omitted: {', '.join(summary_parts)}]"+ output_lines.append(summary)++ return "\n".join(output_lines), stats++ def _store_in_ccr(+ self, original: str, compressed: str, original_count: int+ ) -> str | None:+ """Store original in CCR for later retrieval."""+ try:+ from ..cache.compression_store import get_compression_store++ store = get_compression_store()+ return store.store(+ original_content=original,+ compressed_content=compressed,+ content_type="build_output",+ original_item_count=original_count,+ )+ except ImportError:+ return None+ except Exception:+ return None+++@dataclass+class LogCompressionResult:+ """Result of log compression."""++ compressed: str+ original: str+ original_line_count: int+ compressed_line_count: int+ format_detected: LogFormat+ compression_ratio: float+ cache_key: str | None = None+ stats: dict[str, int] = field(default_factory=dict)++ @property+ def tokens_saved_estimate(self) -> int:+ """Estimate tokens saved (rough: 1 token per 4 chars)."""+ chars_saved = len(self.original) - len(self.compressed)+ return max(0, chars_saved // 4)++ @property+ def lines_omitted(self) -> int:+ """Number of lines omitted."""+ return self.original_line_count - self.compressed_line_countdiff --git a/headroom/transforms/search_compressor.py b/headroom/transforms/search_compressor.pynew file mode 100644index 000000000..e5e2c7e23--- /dev/null+++ b/headroom/transforms/search_compressor.py@@ -0,0 +1,367 @@+"""Search results compressor for grep/ripgrep output.++This module compresses search results (grep, ripgrep, ag) which are one of+the most common outputs in coding tasks. Typical compression: 5-10x.++Input Format (grep -n style):+ src/utils.py:42:def process_data(items):+ src/utils.py:43: \"\"\"Process items with validation.\"\"\"+ src/models.py:15:class DataProcessor:++Compression Strategy:+1. Parse into {file: [(line, content), ...]} structure+2. Group by file+3. For each file: keep first match, last match, context-relevant matches+4. Deduplicate near-identical lines+5. Add summary: [... and N more matches in file.py]++Integrates with CCR for reversible compression.+"""++from __future__ import annotations++import hashlib+import re+from dataclasses import dataclass, field+++@dataclass+class SearchMatch:+ """A single search match."""++ file: str+ line_number: int+ content: str+ score: float = 0.0 # Relevance score+++@dataclass+class FileMatches:+ """All matches in a single file."""++ file: str+ matches: list[SearchMatch] = field(default_factory=list)++ @property+ def first(self) -> SearchMatch | None:+ return self.matches[0] if self.matches else None++ @property+ def last(self) -> SearchMatch | None:+ return self.matches[-1] if self.matches else None+++@dataclass+class SearchCompressorConfig:+ """Configuration for search result compression."""++ # Per-file limits+ max_matches_per_file: int = 5+ always_keep_first: bool = True+ always_keep_last: bool = True++ # Global limits+ max_total_matches: int = 30+ max_files: int = 15++ # Context matching+ context_keywords: list[str] = field(default_factory=list)+ boost_errors: bool = True++ # CCR integration+ enable_ccr: bool = True+ min_matches_for_ccr: int = 10+++class SearchCompressor:+ """Compresses grep/ripgrep search results.++ Example:+ >>> compressor = SearchCompressor()+ >>> result = compressor.compress(search_output, context="find error handlers")+ >>> print(result.compressed) # Reduced output with summary+ """++ # Pattern to parse grep-style output: file:line:content+ _GREP_PATTERN = re.compile(r"^([^:]+):(\d+):(.*)$")++ # Pattern for ripgrep with context (file-line-content or file:line:content)+ _RG_CONTEXT_PATTERN = re.compile(r"^([^:-]+)[:-](\d+)[:-](.*)$")++ # Error/important patterns to prioritize+ _PRIORITY_PATTERNS = [+ re.compile(r"\b(error|exception|fail|fatal)\b", re.IGNORECASE),+ re.compile(r"\b(warn|warning)\b", re.IGNORECASE),+ re.compile(r"\b(todo|fixme|hack|xxx)\b", re.IGNORECASE),+ ]++ def __init__(self, config: SearchCompressorConfig | None = None):+ """Initialize search compressor.++ Args:+ config: Compression configuration.+ """+ self.config = config or SearchCompressorConfig()++ def compress(+ self,+ content: str,+ context: str = "",+ ) -> "SearchCompressionResult":+ """Compress search results.++ Args:+ content: Raw grep/ripgrep output.+ context: User query context for relevance scoring.++ Returns:+ SearchCompressionResult with compressed output and metadata.+ """+ # Parse search results+ file_matches = self._parse_search_results(content)++ if not file_matches:+ return SearchCompressionResult(+ compressed=content,+ original=content,+ original_match_count=0,+ compressed_match_count=0,+ files_affected=0,+ compression_ratio=1.0,+ )++ # Count original matches+ original_count = sum(len(fm.matches) for fm in file_matches.values())++ # Score matches by relevance+ self._score_matches(file_matches, context)++ # Select top matches per file+ selected = self._select_matches(file_matches)++ # Format compressed output+ compressed, summaries = self._format_output(selected, file_matches)++ # Count compressed matches+ compressed_count = sum(len(fm.matches) for fm in selected.values())++ # Calculate compression ratio+ ratio = len(compressed) / max(len(content), 1)++ # Store in CCR if significant compression+ cache_key = None+ if (+ self.config.enable_ccr+ and original_count >= self.config.min_matches_for_ccr+ and ratio < 0.8+ ):+ cache_key = self._store_in_ccr(content, compressed, original_count)+ if cache_key:+ compressed += f"\n[{original_count} matches compressed. hash={cache_key}]"++ return SearchCompressionResult(+ compressed=compressed,+ original=content,+ original_match_count=original_count,+ compressed_match_count=compressed_count,+ files_affected=len(file_matches),+ compression_ratio=ratio,+ cache_key=cache_key,+ summaries=summaries,+ )++ def _parse_search_results(self, content: str) -> dict[str, FileMatches]:+ """Parse grep-style output into structured data."""+ file_matches: dict[str, FileMatches] = {}++ for line in content.split("\n"):+ line = line.strip()+ if not line:+ continue++ # Try grep pattern first+ match = self._GREP_PATTERN.match(line)+ if not match:+ match = self._RG_CONTEXT_PATTERN.match(line)++ if match:+ file_path, line_num, match_content = match.groups()++ if file_path not in file_matches:+ file_matches[file_path] = FileMatches(file=file_path)++ file_matches[file_path].matches.append(+ SearchMatch(+ file=file_path,+ line_number=int(line_num),+ content=match_content,+ )+ )++ return file_matches++ def _score_matches(+ self,+ file_matches: dict[str, FileMatches],+ context: str,+ ) -> None:+ """Score matches by relevance to context."""+ context_lower = context.lower()+ context_words = set(context_lower.split())++ for fm in file_matches.values():+ for match in fm.matches:+ score = 0.0+ content_lower = match.content.lower()++ # Score by context word overlap+ for word in context_words:+ if len(word) > 2 and word in content_lower:+ score += 0.3++ # Boost error/warning patterns+ if self.config.boost_errors:+ for i, pattern in enumerate(self._PRIORITY_PATTERNS):+ if pattern.search(match.content):+ score += 0.5 - (i * 0.1) # Higher boost for errors++ # Boost for keyword matches+ for keyword in self.config.context_keywords:+ if keyword.lower() in content_lower:+ score += 0.4++ match.score = min(1.0, score)++ def _select_matches(+ self,+ file_matches: dict[str, FileMatches],+ ) -> dict[str, FileMatches]:+ """Select top matches per file and globally."""+ selected: dict[str, FileMatches] = {}++ # Sort files by total match score (highest first)+ sorted_files = sorted(+ file_matches.items(),+ key=lambda x: sum(m.score for m in x[1].matches),+ reverse=True,+ )++ # Limit number of files+ sorted_files = sorted_files[: self.config.max_files]++ total_selected = 0+ for file_path, fm in sorted_files:+ if total_selected >= self.config.max_total_matches:+ break++ # Sort matches by score+ sorted_matches = sorted(fm.matches, key=lambda m: m.score, reverse=True)++ # Select matches for this file+ file_selected: list[SearchMatch] = []+ remaining_slots = min(+ self.config.max_matches_per_file,+ self.config.max_total_matches - total_selected,+ )++ # Always include first and last if configured+ if self.config.always_keep_first and fm.first:+ file_selected.append(fm.first)+ remaining_slots -= 1++ if (+ self.config.always_keep_last+ and fm.last+ and fm.last != fm.first+ and remaining_slots > 0+ ):+ file_selected.append(fm.last)+ remaining_slots -= 1++ # Fill remaining slots with highest-scoring matches+ for match in sorted_matches:+ if remaining_slots <= 0:+ break+ if match not in file_selected:+ file_selected.append(match)+ remaining_slots -= 1++ # Sort by line number for output+ file_selected.sort(key=lambda m: m.line_number)++ selected[file_path] = FileMatches(file=file_path, matches=file_selected)+ total_selected += len(file_selected)++ return selected++ def _format_output(+ self,+ selected: dict[str, FileMatches],+ original: dict[str, FileMatches],+ ) -> tuple[str, dict[str, str]]:+ """Format selected matches back to grep-style output."""+ lines: list[str] = []+ summaries: dict[str, str] = {}++ for file_path, fm in sorted(selected.items()):+ for match in fm.matches:+ lines.append(f"{match.file}:{match.line_number}:{match.content}")++ # Add summary if matches were omitted+ original_fm = original.get(file_path)+ if original_fm and len(original_fm.matches) > len(fm.matches):+ omitted = len(original_fm.matches) - len(fm.matches)+ summary = f"[... and {omitted} more matches in {file_path}]"+ lines.append(summary)+ summaries[file_path] = summary++ return "\n".join(lines), summaries++ def _store_in_ccr(+ self,+ original: str,+ compressed: str,+ original_count: int,+ ) -> str | None:+ """Store original in CCR for later retrieval."""+ try:+ from ..cache.compression_store import get_compression_store++ store = get_compression_store()+ return store.store(+ original_content=original,+ compressed_content=compressed,+ content_type="search_results",+ original_item_count=original_count,+ )+ except ImportError:+ # CCR not available+ return None+ except Exception:+ # Silently fail CCR storage+ return None+++@dataclass+class SearchCompressionResult:+ """Result of search result compression."""++ compressed: str+ original: str+ original_match_count: int+ compressed_match_count: int+ files_affected: int+ compression_ratio: float+ cache_key: str | None = None+ summaries: dict[str, str] = field(default_factory=dict)++ @property+ def tokens_saved_estimate(self) -> int:+ """Estimate tokens saved (rough: 1 token per 4 chars)."""+ chars_saved = len(self.original) - len(self.compressed)+ return max(0, chars_saved // 4)++ @property+ def matches_omitted(self) -> int:+ """Number of matches omitted."""+ return self.original_match_count - self.compressed_match_countdiff --git a/headroom/transforms/smart_crusher.py b/headroom/transforms/smart_crusher.pyindex 787fb14fa..6dbcfebaf 100644--- a/headroom/transforms/smart_crusher.py+++ b/headroom/transforms/smart_crusher.py@@ -4,6 +4,19 @@ This module provides intelligent JSON compression based on statistical analysisrather than fixed rules. It analyzes data patterns and applies optimal compressionstrategies to maximize token reduction while preserving important information.+SCOPE: SmartCrusher handles JSON arrays only. Non-JSON content (plain text,+search results, logs, code, diffs) passes through UNCHANGED.++TEXT COMPRESSION IS OPT-IN: For text-based content, Headroom provides standalone+utilities that applications can use explicitly:+- SearchCompressor: For grep/ripgrep output (file:line:content format)+- LogCompressor: For build/test logs (pytest, npm, cargo output)+- TextCompressor: For generic plain text with anchor preservation++Applications should decide when and how to use text compression based on their+specific needs. This design prevents lossy text compression from being applied+automatically, which could lose important context in coding tasks.+SCHEMA-PRESERVING: Output contains only items from the original array.No wrappers, no generated text, no metadata keys. This ensures downstreamtools and parsers work unchanged.@@ -1731,8 +1744,11 @@ class SmartCrusher(Transform):"""Apply smart crushing to content.+ Handles both JSON (existing SmartCrusher logic) and plain text content+ (search results, logs, generic text) using specialized compressors.+Args:- content: JSON string to crush.+ content: Content to crush (JSON or plain text).query_context: Context string from user messages for relevance scoring.tool_name: Name of the tool that produced this output.@@ -1741,6 +1757,9 @@ class SmartCrusher(Transform):"""parsed, success = safe_json_loads(content)if not success:+ # Not JSON - pass through unchanged+ # Text compression utilities (SearchCompressor, LogCompressor, TextCompressor)+ # are available as standalone tools for applications to use explicitlyreturn content, False, ""# Recursively process and crush arraysdiff --git a/headroom/transforms/text_compressor.py b/headroom/transforms/text_compressor.pynew file mode 100644index 000000000..dceba9365--- /dev/null+++ b/headroom/transforms/text_compressor.py@@ -0,0 +1,270 @@+"""Generic text compressor for plain text content.++This module provides a fallback compressor for plain text that doesn't match+any specialized format (search results, logs, code, diffs). Uses line-based+sampling with anchor preservation.++Compression Strategy:+1. Identify anchor lines (contain context keywords)+2. Keep first N and last M lines+3. Sample from middle based on line importance+4. Add summary of omitted content+"""++from __future__ import annotations++import re+from dataclasses import dataclass, field+++@dataclass+class TextCompressorConfig:+ """Configuration for text compression."""++ # Line limits+ keep_first_lines: int = 10+ keep_last_lines: int = 10+ max_total_lines: int = 50++ # Sampling+ sample_every_n_lines: int = 10++ # Anchor detection+ anchor_keywords: list[str] = field(default_factory=list)+ boost_pattern_lines: bool = True++ # CCR integration+ enable_ccr: bool = True+ min_lines_for_ccr: int = 100+++class TextCompressor:+ """Compresses generic plain text.++ Example:+ >>> compressor = TextCompressor()+ >>> result = compressor.compress(large_text, context="find errors")+ >>> print(result.compressed)+ """++ # Patterns that indicate important lines+ _IMPORTANT_PATTERNS = [+ re.compile(r"\b(error|exception|fail|warning)\b", re.IGNORECASE),+ re.compile(r"\b(important|note|todo|fixme)\b", re.IGNORECASE),+ re.compile(r"^#+\s"), # Markdown headers+ re.compile(r"^\*\*"), # Bold text+ re.compile(r"^>\s"), # Quotes+ ]++ def __init__(self, config: TextCompressorConfig | None = None):+ """Initialize text compressor.++ Args:+ config: Compression configuration.+ """+ self.config = config or TextCompressorConfig()++ def compress(self, content: str, context: str = "") -> "TextCompressionResult":+ """Compress text content.++ Args:+ content: Raw text content.+ context: User query context for anchor detection.++ Returns:+ TextCompressionResult with compressed output.+ """+ lines = content.split("\n")++ if len(lines) <= self.config.max_total_lines:+ return TextCompressionResult(+ compressed=content,+ original=content,+ original_line_count=len(lines),+ compressed_line_count=len(lines),+ compression_ratio=1.0,+ )++ # Score lines by importance+ scored_lines = self._score_lines(lines, context)++ # Select lines+ selected = self._select_lines(scored_lines, lines)++ # Format output+ compressed = self._format_output(selected, len(lines))++ ratio = len(compressed) / max(len(content), 1)++ # Store in CCR if significant compression+ cache_key = None+ if (+ self.config.enable_ccr+ and len(lines) >= self.config.min_lines_for_ccr+ and ratio < 0.7+ ):+ cache_key = self._store_in_ccr(content, compressed, len(lines))+ if cache_key:+ compressed += f"\n[{len(lines)} lines compressed. hash={cache_key}]"++ return TextCompressionResult(+ compressed=compressed,+ original=content,+ original_line_count=len(lines),+ compressed_line_count=len(selected),+ compression_ratio=ratio,+ cache_key=cache_key,+ )++ def _score_lines(+ self, lines: list[str], context: str+ ) -> list[tuple[int, str, float]]:+ """Score lines by importance."""+ context_lower = context.lower()+ context_words = set(context_lower.split()) if context else set()+ anchor_keywords = set(k.lower() for k in self.config.anchor_keywords)++ scored: list[tuple[int, str, float]] = []++ for i, line in enumerate(lines):+ score = 0.0+ line_lower = line.lower()++ # Boost if contains context words+ for word in context_words:+ if len(word) > 2 and word in line_lower:+ score += 0.3++ # Boost if contains anchor keywords+ for keyword in anchor_keywords:+ if keyword in line_lower:+ score += 0.4++ # Boost if matches important patterns+ if self.config.boost_pattern_lines:+ for pattern in self._IMPORTANT_PATTERNS:+ if pattern.search(line):+ score += 0.2+ break++ # Small boost for non-empty lines+ if line.strip():+ score += 0.1++ scored.append((i, line, min(1.0, score)))++ return scored++ def _select_lines(+ self, scored_lines: list[tuple[int, str, float]], original_lines: list[str]+ ) -> list[tuple[int, str]]:+ """Select lines to keep."""+ total = len(scored_lines)+ selected_indices: set[int] = set()++ # Always keep first N lines+ for i in range(min(self.config.keep_first_lines, total)):+ selected_indices.add(i)++ # Always keep last M lines+ for i in range(max(0, total - self.config.keep_last_lines), total):+ selected_indices.add(i)++ # Add high-scoring lines+ high_score_lines = [+ (idx, line, score)+ for idx, line, score in scored_lines+ if score >= 0.3 and idx not in selected_indices+ ]+ high_score_lines.sort(key=lambda x: x[2], reverse=True)++ remaining_slots = self.config.max_total_lines - len(selected_indices)+ for idx, line, score in high_score_lines[:remaining_slots]:+ selected_indices.add(idx)+ remaining_slots -= 1+ if remaining_slots <= 0:+ break++ # Sample from remaining middle lines+ if remaining_slots > 0:+ middle_start = self.config.keep_first_lines+ middle_end = total - self.config.keep_last_lines++ for i in range(middle_start, middle_end, self.config.sample_every_n_lines):+ if i not in selected_indices:+ selected_indices.add(i)+ remaining_slots -= 1+ if remaining_slots <= 0:+ break++ # Sort by line number and return+ selected = sorted(selected_indices)+ return [(i, original_lines[i]) for i in selected]++ def _format_output(+ self, selected: list[tuple[int, str]], total_lines: int+ ) -> str:+ """Format selected lines with ellipsis markers."""+ if not selected:+ return f"[{total_lines} lines omitted]"++ output_lines: list[str] = []+ prev_idx = -1++ for idx, line in selected:+ # Add ellipsis if there's a gap+ if prev_idx >= 0 and idx - prev_idx > 1:+ gap = idx - prev_idx - 1+ output_lines.append(f"[... {gap} lines omitted ...]")++ output_lines.append(line)+ prev_idx = idx++ # Add trailing ellipsis if needed+ if selected and selected[-1][0] < total_lines - 1:+ gap = total_lines - selected[-1][0] - 1+ output_lines.append(f"[... {gap} lines omitted ...]")++ return "\n".join(output_lines)++ def _store_in_ccr(+ self, original: str, compressed: str, original_count: int+ ) -> str | None:+ """Store original in CCR for later retrieval."""+ try:+ from ..cache.compression_store import get_compression_store++ store = get_compression_store()+ return store.store(+ original_content=original,+ compressed_content=compressed,+ content_type="plain_text",+ original_item_count=original_count,+ )+ except ImportError:+ return None+ except Exception:+ return None+++@dataclass+class TextCompressionResult:+ """Result of text compression."""++ compressed: str+ original: str+ original_line_count: int+ compressed_line_count: int+ compression_ratio: float+ cache_key: str | None = None++ @property+ def tokens_saved_estimate(self) -> int:+ """Estimate tokens saved."""+ chars_saved = len(self.original) - len(self.compressed)+ return max(0, chars_saved // 4)++ @property+ def lines_omitted(self) -> int:+ """Number of lines omitted."""+ return self.original_line_count - self.compressed_line_countdiff --git a/tests/test_text_compressors.py b/tests/test_text_compressors.pynew file mode 100644index 000000000..5206f019e--- /dev/null+++ b/tests/test_text_compressors.py@@ -0,0 +1,443 @@+"""Tests for text-based compressors (coding task support).++Tests content detection, search compressor, log compressor, and text compressor.+"""++import pytest++from headroom.transforms import (+ ContentType,+ DetectionResult,+ LogCompressor,+ LogCompressorConfig,+ SearchCompressor,+ SearchCompressorConfig,+ TextCompressor,+ TextCompressorConfig,+ detect_content_type,+)+++class TestContentDetector:+ """Tests for content type detection."""++ def test_detect_json_array_of_dicts(self):+ """JSON arrays of dicts are detected correctly."""+ content = '[{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}]'+ result = detect_content_type(content)+ assert result.content_type == ContentType.JSON_ARRAY+ assert result.confidence >= 0.8+ assert result.metadata.get("is_dict_array") is True++ def test_detect_json_array_non_dict(self):+ """JSON arrays of non-dicts are detected."""+ content = '[1, 2, 3, 4, 5]'+ result = detect_content_type(content)+ assert result.content_type == ContentType.JSON_ARRAY+ assert result.metadata.get("is_dict_array") is False++ def test_detect_search_results(self):+ """grep-style search results are detected."""+ content = """src/main.py:42:def process_data(items):+src/main.py:43: \"\"\"Process items.\"\"\"+src/utils.py:15:def validate(data):+src/utils.py:16: return data is not None+src/models.py:100:class DataProcessor:+"""+ result = detect_content_type(content)+ assert result.content_type == ContentType.SEARCH_RESULTS+ assert result.confidence >= 0.6++ def test_detect_build_output(self):+ """Build/test output is detected."""+ content = """+============================= test session starts ==============================+platform darwin -- Python 3.11.0+collected 15 items++tests/test_foo.py::test_basic PASSED+tests/test_foo.py::test_edge_case FAILED+tests/test_bar.py::test_another ERROR++=================================== FAILURES ===================================+tests/test_foo.py::test_edge_case - AssertionError: expected 5, got 3++=========================== short test summary info ============================+FAILED tests/test_foo.py::test_edge_case+ERROR tests/test_bar.py::test_another+========================= 1 failed, 1 passed, 1 error =========================+"""+ result = detect_content_type(content)+ assert result.content_type == ContentType.BUILD_OUTPUT+ assert result.confidence >= 0.5++ def test_detect_git_diff(self):+ """Git diff format is detected."""+ content = """diff --git a/src/main.py b/src/main.py+index abc123..def456 100644+--- a/src/main.py++++ b/src/main.py+@@ -10,7 +10,7 @@ def process():+- old_line = True++ new_line = True+ unchanged = "same"+"""+ result = detect_content_type(content)+ assert result.content_type == ContentType.GIT_DIFF+ assert result.confidence >= 0.7++ def test_detect_python_code(self):+ """Python source code is detected."""+ content = """+import json+from typing import Any++def process_data(items: list[dict]) -> dict[str, Any]:+ \"\"\"Process a list of items.++ Args:+ items: List of dictionaries to process.++ Returns:+ Processed result dictionary.+ \"\"\"+ result = {}+ for item in items:+ key = item.get("id")+ result[key] = item+ return result++class DataProcessor:+ def __init__(self):+ self.cache = {}++ async def async_process(self, data):+ return await self._do_process(data)+"""+ result = detect_content_type(content)+ assert result.content_type == ContentType.SOURCE_CODE+ assert result.metadata.get("language") == "python"++ def test_detect_plain_text(self):+ """Plain text falls back correctly."""+ content = """This is just some random text+that doesn't match any specific pattern.+It's just prose, really.+Nothing special about it."""+ result = detect_content_type(content)+ assert result.content_type == ContentType.PLAIN_TEXT+++class TestSearchCompressor:+ """Tests for search results compression."""++ def test_compress_search_results(self):+ """Search results are compressed."""+ content = "\n".join(+ [f"src/file{i}.py:{i * 10}:def function_{i}():" for i in range(100)]+ )++ compressor = SearchCompressor()+ result = compressor.compress(content, context="find function_50")++ assert result.original_match_count == 100+ assert result.compressed_match_count < 100+ assert "function_" in result.compressed++ def test_keeps_first_and_last(self):+ """First and last matches are preserved."""+ content = "\n".join(+ [f"src/file.py:{i}:line {i}" for i in range(1, 101)]+ )++ compressor = SearchCompressor(+ config=SearchCompressorConfig(+ always_keep_first=True,+ always_keep_last=True,+ )+ )+ result = compressor.compress(content)++ assert "src/file.py:1:line 1" in result.compressed+ assert "src/file.py:100:line 100" in result.compressed++ def test_prioritizes_errors(self):+ """Error lines are prioritized."""+ lines = [f"src/file.py:{i}:normal line" for i in range(1, 50)]+ lines.append("src/file.py:50:ERROR: something failed")+ lines.extend([f"src/file.py:{i}:normal line" for i in range(51, 100)])+ content = "\n".join(lines)++ compressor = SearchCompressor()+ result = compressor.compress(content)++ assert "ERROR: something failed" in result.compressed++ def test_small_results_unchanged(self):+ """Small search results pass through unchanged."""+ content = "src/file.py:1:def foo():\nsrc/file.py:2: pass"++ compressor = SearchCompressor()+ result = compressor.compress(content)++ assert result.compression_ratio == 1.0+ assert result.compressed == content+++class TestLogCompressor:+ """Tests for log/build output compression."""++ def test_compress_pytest_output(self):+ """pytest output is compressed."""+ lines = ["=" * 40 + " test session starts " + "=" * 40]+ lines.append("collected 100 items")+ lines.extend([f"tests/test_{i}.py::test_case_{i} PASSED" for i in range(95)])+ lines.extend([+ "tests/test_fail.py::test_case_fail FAILED",+ "",+ "=" * 40 + " FAILURES " + "=" * 40,+ "tests/test_fail.py::test_case_fail",+ "AssertionError: expected True, got False",+ "",+ "=" * 40 + " short test summary " + "=" * 40,+ "FAILED tests/test_fail.py::test_case_fail",+ "1 failed, 95 passed",+ ])+ content = "\n".join(lines)++ compressor = LogCompressor()+ result = compressor.compress(content)++ # Should keep failures and summary+ assert "FAILED" in result.compressed+ assert "AssertionError" in result.compressed+ assert result.compression_ratio < 0.5++ def test_keeps_errors_and_stack_traces(self):+ """Errors and stack traces are preserved."""+ content = """+INFO: Starting process+INFO: Loading data+INFO: Processing item 1+INFO: Processing item 2+ERROR: Failed to process item 3+Traceback (most recent call last):+ File "main.py", line 42, in process+ result = compute(data)+ File "utils.py", line 15, in compute+ return data / 0+ZeroDivisionError: division by zero+INFO: Continuing with remaining items+INFO: Done+"""+ compressor = LogCompressor()+ result = compressor.compress(content)++ assert "ERROR: Failed to process" in result.compressed+ assert "Traceback" in result.compressed+ assert "ZeroDivisionError" in result.compressed++ def test_small_logs_unchanged(self):+ """Small logs pass through unchanged."""+ content = "INFO: Starting\nINFO: Done"++ compressor = LogCompressor(+ config=LogCompressorConfig(min_lines_for_ccr=100)+ )+ result = compressor.compress(content)++ assert result.compression_ratio == 1.0+++class TestTextCompressor:+ """Tests for generic text compression."""++ def test_compress_large_text(self):+ """Large text is compressed."""+ lines = [f"Line {i}: This is some content that repeats" for i in range(200)]+ content = "\n".join(lines)++ compressor = TextCompressor()+ result = compressor.compress(content)++ assert result.compression_ratio < 0.5+ assert "Line 0:" in result.compressed # First line kept+ assert "lines omitted" in result.compressed++ def test_preserves_context_matches(self):+ """Lines matching context are preserved."""+ lines = [f"Line {i}: normal content" for i in range(100)]+ lines[50] = "Line 50: IMPORTANT error message"+ content = "\n".join(lines)++ compressor = TextCompressor(+ config=TextCompressorConfig(+ anchor_keywords=["error", "important"],+ )+ )+ result = compressor.compress(content, context="find errors")++ assert "IMPORTANT error message" in result.compressed++ def test_small_text_unchanged(self):+ """Small text passes through unchanged."""+ content = "Short text\nthat fits easily"++ compressor = TextCompressor()+ result = compressor.compress(content)++ assert result.compression_ratio == 1.0+ assert result.compressed == content+++class TestSmartCrusherTextIntegration:+ """Tests for SmartCrusher behavior with different content types.++ NOTE: SmartCrusher is designed for JSON compression only.+ Plain text content (search results, logs, etc.) passes through UNCHANGED.++ Text compression utilities (SearchCompressor, LogCompressor, TextCompressor)+ are available as standalone tools for applications to use explicitly.+ This is intentional - text compression is opt-in, not automatic.+ """++ @staticmethod+ def _get_tokenizer():+ """Get a tokenizer for tests using OpenAI provider."""+ from headroom.providers import OpenAIProvider+ from headroom.tokenizer import Tokenizer++ provider = OpenAIProvider()+ token_counter = provider.get_token_counter("gpt-4o")+ return Tokenizer(token_counter, "gpt-4o")++ def test_smart_crusher_passes_through_search_results_unchanged(self):+ """SmartCrusher passes non-JSON search results through unchanged.++ Applications should use SearchCompressor directly if compression is needed.+ """+ from headroom.transforms import SmartCrusher, SmartCrusherConfig++ # Create search results content+ search_results = "\n".join(+ [f"src/file{i}.py:{i}:def function_{i}():" for i in range(100)]+ )++ messages = [+ {"role": "user", "content": "Find all function definitions"},+ {"role": "tool", "content": search_results},+ ]++ crusher = SmartCrusher(+ config=SmartCrusherConfig(min_tokens_to_crush=10)+ )+ tokenizer = self._get_tokenizer()++ result = crusher.apply(messages, tokenizer)++ # Non-JSON passes through UNCHANGED - this is correct behavior+ tool_content = result.messages[1]["content"]+ assert tool_content == search_results++ def test_smart_crusher_passes_through_log_output_unchanged(self):+ """SmartCrusher passes non-JSON log output through unchanged.++ Applications should use LogCompressor directly if compression is needed.+ """+ from headroom.transforms import SmartCrusher, SmartCrusherConfig++ # Create log content+ lines = ["INFO: Processing item " + str(i) for i in range(100)]+ lines.append("ERROR: Critical failure at item 50")+ lines.append("Traceback (most recent call last):")+ lines.append(' File "main.py", line 100, in process')+ lines.append("RuntimeError: something broke")+ log_content = "\n".join(lines)++ messages = [+ {"role": "user", "content": "Run the tests"},+ {"role": "tool", "content": log_content},+ ]++ crusher = SmartCrusher(+ config=SmartCrusherConfig(min_tokens_to_crush=10)+ )+ tokenizer = self._get_tokenizer()++ result = crusher.apply(messages, tokenizer)++ # Non-JSON passes through UNCHANGED - this is correct behavior+ tool_content = result.messages[1]["content"]+ assert tool_content == log_content++ def test_search_compressor_available_as_standalone(self):+ """SearchCompressor is available for explicit use by applications."""+ # Create search results content+ search_results = "\n".join(+ [f"src/file{i}.py:{i}:def function_{i}():" for i in range(100)]+ )++ # Application explicitly chooses to compress+ compressor = SearchCompressor()+ result = compressor.compress(search_results, context="find function_50")++ # Compression happens when explicitly requested+ assert result.original_match_count == 100+ assert result.compressed_match_count < 100+ assert "function_" in result.compressed++ def test_log_compressor_available_as_standalone(self):+ """LogCompressor is available for explicit use by applications."""+ # Create log content+ lines = ["INFO: Processing item " + str(i) for i in range(100)]+ lines.append("ERROR: Critical failure at item 50")+ lines.append("Traceback (most recent call last):")+ lines.append(' File "main.py", line 100, in process')+ lines.append("RuntimeError: something broke")+ log_content = "\n".join(lines)++ # Application explicitly chooses to compress+ compressor = LogCompressor()+ result = compressor.compress(log_content)++ # Compression happens when explicitly requested, errors preserved+ assert "ERROR: Critical failure" in result.compressed+ assert "RuntimeError" in result.compressed+ assert result.compression_ratio < 1.0 # Some compression occurred++ def test_smart_crusher_json_still_works(self):+ """SmartCrusher still handles JSON correctly."""+ from headroom.transforms import SmartCrusher, SmartCrusherConfig+ import json+ import re++ # Create JSON array content with larger items to trigger compression+ items = [+ {"id": i, "name": f"Item {i}", "value": i * 10, "description": f"This is item number {i}"}+ for i in range(500)+ ]+ json_content = json.dumps(items)++ messages = [+ {"role": "user", "content": "Get all items"},+ {"role": "tool", "content": json_content},+ ]++ crusher = SmartCrusher(+ config=SmartCrusherConfig(min_tokens_to_crush=10, min_items_to_analyze=5)+ )+ tokenizer = self._get_tokenizer()++ result = crusher.apply(messages, tokenizer)++ # Check JSON compression happened (character-level, not necessarily item count)+ tool_content = result.messages[1]["content"]+ # SmartCrusher compresses JSON (may reduce chars via field pruning, etc.)+ # or adds a digest marker - either way it processes the JSON+ assert len(tool_content) <= len(json_content) + 100 # Allow for digest marker++ # Extract JSON part (may have headroom digest marker appended)+ base_content = re.split(r"\n<headroom:", tool_content)[0]+ parsed = json.loads(base_content)+ assert isinstance(parsed, list)+ assert len(parsed) > 0 # JSON is still valid and contains items