Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
Add DiffCompressor and fix hnswlib SIGILL crash on CI
chopratejas5c740ea427a1
diff --git a/headroom/memory/adapters/__init__.py b/headroom/memory/adapters/__init__.pyindex 160823f33..d0dc11bb8 100644--- a/headroom/memory/adapters/__init__.py+++ b/headroom/memory/adapters/__init__.py@@ -21,21 +21,31 @@ from headroom.memory.adapters.graph import InMemoryGraphStorefrom headroom.memory.adapters.sqlite import SQLiteMemoryStore# Check for optional dependencies availability-try:- from headroom.memory.adapters.hnsw import HNSW_AVAILABLE-except ImportError:- HNSW_AVAILABLE = False+# Note: We don't import from hnsw.py here because hnswlib may crash with+# "Illegal instruction" on CPUs without required instructions (e.g., AVX).+# Instead, we check lazily when HNSWVectorIndex is actually used.+# HNSW_AVAILABLE is handled through __getattr__ to ensure lazy checking.# Lazy imports for optional adapters+_HNSW_AVAILABLE: bool | None = None # Internal cache for HNSW_AVAILABLE_HNSWVectorIndex = None_LocalEmbedder = None_OpenAIEmbedder = None_OllamaEmbedder = None-def __getattr__(name: str) -> type:+def __getattr__(name: str) -> type | bool:"""Lazy import for optional adapters."""global _HNSWVectorIndex, _LocalEmbedder, _OpenAIEmbedder, _OllamaEmbedder+ global _HNSW_AVAILABLE++ if name == "HNSW_AVAILABLE":+ # Lazily check hnswlib availability+ if _HNSW_AVAILABLE is None:+ from headroom.memory.adapters.hnsw import _check_hnswlib_available++ _HNSW_AVAILABLE = _check_hnswlib_available()+ return _HNSW_AVAILABLEif name == "HNSWVectorIndex":if _HNSWVectorIndex is None:diff --git a/headroom/memory/adapters/hnsw.py b/headroom/memory/adapters/hnsw.pyindex 22bc4f4a2..33f95eeb4 100644--- a/headroom/memory/adapters/hnsw.py+++ b/headroom/memory/adapters/hnsw.py@@ -22,17 +22,44 @@ from typing import TYPE_CHECKING, Anyimport numpy as np+from ..models import Memory, ScopeLevel+from ..ports import VectorFilter, VectorSearchResult+# hnswlib is optional - may not compile on all platforms-try:- import hnswlib+# NOTE: We don't import hnswlib at module level because it can crash with SIGILL+# (Illegal Instruction) on CPUs without required AVX instructions. The crash+# happens at the C level before Python's try/except can catch it.+# Instead, we import lazily when HNSWVectorIndex is actually instantiated.+hnswlib: Any = None # Will be imported lazily+HNSW_AVAILABLE: bool | None = None # None = not yet checked, True/False = checked- HNSW_AVAILABLE = True-except ImportError:- hnswlib = None # type: ignore[assignment]- HNSW_AVAILABLE = False-from ..models import Memory, ScopeLevel-from ..ports import VectorFilter, VectorSearchResult+def _check_hnswlib_available() -> bool:+ """Check if hnswlib is available, importing it lazily.++ Returns:+ True if hnswlib is available and working.++ Note:+ This function caches the result in HNSW_AVAILABLE.+ On CPUs without AVX support, importing hnswlib may crash+ the process with SIGILL before we can catch the error.+ """+ global hnswlib, HNSW_AVAILABLE++ if HNSW_AVAILABLE is not None:+ return HNSW_AVAILABLE++ try:+ import hnswlib as _hnswlib++ hnswlib = _hnswlib+ HNSW_AVAILABLE = True+ except ImportError:+ HNSW_AVAILABLE = False++ return HNSW_AVAILABLE+if TYPE_CHECKING:pass@@ -187,12 +214,13 @@ class HNSWVectorIndex:ValueError: If auto_save is True but save_path is not provided.ImportError: If hnswlib is not installed."""- if not HNSW_AVAILABLE:+ if not _check_hnswlib_available():raise ImportError("hnswlib is required for HNSWVectorIndex. ""Install with: pip install hnswlib\n""Note: hnswlib requires C++ compilation and may not be "- "available on all platforms."+ "available on all platforms (crashes with SIGILL on CPUs "+ "without AVX support).")if auto_save and save_path is None:@@ -208,7 +236,8 @@ class HNSWVectorIndex:# Initialize HNSW index with cosine similarity# hnswlib uses 'cosine' space which internally normalizes vectors- self._index = hnswlib.Index(space="cosine", dim=dimension)+ # Note: hnswlib is guaranteed non-None here due to _check_hnswlib_available() above+ self._index = hnswlib.Index(space="cosine", dim=dimension) # type: ignore[union-attr]self._index.init_index(max_elements=max_elements,ef_construction=ef_construction,@@ -729,7 +758,7 @@ class HNSWVectorIndex:self._ef_search = meta_data["ef_search"]# Create new index and load from file- self._index = hnswlib.Index(space="cosine", dim=self._dimension)+ self._index = hnswlib.Index(space="cosine", dim=self._dimension) # type: ignore[union-attr]self._index.load_index(str(hnsw_path),max_elements=self._max_elements,@@ -757,7 +786,7 @@ class HNSWVectorIndex:"""Clear all entries from the index."""with self._lock:# Reinitialize the index- self._index = hnswlib.Index(space="cosine", dim=self._dimension)+ self._index = hnswlib.Index(space="cosine", dim=self._dimension) # type: ignore[union-attr]self._index.init_index(max_elements=self._max_elements,ef_construction=self._ef_construction,diff --git a/headroom/transforms/__init__.py b/headroom/transforms/__init__.pyindex 43ffea900..f0093d5e9 100644--- a/headroom/transforms/__init__.py+++ b/headroom/transforms/__init__.py@@ -11,6 +11,7 @@ from .anchor_selector import (from .base import Transformfrom .cache_aligner import CacheAlignerfrom .content_detector import ContentType, DetectionResult, detect_content_type+from .diff_compressor import DiffCompressionResult, DiffCompressor, DiffCompressorConfigfrom .intelligent_context import ContextStrategy, IntelligentContextManagerfrom .log_compressor import LogCompressionResult, LogCompressor, LogCompressorConfigfrom .pipeline import TransformPipeline@@ -84,6 +85,9 @@ __all__ = ["LogCompressor","LogCompressorConfig","LogCompressionResult",+ "DiffCompressor",+ "DiffCompressorConfig",+ "DiffCompressionResult","TextCompressor","TextCompressorConfig","TextCompressionResult",diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.pyindex ee90a114e..f0136b3a8 100644--- a/headroom/transforms/content_router.py+++ b/headroom/transforms/content_router.py@@ -461,6 +461,7 @@ class ContentRouter(Transform):self._smart_crusher: Any = Noneself._search_compressor: Any = Noneself._log_compressor: Any = None+ self._diff_compressor: Any = Noneself._llmlingua: Any = Noneself._text_compressor: Any = Noneself._image_optimizer: Any = None@@ -772,6 +773,15 @@ class ContentRouter(Transform):result.compressed_line_count,)+ elif strategy == CompressionStrategy.DIFF:+ compressor = self._get_diff_compressor()+ if compressor:+ result = compressor.compress(content, context=context)+ compressed, compressed_tokens = (+ result.compressed,+ result.compressed_line_count,+ )+elif strategy == CompressionStrategy.LLMLINGUA:compressed, compressed_tokens = self._try_llmlingua(content, context)@@ -907,6 +917,17 @@ class ContentRouter(Transform):logger.debug("LogCompressor not available")return self._log_compressor+ def _get_diff_compressor(self) -> Any:+ """Get DiffCompressor (lazy load)."""+ if self._diff_compressor is None:+ try:+ from .diff_compressor import DiffCompressor++ self._diff_compressor = DiffCompressor()+ except ImportError:+ logger.debug("DiffCompressor not available")+ return self._diff_compressor+def _get_llmlingua(self) -> Any:"""Get LLMLinguaCompressor (lazy load)."""if self._llmlingua is None:diff --git a/headroom/transforms/diff_compressor.py b/headroom/transforms/diff_compressor.pynew file mode 100644index 000000000..91b6aaf49--- /dev/null+++ b/headroom/transforms/diff_compressor.py@@ -0,0 +1,615 @@+"""Git diff output compressor for unified diff format.++This module compresses git diff output which can be very verbose with+many context lines. Typical compression: 3-10x.++Supported formats:+- Unified diff format (git diff, diff -u)+- Combined diff format (merge conflicts)++Compression Strategy:+1. Parse unified diff format into file sections and hunks+2. Always keep file headers (diff --git, ---, +++)+3. Always keep ALL actual changes (+/- lines)+4. Reduce context lines (` ` prefix) to configurable max+5. If too many hunks, keep first N and summarize rest+6. Add summary at end++Key Patterns to Preserve:+- All additions (+)+- All deletions (-)+- Hunk headers (@@ ... @@)+- File headers (diff --git, ---, +++)+- Context around changes (limited)+"""++from __future__ import annotations++import re+from dataclasses import dataclass, field+++@dataclass+class DiffHunk:+ """A single hunk within a diff file."""++ header: str # @@ -start,count +start,count @@ optional function+ lines: list[str] # All lines in the hunk+ additions: int = 0+ deletions: int = 0+ context_lines: int = 0+ score: float = 0.0 # Relevance score for context-aware compression++ @property+ def change_count(self) -> int:+ """Total number of actual changes (additions + deletions)."""+ return self.additions + self.deletions+++@dataclass+class DiffFile:+ """A single file's diff."""++ header: str # diff --git a/... b/...+ old_file: str # --- a/...+ new_file: str # +++ b/...+ hunks: list[DiffHunk] = field(default_factory=list)+ is_binary: bool = False+ is_new_file: bool = False+ is_deleted_file: bool = False+ is_renamed: bool = False++ @property+ def total_additions(self) -> int:+ return sum(h.additions for h in self.hunks)++ @property+ def total_deletions(self) -> int:+ return sum(h.deletions for h in self.hunks)+++@dataclass+class DiffCompressorConfig:+ """Configuration for diff compression."""++ # Context line limits+ max_context_lines: int = 2 # Reduce from default 3 lines before/after changes++ # Hunk limits+ max_hunks_per_file: int = 10++ # File limits+ max_files: int = 20++ # Change preservation+ always_keep_additions: bool = True # Always keep + lines+ always_keep_deletions: bool = True # Always keep - lines++ # CCR integration+ enable_ccr: bool = True+ min_lines_for_ccr: int = 50+++@dataclass+class DiffCompressionResult:+ """Result of diff compression."""++ compressed: str+ original_line_count: int+ compressed_line_count: int+ files_affected: int+ additions: int+ deletions: int+ hunks_kept: int+ hunks_removed: int+ cache_key: str | None = None++ @property+ def compression_ratio(self) -> float:+ """Ratio of compressed to original (lower is better compression)."""+ if self.original_line_count == 0:+ return 1.0+ return self.compressed_line_count / self.original_line_count++ @property+ def tokens_saved_estimate(self) -> int:+ """Estimate tokens saved (rough: 1 token per 4 chars)."""+ # Use line counts as proxy for chars+ lines_saved = self.original_line_count - self.compressed_line_count+ # Estimate ~40 chars per line average for diffs+ chars_saved = lines_saved * 40+ return max(0, chars_saved // 4)+++class DiffCompressor:+ """Compresses git diff output.++ Example:+ >>> compressor = DiffCompressor()+ >>> result = compressor.compress(git_diff_output)+ >>> print(result.compressed) # Reduced diff with summary+ """++ # Pattern for diff --git header+ _DIFF_GIT_PATTERN = re.compile(r"^diff --git a/(.+) b/(.+)$")++ # Pattern for --- a/file or --- /dev/null+ _OLD_FILE_PATTERN = re.compile(r"^--- (a/(.+)|/dev/null)$")++ # Pattern for +++ b/file or +++ /dev/null+ _NEW_FILE_PATTERN = re.compile(r"^\+\+\+ (b/(.+)|/dev/null)$")++ # Pattern for hunk header @@ -start,count +start,count @@ optional context+ _HUNK_HEADER_PATTERN = re.compile(r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)$")++ # Pattern for binary file indication+ _BINARY_PATTERN = re.compile(r"^Binary files .+ differ$")++ # Patterns for new/deleted file mode+ _NEW_FILE_MODE_PATTERN = re.compile(r"^new file mode")+ _DELETED_FILE_MODE_PATTERN = re.compile(r"^deleted file mode")+ _RENAME_PATTERN = re.compile(r"^(rename|similarity|copy) ")++ # Priority patterns for context-aware hunk selection+ _PRIORITY_PATTERNS = [+ re.compile(r"\b(error|exception|fail|bug|fix)\b", re.IGNORECASE),+ re.compile(r"\b(todo|fixme|hack|xxx)\b", re.IGNORECASE),+ re.compile(r"\b(security|auth|password|secret|token)\b", re.IGNORECASE),+ ]++ def __init__(self, config: DiffCompressorConfig | None = None):+ """Initialize diff compressor.++ Args:+ config: Compression configuration.+ """+ self.config = config or DiffCompressorConfig()++ def compress(self, content: str, context: str = "") -> DiffCompressionResult:+ """Compress diff output.++ Args:+ content: Raw git diff output.+ context: User query context for relevance scoring.++ Returns:+ DiffCompressionResult with compressed output and metadata.+ """+ lines = content.split("\n")+ original_line_count = len(lines)++ if original_line_count < self.config.min_lines_for_ccr:+ return DiffCompressionResult(+ compressed=content,+ original_line_count=original_line_count,+ compressed_line_count=original_line_count,+ files_affected=0,+ additions=0,+ deletions=0,+ hunks_kept=0,+ hunks_removed=0,+ )++ # Parse diff into structured format+ diff_files = self._parse_diff(lines)++ if not diff_files:+ return DiffCompressionResult(+ compressed=content,+ original_line_count=original_line_count,+ compressed_line_count=original_line_count,+ files_affected=0,+ additions=0,+ deletions=0,+ hunks_kept=0,+ hunks_removed=0,+ )++ # Score hunks by relevance+ self._score_hunks(diff_files, context)++ # Compress each file's hunks+ compressed_files, stats = self._compress_files(diff_files)++ # Format output+ compressed_output = self._format_output(compressed_files, stats)+ compressed_line_count = len(compressed_output.split("\n"))++ # Store in CCR if significant compression+ cache_key = None+ if self.config.enable_ccr and compressed_line_count < original_line_count * 0.8:+ cache_key = self._store_in_ccr(content, compressed_output, original_line_count)+ if cache_key:+ compressed_output += f"\n[{original_line_count} lines compressed to {compressed_line_count}. Retrieve full diff: hash={cache_key}]"++ return DiffCompressionResult(+ compressed=compressed_output,+ original_line_count=original_line_count,+ compressed_line_count=compressed_line_count,+ files_affected=stats["files_affected"],+ additions=stats["total_additions"],+ deletions=stats["total_deletions"],+ hunks_kept=stats["hunks_kept"],+ hunks_removed=stats["hunks_removed"],+ cache_key=cache_key,+ )++ def _parse_diff(self, lines: list[str]) -> list[DiffFile]:+ """Parse diff content into structured format.++ Args:+ lines: Lines of diff content.++ Returns:+ List of DiffFile objects.+ """+ diff_files: list[DiffFile] = []+ current_file: DiffFile | None = None+ current_hunk: DiffHunk | None = None+ i = 0++ while i < len(lines):+ line = lines[i]++ # Check for diff --git header (new file section)+ if self._DIFF_GIT_PATTERN.match(line):+ # Save previous hunk and file+ if current_hunk and current_file:+ current_file.hunks.append(current_hunk)+ if current_file:+ diff_files.append(current_file)++ current_file = DiffFile(+ header=line,+ old_file="",+ new_file="",+ )+ current_hunk = None+ i += 1+ continue++ # Check for file mode indicators+ if current_file:+ if self._NEW_FILE_MODE_PATTERN.match(line):+ current_file.is_new_file = True+ elif self._DELETED_FILE_MODE_PATTERN.match(line):+ current_file.is_deleted_file = True+ elif self._RENAME_PATTERN.match(line):+ current_file.is_renamed = True+ elif self._BINARY_PATTERN.match(line):+ current_file.is_binary = True++ # Check for --- a/file+ if self._OLD_FILE_PATTERN.match(line):+ if current_file:+ current_file.old_file = line+ i += 1+ continue++ # Check for +++ b/file+ if self._NEW_FILE_PATTERN.match(line):+ if current_file:+ current_file.new_file = line+ i += 1+ continue++ # Check for hunk header+ if self._HUNK_HEADER_PATTERN.match(line):+ # Save previous hunk+ if current_hunk and current_file:+ current_file.hunks.append(current_hunk)++ current_hunk = DiffHunk(+ header=line,+ lines=[],+ )+ i += 1+ continue++ # Process hunk content lines+ if current_hunk is not None:+ if line.startswith("+") and not line.startswith("+++"):+ current_hunk.additions += 1+ current_hunk.lines.append(line)+ elif line.startswith("-") and not line.startswith("---"):+ current_hunk.deletions += 1+ current_hunk.lines.append(line)+ elif line.startswith(" ") or line == "":+ current_hunk.context_lines += 1+ current_hunk.lines.append(line)+ else:+ # Other line (e.g., "\ No newline at end of file")+ current_hunk.lines.append(line)++ i += 1++ # Save final hunk and file+ if current_hunk and current_file:+ current_file.hunks.append(current_hunk)+ if current_file:+ diff_files.append(current_file)++ return diff_files++ def _score_hunks(self, diff_files: list[DiffFile], context: str) -> None:+ """Score hunks by relevance to context.++ Args:+ diff_files: Parsed diff files.+ context: User query context.+ """+ context_lower = context.lower()+ context_words = set(context_lower.split()) if context else set()++ for diff_file in diff_files:+ for hunk in diff_file.hunks:+ score = 0.0++ # Base score from change count (more changes = more important)+ score += min(0.3, hunk.change_count * 0.03)++ hunk_content = "\n".join(hunk.lines).lower()++ # Score by context word overlap+ for word in context_words:+ if len(word) > 2 and word in hunk_content:+ score += 0.2++ # Boost for priority patterns+ for pattern in self._PRIORITY_PATTERNS:+ if pattern.search(hunk_content):+ score += 0.3+ break++ hunk.score = min(1.0, score)++ def _compress_files(self, diff_files: list[DiffFile]) -> tuple[list[DiffFile], dict[str, int]]:+ """Compress hunks in each file.++ Args:+ diff_files: Parsed diff files.++ Returns:+ Tuple of (compressed files, stats dict).+ """+ stats = {+ "files_affected": 0,+ "total_additions": 0,+ "total_deletions": 0,+ "hunks_kept": 0,+ "hunks_removed": 0,+ }++ # Limit files if too many+ if len(diff_files) > self.config.max_files:+ # Sort by total changes (most changes first)+ diff_files = sorted(+ diff_files,+ key=lambda f: f.total_additions + f.total_deletions,+ reverse=True,+ )+ diff_files = diff_files[: self.config.max_files]++ compressed_files: list[DiffFile] = []++ for diff_file in diff_files:+ stats["files_affected"] += 1+ stats["total_additions"] += diff_file.total_additions+ stats["total_deletions"] += diff_file.total_deletions++ # Compress hunks within file+ compressed_hunks = self._compress_hunks(diff_file.hunks)++ stats["hunks_kept"] += len(compressed_hunks)+ stats["hunks_removed"] += len(diff_file.hunks) - len(compressed_hunks)++ # Create compressed file with reduced context in hunks+ new_file = DiffFile(+ header=diff_file.header,+ old_file=diff_file.old_file,+ new_file=diff_file.new_file,+ hunks=compressed_hunks,+ is_binary=diff_file.is_binary,+ is_new_file=diff_file.is_new_file,+ is_deleted_file=diff_file.is_deleted_file,+ is_renamed=diff_file.is_renamed,+ )+ compressed_files.append(new_file)++ return compressed_files, stats++ def _compress_hunks(self, hunks: list[DiffHunk]) -> list[DiffHunk]:+ """Compress hunks by reducing context and limiting count.++ Args:+ hunks: List of hunks to compress.++ Returns:+ Compressed list of hunks.+ """+ if not hunks:+ return []++ # Sort by score if we need to limit+ if len(hunks) > self.config.max_hunks_per_file:+ # Keep first and last hunks (often important)+ first_hunk = hunks[0]+ last_hunk = hunks[-1] if len(hunks) > 1 else None++ # Sort middle hunks by score+ middle_hunks = sorted(+ hunks[1:-1] if last_hunk else [], key=lambda h: h.score, reverse=True+ )++ # Take top scoring middle hunks+ remaining_slots = (+ self.config.max_hunks_per_file - 2+ if last_hunk+ else self.config.max_hunks_per_file - 1+ )+ selected_middle = middle_hunks[:remaining_slots]++ # Rebuild list in original order by re-sorting by appearance+ selected = [first_hunk] + selected_middle+ if last_hunk:+ selected.append(last_hunk)++ # Sort back to original order (using header line numbers as proxy)+ hunks = sorted(selected, key=lambda h: self._extract_line_number(h.header))++ # Reduce context in each hunk+ compressed_hunks = []+ for hunk in hunks:+ compressed_hunk = self._reduce_context(hunk)+ compressed_hunks.append(compressed_hunk)++ return compressed_hunks++ def _extract_line_number(self, header: str) -> int:+ """Extract starting line number from hunk header for sorting."""+ match = self._HUNK_HEADER_PATTERN.match(header)+ if match:+ return int(match.group(1))+ return 0++ def _reduce_context(self, hunk: DiffHunk) -> DiffHunk:+ """Reduce context lines while preserving all changes.++ Args:+ hunk: Hunk to reduce context in.++ Returns:+ New hunk with reduced context.+ """+ max_context = self.config.max_context_lines++ # Identify change positions+ change_positions: list[int] = []+ for i, line in enumerate(hunk.lines):+ if line.startswith("+") or line.startswith("-"):+ change_positions.append(i)++ if not change_positions:+ # No changes, just context - keep minimal+ return DiffHunk(+ header=hunk.header,+ lines=hunk.lines[:max_context] if hunk.lines else [],+ additions=0,+ deletions=0,+ context_lines=min(len(hunk.lines), max_context),+ score=hunk.score,+ )++ # Determine which lines to keep+ keep_indices: set[int] = set()++ for pos in change_positions:+ # Always keep the change line+ keep_indices.add(pos)++ # Keep context before+ for i in range(max(0, pos - max_context), pos):+ keep_indices.add(i)++ # Keep context after+ for i in range(pos + 1, min(len(hunk.lines), pos + max_context + 1)):+ keep_indices.add(i)++ # Build new lines list+ new_lines: list[str] = []+ additions = 0+ deletions = 0+ context_lines = 0++ for i in sorted(keep_indices):+ line = hunk.lines[i]+ new_lines.append(line)+ if line.startswith("+"):+ additions += 1+ elif line.startswith("-"):+ deletions += 1+ else:+ context_lines += 1++ return DiffHunk(+ header=hunk.header,+ lines=new_lines,+ additions=additions,+ deletions=deletions,+ context_lines=context_lines,+ score=hunk.score,+ )++ def _format_output(self, diff_files: list[DiffFile], stats: dict[str, int]) -> str:+ """Format compressed diff files back to unified diff format.++ Args:+ diff_files: Compressed diff files.+ stats: Compression statistics.++ Returns:+ Formatted diff string.+ """+ output_lines: list[str] = []++ for diff_file in diff_files:+ # File header+ output_lines.append(diff_file.header)++ # File mode indicators if present+ if diff_file.is_new_file:+ output_lines.append("new file mode 100644")+ elif diff_file.is_deleted_file:+ output_lines.append("deleted file mode 100644")++ if diff_file.is_binary:+ output_lines.append("Binary files differ")+ continue++ # Old/new file markers+ if diff_file.old_file:+ output_lines.append(diff_file.old_file)+ if diff_file.new_file:+ output_lines.append(diff_file.new_file)++ # Hunks+ for hunk in diff_file.hunks:+ output_lines.append(hunk.header)+ output_lines.extend(hunk.lines)++ # Add summary+ if stats["hunks_removed"] > 0 or stats["files_affected"] > 0:+ summary_parts = [+ f"{stats['files_affected']} files changed",+ f"+{stats['total_additions']} -{stats['total_deletions']} lines",+ ]+ if stats["hunks_removed"] > 0:+ summary_parts.append(f"{stats['hunks_removed']} hunks omitted")+ output_lines.append(f"[{', '.join(summary_parts)}]")++ return "\n".join(output_lines)++ def _store_in_ccr(self, original: str, compressed: str, original_count: int) -> str | None:+ """Store original in CCR for later retrieval.++ Args:+ original: Original diff content.+ compressed: Compressed diff content.+ original_count: Original line count.++ Returns:+ Cache key if stored, None otherwise.+ """+ try:+ from ..cache.compression_store import get_compression_store++ store = get_compression_store()+ return store.store(+ original,+ compressed,+ original_item_count=original_count,+ )+ except ImportError:+ return None+ except Exception:+ return Nonediff --git a/tests/test_hnsw_only.py b/tests/test_hnsw_only.pyindex c33d9e37b..3e2c9e179 100644--- a/tests/test_hnsw_only.py+++ b/tests/test_hnsw_only.py@@ -13,9 +13,11 @@ import pytestfrom headroom.memory.models import Memoryfrom headroom.memory.ports import VectorFilter-# Check if hnswlib is available+# Check if hnswlib is available (use lazy check to avoid SIGILL on incompatible CPUs)try:- from headroom.memory.adapters.hnsw import HNSW_AVAILABLE+ from headroom.memory.adapters.hnsw import _check_hnswlib_available++ HNSW_AVAILABLE = _check_hnswlib_available()except ImportError:HNSW_AVAILABLE = Falsediff --git a/tests/test_memory/test_hierarchical.py b/tests/test_memory/test_hierarchical.pyindex 0b567f0da..afe86fbd0 100644--- a/tests/test_memory/test_hierarchical.py+++ b/tests/test_memory/test_hierarchical.py@@ -608,9 +608,11 @@ class TestIntegration:# HNSW Vector Index Tests# =============================================================================-# Check if hnswlib is available+# Check if hnswlib is available (use lazy check to avoid SIGILL on incompatible CPUs)try:- from headroom.memory.adapters.hnsw import HNSW_AVAILABLE+ from headroom.memory.adapters.hnsw import _check_hnswlib_available++ HNSW_AVAILABLE = _check_hnswlib_available()except ImportError:HNSW_AVAILABLE = Falsediff --git a/tests/test_transforms/test_diff_compressor.py b/tests/test_transforms/test_diff_compressor.pynew file mode 100644index 000000000..323e7ca3e--- /dev/null+++ b/tests/test_transforms/test_diff_compressor.py@@ -0,0 +1,673 @@+"""Comprehensive tests for diff_compressor.py.++Tests cover:+1. Parsing of unified diff format+2. Context line reduction+3. Hunk selection and limiting+4. Compression ratios+5. Edge cases+"""++from headroom.transforms.diff_compressor import (+ DiffCompressionResult,+ DiffCompressor,+ DiffCompressorConfig,+ DiffFile,+ DiffHunk,+)+++class TestDiffParsing:+ """Tests for parsing unified diff format."""++ def test_parse_simple_diff(self):+ """Simple single-file diff is parsed correctly."""+ content = """diff --git a/src/main.py b/src/main.py+--- a/src/main.py++++ b/src/main.py+@@ -10,6 +10,7 @@ def main():+ print("hello")++ print("world")+ return 0+"""+ compressor = DiffCompressor()+ diff_files = compressor._parse_diff(content.split("\n"))++ assert len(diff_files) == 1+ assert diff_files[0].header == "diff --git a/src/main.py b/src/main.py"+ assert diff_files[0].old_file == "--- a/src/main.py"+ assert diff_files[0].new_file == "+++ b/src/main.py"+ assert len(diff_files[0].hunks) == 1+ assert diff_files[0].hunks[0].additions == 1+ assert diff_files[0].hunks[0].deletions == 0++ def test_parse_multi_file_diff(self):+ """Multi-file diff is parsed into separate DiffFile objects."""+ content = """diff --git a/file1.py b/file1.py+--- a/file1.py++++ b/file1.py+@@ -1,3 +1,4 @@+ line1++added line+ line2+diff --git a/file2.py b/file2.py+--- a/file2.py++++ b/file2.py+@@ -5,4 +5,3 @@+ keep+-removed+ keep2+"""+ compressor = DiffCompressor()+ diff_files = compressor._parse_diff(content.split("\n"))++ assert len(diff_files) == 2+ assert "file1.py" in diff_files[0].header+ assert "file2.py" in diff_files[1].header+ assert diff_files[0].hunks[0].additions == 1+ assert diff_files[0].hunks[0].deletions == 0+ assert diff_files[1].hunks[0].additions == 0+ assert diff_files[1].hunks[0].deletions == 1++ def test_parse_multi_hunk_file(self):+ """File with multiple hunks is parsed correctly."""+ content = """diff --git a/src/utils.py b/src/utils.py+--- a/src/utils.py++++ b/src/utils.py+@@ -10,4 +10,5 @@ def helper():+ pass++ # added comment+ return True+@@ -50,3 +51,4 @@ def other():+ x = 1++ y = 2+ return x+"""+ compressor = DiffCompressor()+ diff_files = compressor._parse_diff(content.split("\n"))++ assert len(diff_files) == 1+ assert len(diff_files[0].hunks) == 2+ assert diff_files[0].total_additions == 2++ def test_parse_new_file(self):+ """New file diff is detected."""+ content = """diff --git a/newfile.py b/newfile.py+new file mode 100644+--- /dev/null++++ b/newfile.py+@@ -0,0 +1,3 @@++def new_func():++ pass++ return None+"""+ compressor = DiffCompressor()+ diff_files = compressor._parse_diff(content.split("\n"))++ assert len(diff_files) == 1+ assert diff_files[0].is_new_file is True+ assert diff_files[0].hunks[0].additions == 3++ def test_parse_deleted_file(self):+ """Deleted file diff is detected."""+ content = """diff --git a/oldfile.py b/oldfile.py+deleted file mode 100644+--- a/oldfile.py++++ /dev/null+@@ -1,2 +0,0 @@+-def old_func():+- pass+"""+ compressor = DiffCompressor()+ diff_files = compressor._parse_diff(content.split("\n"))++ assert len(diff_files) == 1+ assert diff_files[0].is_deleted_file is True+ assert diff_files[0].hunks[0].deletions == 2++ def test_parse_binary_file(self):+ """Binary file diff is detected."""+ content = """diff --git a/image.png b/image.png+Binary files a/image.png and b/image.png differ+"""+ compressor = DiffCompressor()+ diff_files = compressor._parse_diff(content.split("\n"))++ assert len(diff_files) == 1+ assert diff_files[0].is_binary is True+++class TestContextReduction:+ """Tests for context line reduction."""++ def test_reduce_context_lines(self):+ """Context lines are reduced to configured maximum."""+ content = """diff --git a/file.py b/file.py+--- a/file.py++++ b/file.py+@@ -1,10 +1,11 @@+ context1+ context2+ context3+ context4++added+ context5+ context6+ context7+ context8+"""+ # Default max_context_lines is 2+ compressor = DiffCompressor(+ config=DiffCompressorConfig(+ max_context_lines=2,+ min_lines_for_ccr=5,+ enable_ccr=False,+ )+ )+ result = compressor.compress(content)++ # Should keep 2 context before and 2 after the +added line+ # Plus the added line itself+ lines = result.compressed.split("\n")+ context_count = sum(1 for line in lines if line.startswith(" "))++ # At most 4 context lines (2 before + 2 after)+ assert context_count <= 4++ def test_preserve_all_changes(self):+ """All addition and deletion lines are preserved."""+ content = """diff --git a/file.py b/file.py+--- a/file.py++++ b/file.py+@@ -1,10 +1,10 @@+ ctx1+ ctx2+-removed1++added1+ ctx3+ ctx4+-removed2++added2+ ctx5+ ctx6+"""+ compressor = DiffCompressor(+ config=DiffCompressorConfig(+ min_lines_for_ccr=5,+ enable_ccr=False,+ )+ )+ result = compressor.compress(content)++ assert "-removed1" in result.compressed+ assert "-removed2" in result.compressed+ assert "+added1" in result.compressed+ assert "+added2" in result.compressed+++class TestHunkSelection:+ """Tests for hunk selection when limiting."""++ def test_max_hunks_per_file(self):+ """Hunks are limited to max_hunks_per_file."""+ # Create a diff with many hunks+ hunks = []+ for i in range(20):+ hunks.append(f"""@@ -{i * 10},3 +{i * 10},4 @@+ context++added_{i}+ more+""")++ content = f"""diff --git a/bigfile.py b/bigfile.py+--- a/bigfile.py++++ b/bigfile.py+{"".join(hunks)}"""++ compressor = DiffCompressor(+ config=DiffCompressorConfig(+ max_hunks_per_file=5,+ min_lines_for_ccr=10,+ enable_ccr=False,+ )+ )+ result = compressor.compress(content)++ # Should have at most 5 hunks+ hunk_count = result.compressed.count("@@")+ # Each hunk has one @@ header (we count full hunk headers)+ assert hunk_count <= 10 # Each hunk header appears twice @@...@@++ def test_keeps_first_and_last_hunk(self):+ """First and last hunks are preserved when limiting."""+ hunks = []+ for i in range(10):+ hunks.append(f"""@@ -{i * 10},3 +{i * 10},4 @@+ context++added_{i}+ more+""")++ content = f"""diff --git a/file.py b/file.py+--- a/file.py++++ b/file.py+{"".join(hunks)}"""++ compressor = DiffCompressor(+ config=DiffCompressorConfig(+ max_hunks_per_file=3,+ min_lines_for_ccr=10,+ enable_ccr=False,+ )+ )+ result = compressor.compress(content)++ # First hunk (added_0) should be present+ assert "+added_0" in result.compressed+ # Last hunk (added_9) should be present+ assert "+added_9" in result.compressed+++class TestFileSelection:+ """Tests for file selection when limiting."""++ def test_max_files(self):+ """Files are limited to max_files."""+ # Create diff with many files+ files = []+ for i in range(30):+ files.append(f"""diff --git a/file{i}.py b/file{i}.py+--- a/file{i}.py++++ b/file{i}.py+@@ -1,2 +1,3 @@+ ctx++added+ ctx2+""")++ content = "\n".join(files)++ compressor = DiffCompressor(+ config=DiffCompressorConfig(+ max_files=10,+ min_lines_for_ccr=20,+ enable_ccr=False,+ )+ )+ result = compressor.compress(content)++ # Count diff --git headers+ file_count = result.compressed.count("diff --git")+ assert file_count <= 10+++class TestCompressionResult:+ """Tests for DiffCompressionResult properties."""++ def test_compression_ratio_calculation(self):+ """Compression ratio is calculated correctly."""+ result = DiffCompressionResult(+ compressed="a\nb\nc",+ original_line_count=100,+ compressed_line_count=10,+ files_affected=2,+ additions=5,+ deletions=3,+ hunks_kept=2,+ hunks_removed=5,+ )++ assert result.compression_ratio == 0.1++ def test_tokens_saved_estimate(self):+ """Token savings estimation works correctly."""+ result = DiffCompressionResult(+ compressed="short",+ original_line_count=100,+ compressed_line_count=10,+ files_affected=1,+ additions=10,+ deletions=5,+ hunks_kept=1,+ hunks_removed=0,+ )++ # 90 lines saved * 40 chars/line / 4 chars/token = 900 tokens+ assert result.tokens_saved_estimate == 900+++class TestHunkScoring:+ """Tests for context-aware hunk scoring."""++ def test_score_by_context_keywords(self):+ """Hunks containing context keywords get higher scores."""+ content = """diff --git a/file.py b/file.py+--- a/file.py++++ b/file.py+@@ -1,3 +1,4 @@+ normal context++normal change+ more context+@@ -10,3 +11,4 @@+ error handling++fix the bug here+ return result+"""+ compressor = DiffCompressor()+ diff_files = compressor._parse_diff(content.split("\n"))+ compressor._score_hunks(diff_files, "fix error bug")++ # Second hunk should have higher score (contains "fix" and "bug")+ assert len(diff_files[0].hunks) == 2+ assert diff_files[0].hunks[1].score > diff_files[0].hunks[0].score++ def test_score_priority_patterns(self):+ """Hunks with priority patterns (error, security) score higher."""+ compressor = DiffCompressor()++ hunk_normal = DiffHunk(+ header="@@ -1,1 +1,2 @@",+ lines=["+normal change"],+ additions=1,+ )+ hunk_error = DiffHunk(+ header="@@ -10,1 +10,2 @@",+ lines=["+fix critical error"],+ additions=1,+ )++ diff_file = DiffFile(+ header="diff --git a/f.py b/f.py",+ old_file="--- a/f.py",+ new_file="+++ b/f.py",+ hunks=[hunk_normal, hunk_error],+ )++ compressor._score_hunks([diff_file], "")++ assert hunk_error.score > hunk_normal.score+++class TestSmallDiffPassthrough:+ """Tests for small diff passthrough behavior."""++ def test_small_diff_unchanged(self):+ """Diffs smaller than threshold pass through unchanged."""+ content = """diff --git a/small.py b/small.py+--- a/small.py++++ b/small.py+@@ -1,2 +1,3 @@+ line1++added+ line2+"""+ compressor = DiffCompressor(+ config=DiffCompressorConfig(+ min_lines_for_ccr=100, # High threshold+ )+ )+ result = compressor.compress(content)++ # Should be unchanged+ assert result.compressed == content+ assert result.compression_ratio == 1.0+++class TestOutputFormatting:+ """Tests for output formatting."""++ def test_summary_line_added(self):+ """Summary line is added at end of compressed diff."""+ # Large diff that will be compressed+ hunks = []+ for i in range(15):+ hunks.append(f"""@@ -{i * 10},5 +{i * 10},6 @@+ ctx1+ ctx2++added_{i}+ ctx3+ ctx4+""")++ content = f"""diff --git a/file.py b/file.py+--- a/file.py++++ b/file.py+{"".join(hunks)}"""++ compressor = DiffCompressor(+ config=DiffCompressorConfig(+ max_hunks_per_file=5,+ min_lines_for_ccr=10,+ enable_ccr=False,+ )+ )+ result = compressor.compress(content)++ # Should have summary at end+ assert "files changed" in result.compressed+ assert "hunks omitted" in result.compressed++ def test_preserves_diff_format(self):+ """Output preserves valid unified diff format."""+ content = """diff --git a/test.py b/test.py+--- a/test.py++++ b/test.py+@@ -1,3 +1,4 @@+ def test():++ # new comment+ pass+ return True+"""+ compressor = DiffCompressor(+ config=DiffCompressorConfig(+ min_lines_for_ccr=5,+ enable_ccr=False,+ )+ )+ result = compressor.compress(content)++ # Should have all standard diff markers+ assert "diff --git" in result.compressed+ assert "---" in result.compressed+ assert "+++" in result.compressed+ assert "@@" in result.compressed+++class TestEdgeCases:+ """Tests for edge cases and boundary conditions."""++ def test_empty_input(self):+ """Empty input is handled gracefully."""+ compressor = DiffCompressor()+ result = compressor.compress("")++ assert result.compressed == ""+ assert result.compression_ratio == 1.0++ def test_non_diff_input(self):+ """Non-diff input passes through unchanged."""+ content = "This is not a diff\nJust regular text"+ compressor = DiffCompressor()+ result = compressor.compress(content)++ # Should pass through (no diff --git found)+ assert result.compressed == content++ def test_unicode_content(self):+ """Unicode characters in diff are handled."""+ content = """diff --git a/i18n.py b/i18n.py+--- a/i18n.py++++ b/i18n.py+@@ -1,2 +1,3 @@+ msg = "hello"++msg_ja = "こんにちは"+ return msg+"""+ compressor = DiffCompressor()+ result = compressor.compress(content)++ assert "こんにちは" in result.compressed++ def test_no_newline_at_eof(self):+ """Handles 'No newline at end of file' indicator."""+ content = """diff --git a/file.py b/file.py+--- a/file.py++++ b/file.py+@@ -1,2 +1,2 @@+ line1+-line2+\\ No newline at end of file++line2_modified+\\ No newline at end of file+"""+ compressor = DiffCompressor()+ result = compressor.compress(content)++ # Should not crash and preserve the indicator+ assert "No newline" in result.compressed or "-line2" in result.compressed++ def test_empty_hunks(self):+ """Files with no actual hunks are handled."""+ content = """diff --git a/file.py b/file.py+--- a/file.py++++ b/file.py+"""+ compressor = DiffCompressor()+ result = compressor.compress(content)++ # Should not crash+ assert result.compressed is not None+++class TestDiffHunkDataclass:+ """Tests for DiffHunk dataclass."""++ def test_change_count_property(self):+ """change_count returns sum of additions and deletions."""+ hunk = DiffHunk(+ header="@@ -1,5 +1,6 @@",+ lines=["+a", "+b", "-c", " ctx"],+ additions=2,+ deletions=1,+ )+ assert hunk.change_count == 3++ def test_default_values(self):+ """DiffHunk default values are correct."""+ hunk = DiffHunk(header="@@", lines=[])+ assert hunk.additions == 0+ assert hunk.deletions == 0+ assert hunk.context_lines == 0+ assert hunk.score == 0.0+++class TestDiffFileDataclass:+ """Tests for DiffFile dataclass."""++ def test_total_additions_property(self):+ """total_additions sums across all hunks."""+ hunk1 = DiffHunk(header="@@", lines=[], additions=3)+ hunk2 = DiffHunk(header="@@", lines=[], additions=5)+ diff_file = DiffFile(+ header="diff --git",+ old_file="---",+ new_file="+++",+ hunks=[hunk1, hunk2],+ )+ assert diff_file.total_additions == 8++ def test_total_deletions_property(self):+ """total_deletions sums across all hunks."""+ hunk1 = DiffHunk(header="@@", lines=[], deletions=2)+ hunk2 = DiffHunk(header="@@", lines=[], deletions=4)+ diff_file = DiffFile(+ header="diff --git",+ old_file="---",+ new_file="+++",+ hunks=[hunk1, hunk2],+ )+ assert diff_file.total_deletions == 6+++class TestConfigOptions:+ """Tests for configuration options."""++ def test_max_context_lines_config(self):+ """max_context_lines configuration controls context reduction."""+ content = """diff --git a/file.py b/file.py+--- a/file.py++++ b/file.py+@@ -1,10 +1,11 @@+ c1+ c2+ c3+ c4+ c5++added+ c6+ c7+ c8+ c9+ c10+"""+ # With max_context_lines=1+ compressor = DiffCompressor(+ config=DiffCompressorConfig(+ max_context_lines=1,+ min_lines_for_ccr=5,+ enable_ccr=False,+ )+ )+ result = compressor.compress(content)++ # Count context lines (lines starting with space)+ context_count = sum(1 for line in result.compressed.split("\n") if line.startswith(" "))++ # Should have at most 2 context lines (1 before + 1 after)+ assert context_count <= 2++ def test_always_keep_additions_default(self):+ """Additions are always kept by default."""+ content = """diff --git a/file.py b/file.py+--- a/file.py++++ b/file.py+@@ -1,3 +1,5 @@+ ctx++add1++add2+ ctx+"""+ compressor = DiffCompressor(+ config=DiffCompressorConfig(+ always_keep_additions=True,+ min_lines_for_ccr=2,+ enable_ccr=False,+ )+ )+ result = compressor.compress(content)++ assert "+add1" in result.compressed+ assert "+add2" in result.compressed++ def test_always_keep_deletions_default(self):+ """Deletions are always kept by default."""+ content = """diff --git a/file.py b/file.py+--- a/file.py++++ b/file.py+@@ -1,5 +1,3 @@+ ctx+-del1+-del2+ ctx+"""+ compressor = DiffCompressor(+ config=DiffCompressorConfig(+ always_keep_deletions=True,+ min_lines_for_ccr=2,+ enable_ccr=False,+ )+ )+ result = compressor.compress(content)++ assert "-del1" in result.compressed+ assert "-del2" in result.compressed