Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

Add DiffCompressor and fix hnswlib SIGILL crash on CI

chopratejas5c740ea427a1
diff --git a/headroom/memory/adapters/__init__.py b/headroom/memory/adapters/__init__.py
index 160823f33..d0dc11bb8 100644
--- a/headroom/memory/adapters/__init__.py
+++ b/headroom/memory/adapters/__init__.py
@@ -21,21 +21,31 @@ from headroom.memory.adapters.graph import InMemoryGraphStore
from headroom.memory.adapters.sqlite import SQLiteMemoryStore
# Check for optional dependencies availability
-try:
- from headroom.memory.adapters.hnsw import HNSW_AVAILABLE
-except ImportError:
- HNSW_AVAILABLE = False
+# Note: We don't import from hnsw.py here because hnswlib may crash with
+# "Illegal instruction" on CPUs without required instructions (e.g., AVX).
+# Instead, we check lazily when HNSWVectorIndex is actually used.
+# HNSW_AVAILABLE is handled through __getattr__ to ensure lazy checking.
# Lazy imports for optional adapters
+_HNSW_AVAILABLE: bool | None = None # Internal cache for HNSW_AVAILABLE
_HNSWVectorIndex = None
_LocalEmbedder = None
_OpenAIEmbedder = None
_OllamaEmbedder = None
-def __getattr__(name: str) -> type:
+def __getattr__(name: str) -> type | bool:
"""Lazy import for optional adapters."""
global _HNSWVectorIndex, _LocalEmbedder, _OpenAIEmbedder, _OllamaEmbedder
+ global _HNSW_AVAILABLE
+
+ if name == "HNSW_AVAILABLE":
+ # Lazily check hnswlib availability
+ if _HNSW_AVAILABLE is None:
+ from headroom.memory.adapters.hnsw import _check_hnswlib_available
+
+ _HNSW_AVAILABLE = _check_hnswlib_available()
+ return _HNSW_AVAILABLE
if name == "HNSWVectorIndex":
if _HNSWVectorIndex is None:
diff --git a/headroom/memory/adapters/hnsw.py b/headroom/memory/adapters/hnsw.py
index 22bc4f4a2..33f95eeb4 100644
--- a/headroom/memory/adapters/hnsw.py
+++ b/headroom/memory/adapters/hnsw.py
@@ -22,17 +22,44 @@ from typing import TYPE_CHECKING, Any
import numpy as np
+from ..models import Memory, ScopeLevel
+from ..ports import VectorFilter, VectorSearchResult
+
# hnswlib is optional - may not compile on all platforms
-try:
- import hnswlib
+# NOTE: We don't import hnswlib at module level because it can crash with SIGILL
+# (Illegal Instruction) on CPUs without required AVX instructions. The crash
+# happens at the C level before Python's try/except can catch it.
+# Instead, we import lazily when HNSWVectorIndex is actually instantiated.
+hnswlib: Any = None # Will be imported lazily
+HNSW_AVAILABLE: bool | None = None # None = not yet checked, True/False = checked
- HNSW_AVAILABLE = True
-except ImportError:
- hnswlib = None # type: ignore[assignment]
- HNSW_AVAILABLE = False
-from ..models import Memory, ScopeLevel
-from ..ports import VectorFilter, VectorSearchResult
+def _check_hnswlib_available() -> bool:
+ """Check if hnswlib is available, importing it lazily.
+
+ Returns:
+ True if hnswlib is available and working.
+
+ Note:
+ This function caches the result in HNSW_AVAILABLE.
+ On CPUs without AVX support, importing hnswlib may crash
+ the process with SIGILL before we can catch the error.
+ """
+ global hnswlib, HNSW_AVAILABLE
+
+ if HNSW_AVAILABLE is not None:
+ return HNSW_AVAILABLE
+
+ try:
+ import hnswlib as _hnswlib
+
+ hnswlib = _hnswlib
+ HNSW_AVAILABLE = True
+ except ImportError:
+ HNSW_AVAILABLE = False
+
+ return HNSW_AVAILABLE
+
if TYPE_CHECKING:
pass
@@ -187,12 +214,13 @@ class HNSWVectorIndex:
ValueError: If auto_save is True but save_path is not provided.
ImportError: If hnswlib is not installed.
"""
- if not HNSW_AVAILABLE:
+ if not _check_hnswlib_available():
raise ImportError(
"hnswlib is required for HNSWVectorIndex. "
"Install with: pip install hnswlib\n"
"Note: hnswlib requires C++ compilation and may not be "
- "available on all platforms."
+ "available on all platforms (crashes with SIGILL on CPUs "
+ "without AVX support)."
)
if auto_save and save_path is None:
@@ -208,7 +236,8 @@ class HNSWVectorIndex:
# Initialize HNSW index with cosine similarity
# hnswlib uses 'cosine' space which internally normalizes vectors
- self._index = hnswlib.Index(space="cosine", dim=dimension)
+ # Note: hnswlib is guaranteed non-None here due to _check_hnswlib_available() above
+ self._index = hnswlib.Index(space="cosine", dim=dimension) # type: ignore[union-attr]
self._index.init_index(
max_elements=max_elements,
ef_construction=ef_construction,
@@ -729,7 +758,7 @@ class HNSWVectorIndex:
self._ef_search = meta_data["ef_search"]
# Create new index and load from file
- self._index = hnswlib.Index(space="cosine", dim=self._dimension)
+ self._index = hnswlib.Index(space="cosine", dim=self._dimension) # type: ignore[union-attr]
self._index.load_index(
str(hnsw_path),
max_elements=self._max_elements,
@@ -757,7 +786,7 @@ class HNSWVectorIndex:
"""Clear all entries from the index."""
with self._lock:
# Reinitialize the index
- self._index = hnswlib.Index(space="cosine", dim=self._dimension)
+ self._index = hnswlib.Index(space="cosine", dim=self._dimension) # type: ignore[union-attr]
self._index.init_index(
max_elements=self._max_elements,
ef_construction=self._ef_construction,
diff --git a/headroom/transforms/__init__.py b/headroom/transforms/__init__.py
index 43ffea900..f0093d5e9 100644
--- a/headroom/transforms/__init__.py
+++ b/headroom/transforms/__init__.py
@@ -11,6 +11,7 @@ from .anchor_selector import (
from .base import Transform
from .cache_aligner import CacheAligner
from .content_detector import ContentType, DetectionResult, detect_content_type
+from .diff_compressor import DiffCompressionResult, DiffCompressor, DiffCompressorConfig
from .intelligent_context import ContextStrategy, IntelligentContextManager
from .log_compressor import LogCompressionResult, LogCompressor, LogCompressorConfig
from .pipeline import TransformPipeline
@@ -84,6 +85,9 @@ __all__ = [
"LogCompressor",
"LogCompressorConfig",
"LogCompressionResult",
+ "DiffCompressor",
+ "DiffCompressorConfig",
+ "DiffCompressionResult",
"TextCompressor",
"TextCompressorConfig",
"TextCompressionResult",
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py
index ee90a114e..f0136b3a8 100644
--- a/headroom/transforms/content_router.py
+++ b/headroom/transforms/content_router.py
@@ -461,6 +461,7 @@ class ContentRouter(Transform):
self._smart_crusher: Any = None
self._search_compressor: Any = None
self._log_compressor: Any = None
+ self._diff_compressor: Any = None
self._llmlingua: Any = None
self._text_compressor: Any = None
self._image_optimizer: Any = None
@@ -772,6 +773,15 @@ class ContentRouter(Transform):
result.compressed_line_count,
)
+ elif strategy == CompressionStrategy.DIFF:
+ compressor = self._get_diff_compressor()
+ if compressor:
+ result = compressor.compress(content, context=context)
+ compressed, compressed_tokens = (
+ result.compressed,
+ result.compressed_line_count,
+ )
+
elif strategy == CompressionStrategy.LLMLINGUA:
compressed, compressed_tokens = self._try_llmlingua(content, context)
@@ -907,6 +917,17 @@ class ContentRouter(Transform):
logger.debug("LogCompressor not available")
return self._log_compressor
+ def _get_diff_compressor(self) -> Any:
+ """Get DiffCompressor (lazy load)."""
+ if self._diff_compressor is None:
+ try:
+ from .diff_compressor import DiffCompressor
+
+ self._diff_compressor = DiffCompressor()
+ except ImportError:
+ logger.debug("DiffCompressor not available")
+ return self._diff_compressor
+
def _get_llmlingua(self) -> Any:
"""Get LLMLinguaCompressor (lazy load)."""
if self._llmlingua is None:
diff --git a/headroom/transforms/diff_compressor.py b/headroom/transforms/diff_compressor.py
new file mode 100644
index 000000000..91b6aaf49
--- /dev/null
+++ b/headroom/transforms/diff_compressor.py
@@ -0,0 +1,615 @@
+"""Git diff output compressor for unified diff format.
+
+This module compresses git diff output which can be very verbose with
+many context lines. Typical compression: 3-10x.
+
+Supported formats:
+- Unified diff format (git diff, diff -u)
+- Combined diff format (merge conflicts)
+
+Compression Strategy:
+1. Parse unified diff format into file sections and hunks
+2. Always keep file headers (diff --git, ---, +++)
+3. Always keep ALL actual changes (+/- lines)
+4. Reduce context lines (` ` prefix) to configurable max
+5. If too many hunks, keep first N and summarize rest
+6. Add summary at end
+
+Key Patterns to Preserve:
+- All additions (+)
+- All deletions (-)
+- Hunk headers (@@ ... @@)
+- File headers (diff --git, ---, +++)
+- Context around changes (limited)
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+
+@dataclass
+class DiffHunk:
+ """A single hunk within a diff file."""
+
+ header: str # @@ -start,count +start,count @@ optional function
+ lines: list[str] # All lines in the hunk
+ additions: int = 0
+ deletions: int = 0
+ context_lines: int = 0
+ score: float = 0.0 # Relevance score for context-aware compression
+
+ @property
+ def change_count(self) -> int:
+ """Total number of actual changes (additions + deletions)."""
+ return self.additions + self.deletions
+
+
+@dataclass
+class DiffFile:
+ """A single file's diff."""
+
+ header: str # diff --git a/... b/...
+ old_file: str # --- a/...
+ new_file: str # +++ b/...
+ hunks: list[DiffHunk] = field(default_factory=list)
+ is_binary: bool = False
+ is_new_file: bool = False
+ is_deleted_file: bool = False
+ is_renamed: bool = False
+
+ @property
+ def total_additions(self) -> int:
+ return sum(h.additions for h in self.hunks)
+
+ @property
+ def total_deletions(self) -> int:
+ return sum(h.deletions for h in self.hunks)
+
+
+@dataclass
+class DiffCompressorConfig:
+ """Configuration for diff compression."""
+
+ # Context line limits
+ max_context_lines: int = 2 # Reduce from default 3 lines before/after changes
+
+ # Hunk limits
+ max_hunks_per_file: int = 10
+
+ # File limits
+ max_files: int = 20
+
+ # Change preservation
+ always_keep_additions: bool = True # Always keep + lines
+ always_keep_deletions: bool = True # Always keep - lines
+
+ # CCR integration
+ enable_ccr: bool = True
+ min_lines_for_ccr: int = 50
+
+
+@dataclass
+class DiffCompressionResult:
+ """Result of diff compression."""
+
+ compressed: str
+ original_line_count: int
+ compressed_line_count: int
+ files_affected: int
+ additions: int
+ deletions: int
+ hunks_kept: int
+ hunks_removed: int
+ cache_key: str | None = None
+
+ @property
+ def compression_ratio(self) -> float:
+ """Ratio of compressed to original (lower is better compression)."""
+ if self.original_line_count == 0:
+ return 1.0
+ return self.compressed_line_count / self.original_line_count
+
+ @property
+ def tokens_saved_estimate(self) -> int:
+ """Estimate tokens saved (rough: 1 token per 4 chars)."""
+ # Use line counts as proxy for chars
+ lines_saved = self.original_line_count - self.compressed_line_count
+ # Estimate ~40 chars per line average for diffs
+ chars_saved = lines_saved * 40
+ return max(0, chars_saved // 4)
+
+
+class DiffCompressor:
+ """Compresses git diff output.
+
+ Example:
+ >>> compressor = DiffCompressor()
+ >>> result = compressor.compress(git_diff_output)
+ >>> print(result.compressed) # Reduced diff with summary
+ """
+
+ # Pattern for diff --git header
+ _DIFF_GIT_PATTERN = re.compile(r"^diff --git a/(.+) b/(.+)$")
+
+ # Pattern for --- a/file or --- /dev/null
+ _OLD_FILE_PATTERN = re.compile(r"^--- (a/(.+)|/dev/null)$")
+
+ # Pattern for +++ b/file or +++ /dev/null
+ _NEW_FILE_PATTERN = re.compile(r"^\+\+\+ (b/(.+)|/dev/null)$")
+
+ # Pattern for hunk header @@ -start,count +start,count @@ optional context
+ _HUNK_HEADER_PATTERN = re.compile(r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)$")
+
+ # Pattern for binary file indication
+ _BINARY_PATTERN = re.compile(r"^Binary files .+ differ$")
+
+ # Patterns for new/deleted file mode
+ _NEW_FILE_MODE_PATTERN = re.compile(r"^new file mode")
+ _DELETED_FILE_MODE_PATTERN = re.compile(r"^deleted file mode")
+ _RENAME_PATTERN = re.compile(r"^(rename|similarity|copy) ")
+
+ # Priority patterns for context-aware hunk selection
+ _PRIORITY_PATTERNS = [
+ re.compile(r"\b(error|exception|fail|bug|fix)\b", re.IGNORECASE),
+ re.compile(r"\b(todo|fixme|hack|xxx)\b", re.IGNORECASE),
+ re.compile(r"\b(security|auth|password|secret|token)\b", re.IGNORECASE),
+ ]
+
+ def __init__(self, config: DiffCompressorConfig | None = None):
+ """Initialize diff compressor.
+
+ Args:
+ config: Compression configuration.
+ """
+ self.config = config or DiffCompressorConfig()
+
+ def compress(self, content: str, context: str = "") -> DiffCompressionResult:
+ """Compress diff output.
+
+ Args:
+ content: Raw git diff output.
+ context: User query context for relevance scoring.
+
+ Returns:
+ DiffCompressionResult with compressed output and metadata.
+ """
+ lines = content.split("\n")
+ original_line_count = len(lines)
+
+ if original_line_count < self.config.min_lines_for_ccr:
+ return DiffCompressionResult(
+ compressed=content,
+ original_line_count=original_line_count,
+ compressed_line_count=original_line_count,
+ files_affected=0,
+ additions=0,
+ deletions=0,
+ hunks_kept=0,
+ hunks_removed=0,
+ )
+
+ # Parse diff into structured format
+ diff_files = self._parse_diff(lines)
+
+ if not diff_files:
+ return DiffCompressionResult(
+ compressed=content,
+ original_line_count=original_line_count,
+ compressed_line_count=original_line_count,
+ files_affected=0,
+ additions=0,
+ deletions=0,
+ hunks_kept=0,
+ hunks_removed=0,
+ )
+
+ # Score hunks by relevance
+ self._score_hunks(diff_files, context)
+
+ # Compress each file's hunks
+ compressed_files, stats = self._compress_files(diff_files)
+
+ # Format output
+ compressed_output = self._format_output(compressed_files, stats)
+ compressed_line_count = len(compressed_output.split("\n"))
+
+ # Store in CCR if significant compression
+ cache_key = None
+ if self.config.enable_ccr and compressed_line_count < original_line_count * 0.8:
+ cache_key = self._store_in_ccr(content, compressed_output, original_line_count)
+ if cache_key:
+ compressed_output += f"\n[{original_line_count} lines compressed to {compressed_line_count}. Retrieve full diff: hash={cache_key}]"
+
+ return DiffCompressionResult(
+ compressed=compressed_output,
+ original_line_count=original_line_count,
+ compressed_line_count=compressed_line_count,
+ files_affected=stats["files_affected"],
+ additions=stats["total_additions"],
+ deletions=stats["total_deletions"],
+ hunks_kept=stats["hunks_kept"],
+ hunks_removed=stats["hunks_removed"],
+ cache_key=cache_key,
+ )
+
+ def _parse_diff(self, lines: list[str]) -> list[DiffFile]:
+ """Parse diff content into structured format.
+
+ Args:
+ lines: Lines of diff content.
+
+ Returns:
+ List of DiffFile objects.
+ """
+ diff_files: list[DiffFile] = []
+ current_file: DiffFile | None = None
+ current_hunk: DiffHunk | None = None
+ i = 0
+
+ while i < len(lines):
+ line = lines[i]
+
+ # Check for diff --git header (new file section)
+ if self._DIFF_GIT_PATTERN.match(line):
+ # Save previous hunk and file
+ if current_hunk and current_file:
+ current_file.hunks.append(current_hunk)
+ if current_file:
+ diff_files.append(current_file)
+
+ current_file = DiffFile(
+ header=line,
+ old_file="",
+ new_file="",
+ )
+ current_hunk = None
+ i += 1
+ continue
+
+ # Check for file mode indicators
+ if current_file:
+ if self._NEW_FILE_MODE_PATTERN.match(line):
+ current_file.is_new_file = True
+ elif self._DELETED_FILE_MODE_PATTERN.match(line):
+ current_file.is_deleted_file = True
+ elif self._RENAME_PATTERN.match(line):
+ current_file.is_renamed = True
+ elif self._BINARY_PATTERN.match(line):
+ current_file.is_binary = True
+
+ # Check for --- a/file
+ if self._OLD_FILE_PATTERN.match(line):
+ if current_file:
+ current_file.old_file = line
+ i += 1
+ continue
+
+ # Check for +++ b/file
+ if self._NEW_FILE_PATTERN.match(line):
+ if current_file:
+ current_file.new_file = line
+ i += 1
+ continue
+
+ # Check for hunk header
+ if self._HUNK_HEADER_PATTERN.match(line):
+ # Save previous hunk
+ if current_hunk and current_file:
+ current_file.hunks.append(current_hunk)
+
+ current_hunk = DiffHunk(
+ header=line,
+ lines=[],
+ )
+ i += 1
+ continue
+
+ # Process hunk content lines
+ if current_hunk is not None:
+ if line.startswith("+") and not line.startswith("+++"):
+ current_hunk.additions += 1
+ current_hunk.lines.append(line)
+ elif line.startswith("-") and not line.startswith("---"):
+ current_hunk.deletions += 1
+ current_hunk.lines.append(line)
+ elif line.startswith(" ") or line == "":
+ current_hunk.context_lines += 1
+ current_hunk.lines.append(line)
+ else:
+ # Other line (e.g., "\ No newline at end of file")
+ current_hunk.lines.append(line)
+
+ i += 1
+
+ # Save final hunk and file
+ if current_hunk and current_file:
+ current_file.hunks.append(current_hunk)
+ if current_file:
+ diff_files.append(current_file)
+
+ return diff_files
+
+ def _score_hunks(self, diff_files: list[DiffFile], context: str) -> None:
+ """Score hunks by relevance to context.
+
+ Args:
+ diff_files: Parsed diff files.
+ context: User query context.
+ """
+ context_lower = context.lower()
+ context_words = set(context_lower.split()) if context else set()
+
+ for diff_file in diff_files:
+ for hunk in diff_file.hunks:
+ score = 0.0
+
+ # Base score from change count (more changes = more important)
+ score += min(0.3, hunk.change_count * 0.03)
+
+ hunk_content = "\n".join(hunk.lines).lower()
+
+ # Score by context word overlap
+ for word in context_words:
+ if len(word) > 2 and word in hunk_content:
+ score += 0.2
+
+ # Boost for priority patterns
+ for pattern in self._PRIORITY_PATTERNS:
+ if pattern.search(hunk_content):
+ score += 0.3
+ break
+
+ hunk.score = min(1.0, score)
+
+ def _compress_files(self, diff_files: list[DiffFile]) -> tuple[list[DiffFile], dict[str, int]]:
+ """Compress hunks in each file.
+
+ Args:
+ diff_files: Parsed diff files.
+
+ Returns:
+ Tuple of (compressed files, stats dict).
+ """
+ stats = {
+ "files_affected": 0,
+ "total_additions": 0,
+ "total_deletions": 0,
+ "hunks_kept": 0,
+ "hunks_removed": 0,
+ }
+
+ # Limit files if too many
+ if len(diff_files) > self.config.max_files:
+ # Sort by total changes (most changes first)
+ diff_files = sorted(
+ diff_files,
+ key=lambda f: f.total_additions + f.total_deletions,
+ reverse=True,
+ )
+ diff_files = diff_files[: self.config.max_files]
+
+ compressed_files: list[DiffFile] = []
+
+ for diff_file in diff_files:
+ stats["files_affected"] += 1
+ stats["total_additions"] += diff_file.total_additions
+ stats["total_deletions"] += diff_file.total_deletions
+
+ # Compress hunks within file
+ compressed_hunks = self._compress_hunks(diff_file.hunks)
+
+ stats["hunks_kept"] += len(compressed_hunks)
+ stats["hunks_removed"] += len(diff_file.hunks) - len(compressed_hunks)
+
+ # Create compressed file with reduced context in hunks
+ new_file = DiffFile(
+ header=diff_file.header,
+ old_file=diff_file.old_file,
+ new_file=diff_file.new_file,
+ hunks=compressed_hunks,
+ is_binary=diff_file.is_binary,
+ is_new_file=diff_file.is_new_file,
+ is_deleted_file=diff_file.is_deleted_file,
+ is_renamed=diff_file.is_renamed,
+ )
+ compressed_files.append(new_file)
+
+ return compressed_files, stats
+
+ def _compress_hunks(self, hunks: list[DiffHunk]) -> list[DiffHunk]:
+ """Compress hunks by reducing context and limiting count.
+
+ Args:
+ hunks: List of hunks to compress.
+
+ Returns:
+ Compressed list of hunks.
+ """
+ if not hunks:
+ return []
+
+ # Sort by score if we need to limit
+ if len(hunks) > self.config.max_hunks_per_file:
+ # Keep first and last hunks (often important)
+ first_hunk = hunks[0]
+ last_hunk = hunks[-1] if len(hunks) > 1 else None
+
+ # Sort middle hunks by score
+ middle_hunks = sorted(
+ hunks[1:-1] if last_hunk else [], key=lambda h: h.score, reverse=True
+ )
+
+ # Take top scoring middle hunks
+ remaining_slots = (
+ self.config.max_hunks_per_file - 2
+ if last_hunk
+ else self.config.max_hunks_per_file - 1
+ )
+ selected_middle = middle_hunks[:remaining_slots]
+
+ # Rebuild list in original order by re-sorting by appearance
+ selected = [first_hunk] + selected_middle
+ if last_hunk:
+ selected.append(last_hunk)
+
+ # Sort back to original order (using header line numbers as proxy)
+ hunks = sorted(selected, key=lambda h: self._extract_line_number(h.header))
+
+ # Reduce context in each hunk
+ compressed_hunks = []
+ for hunk in hunks:
+ compressed_hunk = self._reduce_context(hunk)
+ compressed_hunks.append(compressed_hunk)
+
+ return compressed_hunks
+
+ def _extract_line_number(self, header: str) -> int:
+ """Extract starting line number from hunk header for sorting."""
+ match = self._HUNK_HEADER_PATTERN.match(header)
+ if match:
+ return int(match.group(1))
+ return 0
+
+ def _reduce_context(self, hunk: DiffHunk) -> DiffHunk:
+ """Reduce context lines while preserving all changes.
+
+ Args:
+ hunk: Hunk to reduce context in.
+
+ Returns:
+ New hunk with reduced context.
+ """
+ max_context = self.config.max_context_lines
+
+ # Identify change positions
+ change_positions: list[int] = []
+ for i, line in enumerate(hunk.lines):
+ if line.startswith("+") or line.startswith("-"):
+ change_positions.append(i)
+
+ if not change_positions:
+ # No changes, just context - keep minimal
+ return DiffHunk(
+ header=hunk.header,
+ lines=hunk.lines[:max_context] if hunk.lines else [],
+ additions=0,
+ deletions=0,
+ context_lines=min(len(hunk.lines), max_context),
+ score=hunk.score,
+ )
+
+ # Determine which lines to keep
+ keep_indices: set[int] = set()
+
+ for pos in change_positions:
+ # Always keep the change line
+ keep_indices.add(pos)
+
+ # Keep context before
+ for i in range(max(0, pos - max_context), pos):
+ keep_indices.add(i)
+
+ # Keep context after
+ for i in range(pos + 1, min(len(hunk.lines), pos + max_context + 1)):
+ keep_indices.add(i)
+
+ # Build new lines list
+ new_lines: list[str] = []
+ additions = 0
+ deletions = 0
+ context_lines = 0
+
+ for i in sorted(keep_indices):
+ line = hunk.lines[i]
+ new_lines.append(line)
+ if line.startswith("+"):
+ additions += 1
+ elif line.startswith("-"):
+ deletions += 1
+ else:
+ context_lines += 1
+
+ return DiffHunk(
+ header=hunk.header,
+ lines=new_lines,
+ additions=additions,
+ deletions=deletions,
+ context_lines=context_lines,
+ score=hunk.score,
+ )
+
+ def _format_output(self, diff_files: list[DiffFile], stats: dict[str, int]) -> str:
+ """Format compressed diff files back to unified diff format.
+
+ Args:
+ diff_files: Compressed diff files.
+ stats: Compression statistics.
+
+ Returns:
+ Formatted diff string.
+ """
+ output_lines: list[str] = []
+
+ for diff_file in diff_files:
+ # File header
+ output_lines.append(diff_file.header)
+
+ # File mode indicators if present
+ if diff_file.is_new_file:
+ output_lines.append("new file mode 100644")
+ elif diff_file.is_deleted_file:
+ output_lines.append("deleted file mode 100644")
+
+ if diff_file.is_binary:
+ output_lines.append("Binary files differ")
+ continue
+
+ # Old/new file markers
+ if diff_file.old_file:
+ output_lines.append(diff_file.old_file)
+ if diff_file.new_file:
+ output_lines.append(diff_file.new_file)
+
+ # Hunks
+ for hunk in diff_file.hunks:
+ output_lines.append(hunk.header)
+ output_lines.extend(hunk.lines)
+
+ # Add summary
+ if stats["hunks_removed"] > 0 or stats["files_affected"] > 0:
+ summary_parts = [
+ f"{stats['files_affected']} files changed",
+ f"+{stats['total_additions']} -{stats['total_deletions']} lines",
+ ]
+ if stats["hunks_removed"] > 0:
+ summary_parts.append(f"{stats['hunks_removed']} hunks omitted")
+ output_lines.append(f"[{', '.join(summary_parts)}]")
+
+ return "\n".join(output_lines)
+
+ def _store_in_ccr(self, original: str, compressed: str, original_count: int) -> str | None:
+ """Store original in CCR for later retrieval.
+
+ Args:
+ original: Original diff content.
+ compressed: Compressed diff content.
+ original_count: Original line count.
+
+ Returns:
+ Cache key if stored, None otherwise.
+ """
+ try:
+ from ..cache.compression_store import get_compression_store
+
+ store = get_compression_store()
+ return store.store(
+ original,
+ compressed,
+ original_item_count=original_count,
+ )
+ except ImportError:
+ return None
+ except Exception:
+ return None
diff --git a/tests/test_hnsw_only.py b/tests/test_hnsw_only.py
index c33d9e37b..3e2c9e179 100644
--- a/tests/test_hnsw_only.py
+++ b/tests/test_hnsw_only.py
@@ -13,9 +13,11 @@ import pytest
from headroom.memory.models import Memory
from headroom.memory.ports import VectorFilter
-# Check if hnswlib is available
+# Check if hnswlib is available (use lazy check to avoid SIGILL on incompatible CPUs)
try:
- from headroom.memory.adapters.hnsw import HNSW_AVAILABLE
+ from headroom.memory.adapters.hnsw import _check_hnswlib_available
+
+ HNSW_AVAILABLE = _check_hnswlib_available()
except ImportError:
HNSW_AVAILABLE = False
diff --git a/tests/test_memory/test_hierarchical.py b/tests/test_memory/test_hierarchical.py
index 0b567f0da..afe86fbd0 100644
--- a/tests/test_memory/test_hierarchical.py
+++ b/tests/test_memory/test_hierarchical.py
@@ -608,9 +608,11 @@ class TestIntegration:
# HNSW Vector Index Tests
# =============================================================================
-# Check if hnswlib is available
+# Check if hnswlib is available (use lazy check to avoid SIGILL on incompatible CPUs)
try:
- from headroom.memory.adapters.hnsw import HNSW_AVAILABLE
+ from headroom.memory.adapters.hnsw import _check_hnswlib_available
+
+ HNSW_AVAILABLE = _check_hnswlib_available()
except ImportError:
HNSW_AVAILABLE = False
diff --git a/tests/test_transforms/test_diff_compressor.py b/tests/test_transforms/test_diff_compressor.py
new file mode 100644
index 000000000..323e7ca3e
--- /dev/null
+++ b/tests/test_transforms/test_diff_compressor.py
@@ -0,0 +1,673 @@
+"""Comprehensive tests for diff_compressor.py.
+
+Tests cover:
+1. Parsing of unified diff format
+2. Context line reduction
+3. Hunk selection and limiting
+4. Compression ratios
+5. Edge cases
+"""
+
+from headroom.transforms.diff_compressor import (
+ DiffCompressionResult,
+ DiffCompressor,
+ DiffCompressorConfig,
+ DiffFile,
+ DiffHunk,
+)
+
+
+class TestDiffParsing:
+ """Tests for parsing unified diff format."""
+
+ def test_parse_simple_diff(self):
+ """Simple single-file diff is parsed correctly."""
+ content = """diff --git a/src/main.py b/src/main.py
+--- a/src/main.py
++++ b/src/main.py
+@@ -10,6 +10,7 @@ def main():
+ print("hello")
++ print("world")
+ return 0
+"""
+ compressor = DiffCompressor()
+ diff_files = compressor._parse_diff(content.split("\n"))
+
+ assert len(diff_files) == 1
+ assert diff_files[0].header == "diff --git a/src/main.py b/src/main.py"
+ assert diff_files[0].old_file == "--- a/src/main.py"
+ assert diff_files[0].new_file == "+++ b/src/main.py"
+ assert len(diff_files[0].hunks) == 1
+ assert diff_files[0].hunks[0].additions == 1
+ assert diff_files[0].hunks[0].deletions == 0
+
+ def test_parse_multi_file_diff(self):
+ """Multi-file diff is parsed into separate DiffFile objects."""
+ content = """diff --git a/file1.py b/file1.py
+--- a/file1.py
++++ b/file1.py
+@@ -1,3 +1,4 @@
+ line1
++added line
+ line2
+diff --git a/file2.py b/file2.py
+--- a/file2.py
++++ b/file2.py
+@@ -5,4 +5,3 @@
+ keep
+-removed
+ keep2
+"""
+ compressor = DiffCompressor()
+ diff_files = compressor._parse_diff(content.split("\n"))
+
+ assert len(diff_files) == 2
+ assert "file1.py" in diff_files[0].header
+ assert "file2.py" in diff_files[1].header
+ assert diff_files[0].hunks[0].additions == 1
+ assert diff_files[0].hunks[0].deletions == 0
+ assert diff_files[1].hunks[0].additions == 0
+ assert diff_files[1].hunks[0].deletions == 1
+
+ def test_parse_multi_hunk_file(self):
+ """File with multiple hunks is parsed correctly."""
+ content = """diff --git a/src/utils.py b/src/utils.py
+--- a/src/utils.py
++++ b/src/utils.py
+@@ -10,4 +10,5 @@ def helper():
+ pass
++ # added comment
+ return True
+@@ -50,3 +51,4 @@ def other():
+ x = 1
++ y = 2
+ return x
+"""
+ compressor = DiffCompressor()
+ diff_files = compressor._parse_diff(content.split("\n"))
+
+ assert len(diff_files) == 1
+ assert len(diff_files[0].hunks) == 2
+ assert diff_files[0].total_additions == 2
+
+ def test_parse_new_file(self):
+ """New file diff is detected."""
+ content = """diff --git a/newfile.py b/newfile.py
+new file mode 100644
+--- /dev/null
++++ b/newfile.py
+@@ -0,0 +1,3 @@
++def new_func():
++ pass
++ return None
+"""
+ compressor = DiffCompressor()
+ diff_files = compressor._parse_diff(content.split("\n"))
+
+ assert len(diff_files) == 1
+ assert diff_files[0].is_new_file is True
+ assert diff_files[0].hunks[0].additions == 3
+
+ def test_parse_deleted_file(self):
+ """Deleted file diff is detected."""
+ content = """diff --git a/oldfile.py b/oldfile.py
+deleted file mode 100644
+--- a/oldfile.py
++++ /dev/null
+@@ -1,2 +0,0 @@
+-def old_func():
+- pass
+"""
+ compressor = DiffCompressor()
+ diff_files = compressor._parse_diff(content.split("\n"))
+
+ assert len(diff_files) == 1
+ assert diff_files[0].is_deleted_file is True
+ assert diff_files[0].hunks[0].deletions == 2
+
+ def test_parse_binary_file(self):
+ """Binary file diff is detected."""
+ content = """diff --git a/image.png b/image.png
+Binary files a/image.png and b/image.png differ
+"""
+ compressor = DiffCompressor()
+ diff_files = compressor._parse_diff(content.split("\n"))
+
+ assert len(diff_files) == 1
+ assert diff_files[0].is_binary is True
+
+
+class TestContextReduction:
+ """Tests for context line reduction."""
+
+ def test_reduce_context_lines(self):
+ """Context lines are reduced to configured maximum."""
+ content = """diff --git a/file.py b/file.py
+--- a/file.py
++++ b/file.py
+@@ -1,10 +1,11 @@
+ context1
+ context2
+ context3
+ context4
++added
+ context5
+ context6
+ context7
+ context8
+"""
+ # Default max_context_lines is 2
+ compressor = DiffCompressor(
+ config=DiffCompressorConfig(
+ max_context_lines=2,
+ min_lines_for_ccr=5,
+ enable_ccr=False,
+ )
+ )
+ result = compressor.compress(content)
+
+ # Should keep 2 context before and 2 after the +added line
+ # Plus the added line itself
+ lines = result.compressed.split("\n")
+ context_count = sum(1 for line in lines if line.startswith(" "))
+
+ # At most 4 context lines (2 before + 2 after)
+ assert context_count <= 4
+
+ def test_preserve_all_changes(self):
+ """All addition and deletion lines are preserved."""
+ content = """diff --git a/file.py b/file.py
+--- a/file.py
++++ b/file.py
+@@ -1,10 +1,10 @@
+ ctx1
+ ctx2
+-removed1
++added1
+ ctx3
+ ctx4
+-removed2
++added2
+ ctx5
+ ctx6
+"""
+ compressor = DiffCompressor(
+ config=DiffCompressorConfig(
+ min_lines_for_ccr=5,
+ enable_ccr=False,
+ )
+ )
+ result = compressor.compress(content)
+
+ assert "-removed1" in result.compressed
+ assert "-removed2" in result.compressed
+ assert "+added1" in result.compressed
+ assert "+added2" in result.compressed
+
+
+class TestHunkSelection:
+ """Tests for hunk selection when limiting."""
+
+ def test_max_hunks_per_file(self):
+ """Hunks are limited to max_hunks_per_file."""
+ # Create a diff with many hunks
+ hunks = []
+ for i in range(20):
+ hunks.append(f"""@@ -{i * 10},3 +{i * 10},4 @@
+ context
++added_{i}
+ more
+""")
+
+ content = f"""diff --git a/bigfile.py b/bigfile.py
+--- a/bigfile.py
++++ b/bigfile.py
+{"".join(hunks)}"""
+
+ compressor = DiffCompressor(
+ config=DiffCompressorConfig(
+ max_hunks_per_file=5,
+ min_lines_for_ccr=10,
+ enable_ccr=False,
+ )
+ )
+ result = compressor.compress(content)
+
+ # Should have at most 5 hunks
+ hunk_count = result.compressed.count("@@")
+ # Each hunk has one @@ header (we count full hunk headers)
+ assert hunk_count <= 10 # Each hunk header appears twice @@...@@
+
+ def test_keeps_first_and_last_hunk(self):
+ """First and last hunks are preserved when limiting."""
+ hunks = []
+ for i in range(10):
+ hunks.append(f"""@@ -{i * 10},3 +{i * 10},4 @@
+ context
++added_{i}
+ more
+""")
+
+ content = f"""diff --git a/file.py b/file.py
+--- a/file.py
++++ b/file.py
+{"".join(hunks)}"""
+
+ compressor = DiffCompressor(
+ config=DiffCompressorConfig(
+ max_hunks_per_file=3,
+ min_lines_for_ccr=10,
+ enable_ccr=False,
+ )
+ )
+ result = compressor.compress(content)
+
+ # First hunk (added_0) should be present
+ assert "+added_0" in result.compressed
+ # Last hunk (added_9) should be present
+ assert "+added_9" in result.compressed
+
+
+class TestFileSelection:
+ """Tests for file selection when limiting."""
+
+ def test_max_files(self):
+ """Files are limited to max_files."""
+ # Create diff with many files
+ files = []
+ for i in range(30):
+ files.append(f"""diff --git a/file{i}.py b/file{i}.py
+--- a/file{i}.py
++++ b/file{i}.py
+@@ -1,2 +1,3 @@
+ ctx
++added
+ ctx2
+""")
+
+ content = "\n".join(files)
+
+ compressor = DiffCompressor(
+ config=DiffCompressorConfig(
+ max_files=10,
+ min_lines_for_ccr=20,
+ enable_ccr=False,
+ )
+ )
+ result = compressor.compress(content)
+
+ # Count diff --git headers
+ file_count = result.compressed.count("diff --git")
+ assert file_count <= 10
+
+
+class TestCompressionResult:
+ """Tests for DiffCompressionResult properties."""
+
+ def test_compression_ratio_calculation(self):
+ """Compression ratio is calculated correctly."""
+ result = DiffCompressionResult(
+ compressed="a\nb\nc",
+ original_line_count=100,
+ compressed_line_count=10,
+ files_affected=2,
+ additions=5,
+ deletions=3,
+ hunks_kept=2,
+ hunks_removed=5,
+ )
+
+ assert result.compression_ratio == 0.1
+
+ def test_tokens_saved_estimate(self):
+ """Token savings estimation works correctly."""
+ result = DiffCompressionResult(
+ compressed="short",
+ original_line_count=100,
+ compressed_line_count=10,
+ files_affected=1,
+ additions=10,
+ deletions=5,
+ hunks_kept=1,
+ hunks_removed=0,
+ )
+
+ # 90 lines saved * 40 chars/line / 4 chars/token = 900 tokens
+ assert result.tokens_saved_estimate == 900
+
+
+class TestHunkScoring:
+ """Tests for context-aware hunk scoring."""
+
+ def test_score_by_context_keywords(self):
+ """Hunks containing context keywords get higher scores."""
+ content = """diff --git a/file.py b/file.py
+--- a/file.py
++++ b/file.py
+@@ -1,3 +1,4 @@
+ normal context
++normal change
+ more context
+@@ -10,3 +11,4 @@
+ error handling
++fix the bug here
+ return result
+"""
+ compressor = DiffCompressor()
+ diff_files = compressor._parse_diff(content.split("\n"))
+ compressor._score_hunks(diff_files, "fix error bug")
+
+ # Second hunk should have higher score (contains "fix" and "bug")
+ assert len(diff_files[0].hunks) == 2
+ assert diff_files[0].hunks[1].score > diff_files[0].hunks[0].score
+
+ def test_score_priority_patterns(self):
+ """Hunks with priority patterns (error, security) score higher."""
+ compressor = DiffCompressor()
+
+ hunk_normal = DiffHunk(
+ header="@@ -1,1 +1,2 @@",
+ lines=["+normal change"],
+ additions=1,
+ )
+ hunk_error = DiffHunk(
+ header="@@ -10,1 +10,2 @@",
+ lines=["+fix critical error"],
+ additions=1,
+ )
+
+ diff_file = DiffFile(
+ header="diff --git a/f.py b/f.py",
+ old_file="--- a/f.py",
+ new_file="+++ b/f.py",
+ hunks=[hunk_normal, hunk_error],
+ )
+
+ compressor._score_hunks([diff_file], "")
+
+ assert hunk_error.score > hunk_normal.score
+
+
+class TestSmallDiffPassthrough:
+ """Tests for small diff passthrough behavior."""
+
+ def test_small_diff_unchanged(self):
+ """Diffs smaller than threshold pass through unchanged."""
+ content = """diff --git a/small.py b/small.py
+--- a/small.py
++++ b/small.py
+@@ -1,2 +1,3 @@
+ line1
++added
+ line2
+"""
+ compressor = DiffCompressor(
+ config=DiffCompressorConfig(
+ min_lines_for_ccr=100, # High threshold
+ )
+ )
+ result = compressor.compress(content)
+
+ # Should be unchanged
+ assert result.compressed == content
+ assert result.compression_ratio == 1.0
+
+
+class TestOutputFormatting:
+ """Tests for output formatting."""
+
+ def test_summary_line_added(self):
+ """Summary line is added at end of compressed diff."""
+ # Large diff that will be compressed
+ hunks = []
+ for i in range(15):
+ hunks.append(f"""@@ -{i * 10},5 +{i * 10},6 @@
+ ctx1
+ ctx2
++added_{i}
+ ctx3
+ ctx4
+""")
+
+ content = f"""diff --git a/file.py b/file.py
+--- a/file.py
++++ b/file.py
+{"".join(hunks)}"""
+
+ compressor = DiffCompressor(
+ config=DiffCompressorConfig(
+ max_hunks_per_file=5,
+ min_lines_for_ccr=10,
+ enable_ccr=False,
+ )
+ )
+ result = compressor.compress(content)
+
+ # Should have summary at end
+ assert "files changed" in result.compressed
+ assert "hunks omitted" in result.compressed
+
+ def test_preserves_diff_format(self):
+ """Output preserves valid unified diff format."""
+ content = """diff --git a/test.py b/test.py
+--- a/test.py
++++ b/test.py
+@@ -1,3 +1,4 @@
+ def test():
++ # new comment
+ pass
+ return True
+"""
+ compressor = DiffCompressor(
+ config=DiffCompressorConfig(
+ min_lines_for_ccr=5,
+ enable_ccr=False,
+ )
+ )
+ result = compressor.compress(content)
+
+ # Should have all standard diff markers
+ assert "diff --git" in result.compressed
+ assert "---" in result.compressed
+ assert "+++" in result.compressed
+ assert "@@" in result.compressed
+
+
+class TestEdgeCases:
+ """Tests for edge cases and boundary conditions."""
+
+ def test_empty_input(self):
+ """Empty input is handled gracefully."""
+ compressor = DiffCompressor()
+ result = compressor.compress("")
+
+ assert result.compressed == ""
+ assert result.compression_ratio == 1.0
+
+ def test_non_diff_input(self):
+ """Non-diff input passes through unchanged."""
+ content = "This is not a diff\nJust regular text"
+ compressor = DiffCompressor()
+ result = compressor.compress(content)
+
+ # Should pass through (no diff --git found)
+ assert result.compressed == content
+
+ def test_unicode_content(self):
+ """Unicode characters in diff are handled."""
+ content = """diff --git a/i18n.py b/i18n.py
+--- a/i18n.py
++++ b/i18n.py
+@@ -1,2 +1,3 @@
+ msg = "hello"
++msg_ja = "こんにちは"
+ return msg
+"""
+ compressor = DiffCompressor()
+ result = compressor.compress(content)
+
+ assert "こんにちは" in result.compressed
+
+ def test_no_newline_at_eof(self):
+ """Handles 'No newline at end of file' indicator."""
+ content = """diff --git a/file.py b/file.py
+--- a/file.py
++++ b/file.py
+@@ -1,2 +1,2 @@
+ line1
+-line2
+\\ No newline at end of file
++line2_modified
+\\ No newline at end of file
+"""
+ compressor = DiffCompressor()
+ result = compressor.compress(content)
+
+ # Should not crash and preserve the indicator
+ assert "No newline" in result.compressed or "-line2" in result.compressed
+
+ def test_empty_hunks(self):
+ """Files with no actual hunks are handled."""
+ content = """diff --git a/file.py b/file.py
+--- a/file.py
++++ b/file.py
+"""
+ compressor = DiffCompressor()
+ result = compressor.compress(content)
+
+ # Should not crash
+ assert result.compressed is not None
+
+
+class TestDiffHunkDataclass:
+ """Tests for DiffHunk dataclass."""
+
+ def test_change_count_property(self):
+ """change_count returns sum of additions and deletions."""
+ hunk = DiffHunk(
+ header="@@ -1,5 +1,6 @@",
+ lines=["+a", "+b", "-c", " ctx"],
+ additions=2,
+ deletions=1,
+ )
+ assert hunk.change_count == 3
+
+ def test_default_values(self):
+ """DiffHunk default values are correct."""
+ hunk = DiffHunk(header="@@", lines=[])
+ assert hunk.additions == 0
+ assert hunk.deletions == 0
+ assert hunk.context_lines == 0
+ assert hunk.score == 0.0
+
+
+class TestDiffFileDataclass:
+ """Tests for DiffFile dataclass."""
+
+ def test_total_additions_property(self):
+ """total_additions sums across all hunks."""
+ hunk1 = DiffHunk(header="@@", lines=[], additions=3)
+ hunk2 = DiffHunk(header="@@", lines=[], additions=5)
+ diff_file = DiffFile(
+ header="diff --git",
+ old_file="---",
+ new_file="+++",
+ hunks=[hunk1, hunk2],
+ )
+ assert diff_file.total_additions == 8
+
+ def test_total_deletions_property(self):
+ """total_deletions sums across all hunks."""
+ hunk1 = DiffHunk(header="@@", lines=[], deletions=2)
+ hunk2 = DiffHunk(header="@@", lines=[], deletions=4)
+ diff_file = DiffFile(
+ header="diff --git",
+ old_file="---",
+ new_file="+++",
+ hunks=[hunk1, hunk2],
+ )
+ assert diff_file.total_deletions == 6
+
+
+class TestConfigOptions:
+ """Tests for configuration options."""
+
+ def test_max_context_lines_config(self):
+ """max_context_lines configuration controls context reduction."""
+ content = """diff --git a/file.py b/file.py
+--- a/file.py
++++ b/file.py
+@@ -1,10 +1,11 @@
+ c1
+ c2
+ c3
+ c4
+ c5
++added
+ c6
+ c7
+ c8
+ c9
+ c10
+"""
+ # With max_context_lines=1
+ compressor = DiffCompressor(
+ config=DiffCompressorConfig(
+ max_context_lines=1,
+ min_lines_for_ccr=5,
+ enable_ccr=False,
+ )
+ )
+ result = compressor.compress(content)
+
+ # Count context lines (lines starting with space)
+ context_count = sum(1 for line in result.compressed.split("\n") if line.startswith(" "))
+
+ # Should have at most 2 context lines (1 before + 1 after)
+ assert context_count <= 2
+
+ def test_always_keep_additions_default(self):
+ """Additions are always kept by default."""
+ content = """diff --git a/file.py b/file.py
+--- a/file.py
++++ b/file.py
+@@ -1,3 +1,5 @@
+ ctx
++add1
++add2
+ ctx
+"""
+ compressor = DiffCompressor(
+ config=DiffCompressorConfig(
+ always_keep_additions=True,
+ min_lines_for_ccr=2,
+ enable_ccr=False,
+ )
+ )
+ result = compressor.compress(content)
+
+ assert "+add1" in result.compressed
+ assert "+add2" in result.compressed
+
+ def test_always_keep_deletions_default(self):
+ """Deletions are always kept by default."""
+ content = """diff --git a/file.py b/file.py
+--- a/file.py
++++ b/file.py
+@@ -1,5 +1,3 @@
+ ctx
+-del1
+-del2
+ ctx
+"""
+ compressor = DiffCompressor(
+ config=DiffCompressorConfig(
+ always_keep_deletions=True,
+ min_lines_for_ccr=2,
+ enable_ccr=False,
+ )
+ )
+ result = compressor.compress(content)
+
+ assert "-del1" in result.compressed
+ assert "-del2" in result.compressed