Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

refactor: centralize error/priority pattern detection into error_detection module

Tejas Choprac7a76dbf985b
diff --git a/headroom/transforms/diff_compressor.py b/headroom/transforms/diff_compressor.py
index 91b6aaf49..ac8cf2753 100644
--- a/headroom/transforms/diff_compressor.py
+++ b/headroom/transforms/diff_compressor.py
@@ -150,12 +150,10 @@ class DiffCompressor:
_DELETED_FILE_MODE_PATTERN = re.compile(r"^deleted file mode")
_RENAME_PATTERN = re.compile(r"^(rename|similarity|copy) ")
- # Priority patterns for context-aware hunk selection
- _PRIORITY_PATTERNS = [
- re.compile(r"\b(error|exception|fail|bug|fix)\b", re.IGNORECASE),
- re.compile(r"\b(todo|fixme|hack|xxx)\b", re.IGNORECASE),
- re.compile(r"\b(security|auth|password|secret|token)\b", re.IGNORECASE),
- ]
+ # Priority patterns for context-aware hunk selection (centralized)
+ from headroom.transforms.error_detection import PRIORITY_PATTERNS_DIFF
+
+ _PRIORITY_PATTERNS = PRIORITY_PATTERNS_DIFF
def __init__(self, config: DiffCompressorConfig | None = None):
"""Initialize diff compressor.
diff --git a/headroom/transforms/error_detection.py b/headroom/transforms/error_detection.py
new file mode 100644
index 000000000..3b32eae99
--- /dev/null
+++ b/headroom/transforms/error_detection.py
@@ -0,0 +1,128 @@
+"""Centralized error/importance detection for all transforms.
+
+Design principle: Keywords serve as a FALLBACK safety net for error detection.
+When TOIN field semantics are available, they take priority over keywords.
+
+This module prevents each transform from maintaining its own hardcoded keyword
+list, ensuring consistency and a single place to evolve detection logic.
+"""
+
+from __future__ import annotations
+
+import re
+
+# ─── Canonical keyword sets ──────────────────────────────────────────────────
+# These are the FALLBACK when TOIN semantics aren't available yet.
+# They are intentionally broad to avoid missing errors.
+
+ERROR_KEYWORDS: frozenset[str] = frozenset(
+ {
+ "error",
+ "exception",
+ "failed",
+ "failure",
+ "critical",
+ "fatal",
+ "crash",
+ "panic",
+ "abort",
+ "timeout",
+ "denied",
+ "rejected",
+ }
+)
+
+# Broader importance keywords (for line-level scoring, not item preservation)
+IMPORTANCE_KEYWORDS: frozenset[str] = frozenset(
+ ERROR_KEYWORDS
+ | {
+ "warning",
+ "warn",
+ "todo",
+ "fixme",
+ "hack",
+ "xxx",
+ "bug",
+ "fix",
+ "important",
+ "note",
+ }
+)
+
+# Security-related keywords (for diff/search prioritization)
+SECURITY_KEYWORDS: frozenset[str] = frozenset(
+ {
+ "security",
+ "auth",
+ "password",
+ "secret",
+ "token",
+ }
+)
+
+# ─── Compiled patterns (for line-level matching) ────────────────────────────
+# Shared across text_compressor, diff_compressor, search_compressor
+
+ERROR_PATTERN: re.Pattern[str] = re.compile(
+ r"\b(error|exception|fail(?:ed|ure)?|fatal|critical|crash|panic)\b",
+ re.IGNORECASE,
+)
+
+WARNING_PATTERN: re.Pattern[str] = re.compile(
+ r"\b(warn(?:ing)?)\b",
+ re.IGNORECASE,
+)
+
+IMPORTANCE_PATTERN: re.Pattern[str] = re.compile(
+ r"\b(important|note|todo|fixme|hack|xxx|bug|fix)\b",
+ re.IGNORECASE,
+)
+
+SECURITY_PATTERN: re.Pattern[str] = re.compile(
+ r"\b(security|auth|password|secret|token)\b",
+ re.IGNORECASE,
+)
+
+# Pre-built pattern lists for each compressor context
+PRIORITY_PATTERNS_SEARCH: list[re.Pattern[str]] = [
+ ERROR_PATTERN,
+ WARNING_PATTERN,
+ IMPORTANCE_PATTERN,
+]
+
+PRIORITY_PATTERNS_DIFF: list[re.Pattern[str]] = [
+ ERROR_PATTERN,
+ IMPORTANCE_PATTERN,
+ SECURITY_PATTERN,
+]
+
+PRIORITY_PATTERNS_TEXT: list[re.Pattern[str]] = [
+ ERROR_PATTERN,
+ IMPORTANCE_PATTERN,
+ re.compile(r"^#+\s"), # Markdown headers
+ re.compile(r"^\*\*"), # Bold text
+ re.compile(r"^>\s"), # Quotes
+]
+
+# ─── Quick check for message-level error indicators ─────────────────────────
+# Used by intelligent_context.py for message signature creation
+
+ERROR_INDICATOR_KEYWORDS: tuple[str, ...] = (
+ "error",
+ "fail",
+ "exception",
+ "traceback",
+ "fatal",
+ "panic",
+ "crash",
+)
+
+
+def content_has_error_indicators(text: str) -> bool:
+ """Check if text contains error indicators (fast keyword check).
+
+ Used for message signature creation and quick triage, NOT for
+ compression decisions (those should use TOIN when available).
+ """
+ text_lower = text.lower()
+ return any(kw in text_lower for kw in ERROR_INDICATOR_KEYWORDS)
diff --git a/headroom/transforms/intelligent_context.py b/headroom/transforms/intelligent_context.py
index a9ac742dd..f5feba0a6 100644
--- a/headroom/transforms/intelligent_context.py
+++ b/headroom/transforms/intelligent_context.py
@@ -79,12 +79,10 @@ def _create_message_signature(messages: list[dict[str, Any]]) -> Any:
content = msg.get("content", "")
if isinstance(content, str):
total_content_length += len(content)
- # Check for error patterns (learned from TOIN, but basic heuristic here)
- content_lower = content.lower()
- if any(
- indicator in content_lower
- for indicator in ["error", "fail", "exception", "traceback"]
- ):
+ # Check for error patterns (centralized, TOIN takes priority when available)
+ from headroom.transforms.error_detection import content_has_error_indicators
+
+ if content_has_error_indicators(content):
has_error_indicators = True
elif isinstance(content, list):
# Anthropic format with content blocks
diff --git a/headroom/transforms/search_compressor.py b/headroom/transforms/search_compressor.py
index b80566714..73ca015a3 100644
--- a/headroom/transforms/search_compressor.py
+++ b/headroom/transforms/search_compressor.py
@@ -87,12 +87,10 @@ class SearchCompressor:
# Pattern for ripgrep with context (file-line-content or file:line:content)
_RG_CONTEXT_PATTERN = re.compile(r"^([^:-]+)[:-](\d+)[:-](.*)$")
- # Error/important patterns to prioritize
- _PRIORITY_PATTERNS = [
- re.compile(r"\b(error|exception|fail|fatal)\b", re.IGNORECASE),
- re.compile(r"\b(warn|warning)\b", re.IGNORECASE),
- re.compile(r"\b(todo|fixme|hack|xxx)\b", re.IGNORECASE),
- ]
+ # Error/important patterns to prioritize (centralized)
+ from headroom.transforms.error_detection import PRIORITY_PATTERNS_SEARCH
+
+ _PRIORITY_PATTERNS = PRIORITY_PATTERNS_SEARCH
def __init__(self, config: SearchCompressorConfig | None = None):
"""Initialize search compressor.
diff --git a/headroom/transforms/smart_crusher.py b/headroom/transforms/smart_crusher.py
index 6905eafd3..d6bfcaccc 100644
--- a/headroom/transforms/smart_crusher.py
+++ b/headroom/transforms/smart_crusher.py
@@ -77,6 +77,7 @@ from ..utils import (
from .anchor_selector import AnchorSelector
from .anchor_selector import DataPattern as AnchorDataPattern
from .base import Transform
+from .error_detection import ERROR_KEYWORDS
logger = logging.getLogger(__name__)
@@ -623,22 +624,8 @@ def _detect_rare_status_values(items: list[dict], common_fields: set[str]) -> li
# Error keywords for PRESERVATION guarantee (not crushability detection)
# This is for the quality guarantee: "ALL error items are ALWAYS preserved"
# regardless of how common they are. Used in _prioritize_indices().
-_ERROR_KEYWORDS_FOR_PRESERVATION = frozenset(
- {
- "error",
- "exception",
- "failed",
- "failure",
- "critical",
- "fatal",
- "crash",
- "panic",
- "abort",
- "timeout",
- "denied",
- "rejected",
- }
-)
+# Centralized in error_detection module for consistency across transforms.
+_ERROR_KEYWORDS_FOR_PRESERVATION = ERROR_KEYWORDS
def _detect_error_items_for_preservation(items: list[dict]) -> list[int]:
diff --git a/headroom/transforms/text_compressor.py b/headroom/transforms/text_compressor.py
index 71ddaa92b..b41ee3ec6 100644
--- a/headroom/transforms/text_compressor.py
+++ b/headroom/transforms/text_compressor.py
@@ -13,7 +13,6 @@ Compression Strategy:
from __future__ import annotations
-import re
from dataclasses import dataclass, field
@@ -47,14 +46,10 @@ class TextCompressor:
>>> print(result.compressed)
"""
- # Patterns that indicate important lines
- _IMPORTANT_PATTERNS = [
- re.compile(r"\b(error|exception|fail|warning)\b", re.IGNORECASE),
- re.compile(r"\b(important|note|todo|fixme)\b", re.IGNORECASE),
- re.compile(r"^#+\s"), # Markdown headers
- re.compile(r"^\*\*"), # Bold text
- re.compile(r"^>\s"), # Quotes
- ]
+ # Patterns that indicate important lines (centralized in error_detection module)
+ from headroom.transforms.error_detection import PRIORITY_PATTERNS_TEXT
+
+ _IMPORTANT_PATTERNS = PRIORITY_PATTERNS_TEXT
def __init__(self, config: TextCompressorConfig | None = None):
"""Initialize text compressor.