Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
refactor: centralize error/priority pattern detection into error_detection module
Tejas Choprac7a76dbf985b
diff --git a/headroom/transforms/diff_compressor.py b/headroom/transforms/diff_compressor.pyindex 91b6aaf49..ac8cf2753 100644--- a/headroom/transforms/diff_compressor.py+++ b/headroom/transforms/diff_compressor.py@@ -150,12 +150,10 @@ class DiffCompressor:_DELETED_FILE_MODE_PATTERN = re.compile(r"^deleted file mode")_RENAME_PATTERN = re.compile(r"^(rename|similarity|copy) ")- # Priority patterns for context-aware hunk selection- _PRIORITY_PATTERNS = [- re.compile(r"\b(error|exception|fail|bug|fix)\b", re.IGNORECASE),- re.compile(r"\b(todo|fixme|hack|xxx)\b", re.IGNORECASE),- re.compile(r"\b(security|auth|password|secret|token)\b", re.IGNORECASE),- ]+ # Priority patterns for context-aware hunk selection (centralized)+ from headroom.transforms.error_detection import PRIORITY_PATTERNS_DIFF++ _PRIORITY_PATTERNS = PRIORITY_PATTERNS_DIFFdef __init__(self, config: DiffCompressorConfig | None = None):"""Initialize diff compressor.diff --git a/headroom/transforms/error_detection.py b/headroom/transforms/error_detection.pynew file mode 100644index 000000000..3b32eae99--- /dev/null+++ b/headroom/transforms/error_detection.py@@ -0,0 +1,128 @@+"""Centralized error/importance detection for all transforms.++Design principle: Keywords serve as a FALLBACK safety net for error detection.+When TOIN field semantics are available, they take priority over keywords.++This module prevents each transform from maintaining its own hardcoded keyword+list, ensuring consistency and a single place to evolve detection logic.+"""++from __future__ import annotations++import re++# ─── Canonical keyword sets ──────────────────────────────────────────────────+# These are the FALLBACK when TOIN semantics aren't available yet.+# They are intentionally broad to avoid missing errors.++ERROR_KEYWORDS: frozenset[str] = frozenset(+ {+ "error",+ "exception",+ "failed",+ "failure",+ "critical",+ "fatal",+ "crash",+ "panic",+ "abort",+ "timeout",+ "denied",+ "rejected",+ }+)++# Broader importance keywords (for line-level scoring, not item preservation)+IMPORTANCE_KEYWORDS: frozenset[str] = frozenset(+ ERROR_KEYWORDS+ | {+ "warning",+ "warn",+ "todo",+ "fixme",+ "hack",+ "xxx",+ "bug",+ "fix",+ "important",+ "note",+ }+)++# Security-related keywords (for diff/search prioritization)+SECURITY_KEYWORDS: frozenset[str] = frozenset(+ {+ "security",+ "auth",+ "password",+ "secret",+ "token",+ }+)++# ─── Compiled patterns (for line-level matching) ────────────────────────────+# Shared across text_compressor, diff_compressor, search_compressor++ERROR_PATTERN: re.Pattern[str] = re.compile(+ r"\b(error|exception|fail(?:ed|ure)?|fatal|critical|crash|panic)\b",+ re.IGNORECASE,+)++WARNING_PATTERN: re.Pattern[str] = re.compile(+ r"\b(warn(?:ing)?)\b",+ re.IGNORECASE,+)++IMPORTANCE_PATTERN: re.Pattern[str] = re.compile(+ r"\b(important|note|todo|fixme|hack|xxx|bug|fix)\b",+ re.IGNORECASE,+)++SECURITY_PATTERN: re.Pattern[str] = re.compile(+ r"\b(security|auth|password|secret|token)\b",+ re.IGNORECASE,+)++# Pre-built pattern lists for each compressor context+PRIORITY_PATTERNS_SEARCH: list[re.Pattern[str]] = [+ ERROR_PATTERN,+ WARNING_PATTERN,+ IMPORTANCE_PATTERN,+]++PRIORITY_PATTERNS_DIFF: list[re.Pattern[str]] = [+ ERROR_PATTERN,+ IMPORTANCE_PATTERN,+ SECURITY_PATTERN,+]++PRIORITY_PATTERNS_TEXT: list[re.Pattern[str]] = [+ ERROR_PATTERN,+ IMPORTANCE_PATTERN,+ re.compile(r"^#+\s"), # Markdown headers+ re.compile(r"^\*\*"), # Bold text+ re.compile(r"^>\s"), # Quotes+]++# ─── Quick check for message-level error indicators ─────────────────────────+# Used by intelligent_context.py for message signature creation++ERROR_INDICATOR_KEYWORDS: tuple[str, ...] = (+ "error",+ "fail",+ "exception",+ "traceback",+ "fatal",+ "panic",+ "crash",+)+++def content_has_error_indicators(text: str) -> bool:+ """Check if text contains error indicators (fast keyword check).++ Used for message signature creation and quick triage, NOT for+ compression decisions (those should use TOIN when available).+ """+ text_lower = text.lower()+ return any(kw in text_lower for kw in ERROR_INDICATOR_KEYWORDS)diff --git a/headroom/transforms/intelligent_context.py b/headroom/transforms/intelligent_context.pyindex a9ac742dd..f5feba0a6 100644--- a/headroom/transforms/intelligent_context.py+++ b/headroom/transforms/intelligent_context.py@@ -79,12 +79,10 @@ def _create_message_signature(messages: list[dict[str, Any]]) -> Any:content = msg.get("content", "")if isinstance(content, str):total_content_length += len(content)- # Check for error patterns (learned from TOIN, but basic heuristic here)- content_lower = content.lower()- if any(- indicator in content_lower- for indicator in ["error", "fail", "exception", "traceback"]- ):+ # Check for error patterns (centralized, TOIN takes priority when available)+ from headroom.transforms.error_detection import content_has_error_indicators++ if content_has_error_indicators(content):has_error_indicators = Trueelif isinstance(content, list):# Anthropic format with content blocksdiff --git a/headroom/transforms/search_compressor.py b/headroom/transforms/search_compressor.pyindex b80566714..73ca015a3 100644--- a/headroom/transforms/search_compressor.py+++ b/headroom/transforms/search_compressor.py@@ -87,12 +87,10 @@ class SearchCompressor:# Pattern for ripgrep with context (file-line-content or file:line:content)_RG_CONTEXT_PATTERN = re.compile(r"^([^:-]+)[:-](\d+)[:-](.*)$")- # Error/important patterns to prioritize- _PRIORITY_PATTERNS = [- re.compile(r"\b(error|exception|fail|fatal)\b", re.IGNORECASE),- re.compile(r"\b(warn|warning)\b", re.IGNORECASE),- re.compile(r"\b(todo|fixme|hack|xxx)\b", re.IGNORECASE),- ]+ # Error/important patterns to prioritize (centralized)+ from headroom.transforms.error_detection import PRIORITY_PATTERNS_SEARCH++ _PRIORITY_PATTERNS = PRIORITY_PATTERNS_SEARCHdef __init__(self, config: SearchCompressorConfig | None = None):"""Initialize search compressor.diff --git a/headroom/transforms/smart_crusher.py b/headroom/transforms/smart_crusher.pyindex 6905eafd3..d6bfcaccc 100644--- a/headroom/transforms/smart_crusher.py+++ b/headroom/transforms/smart_crusher.py@@ -77,6 +77,7 @@ from ..utils import (from .anchor_selector import AnchorSelectorfrom .anchor_selector import DataPattern as AnchorDataPatternfrom .base import Transform+from .error_detection import ERROR_KEYWORDSlogger = logging.getLogger(__name__)@@ -623,22 +624,8 @@ def _detect_rare_status_values(items: list[dict], common_fields: set[str]) -> li# Error keywords for PRESERVATION guarantee (not crushability detection)# This is for the quality guarantee: "ALL error items are ALWAYS preserved"# regardless of how common they are. Used in _prioritize_indices().-_ERROR_KEYWORDS_FOR_PRESERVATION = frozenset(- {- "error",- "exception",- "failed",- "failure",- "critical",- "fatal",- "crash",- "panic",- "abort",- "timeout",- "denied",- "rejected",- }-)+# Centralized in error_detection module for consistency across transforms.+_ERROR_KEYWORDS_FOR_PRESERVATION = ERROR_KEYWORDSdef _detect_error_items_for_preservation(items: list[dict]) -> list[int]:diff --git a/headroom/transforms/text_compressor.py b/headroom/transforms/text_compressor.pyindex 71ddaa92b..b41ee3ec6 100644--- a/headroom/transforms/text_compressor.py+++ b/headroom/transforms/text_compressor.py@@ -13,7 +13,6 @@ Compression Strategy:from __future__ import annotations-import refrom dataclasses import dataclass, field@@ -47,14 +46,10 @@ class TextCompressor:>>> print(result.compressed)"""- # Patterns that indicate important lines- _IMPORTANT_PATTERNS = [- re.compile(r"\b(error|exception|fail|warning)\b", re.IGNORECASE),- re.compile(r"\b(important|note|todo|fixme)\b", re.IGNORECASE),- re.compile(r"^#+\s"), # Markdown headers- re.compile(r"^\*\*"), # Bold text- re.compile(r"^>\s"), # Quotes- ]+ # Patterns that indicate important lines (centralized in error_detection module)+ from headroom.transforms.error_detection import PRIORITY_PATTERNS_TEXT++ _IMPORTANT_PATTERNS = PRIORITY_PATTERNS_TEXTdef __init__(self, config: TextCompressorConfig | None = None):"""Initialize text compressor.