Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
Protect workflow XML tags from text compression
chopratejas7a79aa0792b0
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.pyindex 20a9d8fd6..d77f658ac 100644--- a/headroom/transforms/content_router.py+++ b/headroom/transforms/content_router.py@@ -416,6 +416,12 @@ class ContentRouterConfig:ccr_enabled: bool = True # Enable CCR marker injection for reversible compressionccr_inject_marker: bool = True # Add retrieval markers to compressed content+ # Tag protection: preserve custom/workflow XML tags from text compression.+ # When False (default), entire <custom-tag>content</custom-tag> blocks are+ # protected verbatim. When True, only the tag markers are protected and+ # the content between them can be compressed.+ compress_tagged_content: bool = False+# Tools to exclude from compression (output passed through unmodified)# Set to None to use DEFAULT_EXCLUDE_TOOLS, or provide custom setexclude_tools: set[str] | None = None@@ -1042,6 +1048,10 @@ class ContentRouter(Transform):Kompress (ModernBERT, trained on 330K structured tool outputs)auto-downloads from HuggingFace on first use. No heuristic fallback.+ Custom/workflow XML tags (<system-reminder>, <tool_call>, <thinking>)+ are protected before compression and restored after. Standard HTML+ tags are left alone (HTMLExtractor handles those separately).+Args:content: Content to compress.context: User context.@@ -1050,27 +1060,58 @@ class ContentRouter(Transform):Returns:Tuple of (compressed, token_count)."""+ from .tag_protector import protect_tags, restore_tags++ # Protect custom tags before any ML compression+ cleaned, protected = protect_tags(+ content,+ compress_tagged_content=self.config.compress_tagged_content,+ )++ # If the entire content is custom tags with nothing to compress+ if protected and not cleaned.strip():+ return content, len(content.split())++ # Use the cleaned (tag-free) text for compression+ text_to_compress = cleaned if protected else content+ compressed: str | None = None+ compressed_tokens: int | None = None+# Primary: Kompress — downloads from chopratejas/kompress-base on first useif self.config.enable_kompress:compressor = self._get_kompress()if compressor:try:- result = compressor.compress(content, context=context, question=question)- return result.compressed, result.compressed_tokens+ result = compressor.compress(+ text_to_compress, context=context, question=question+ )+ compressed = result.compressed+ compressed_tokens = result.compressed_tokensexcept Exception as e:logger.warning("Kompress failed: %s", e)# Fallback: LLMLingua (only if Kompress not installed)- if self.config.enable_llmlingua:+ if compressed is None and self.config.enable_llmlingua:compressor = self._get_llmlingua()if compressor:try:- result = compressor.compress(content, context=context, question=question)- return result.compressed, result.compressed_tokens+ result = compressor.compress(+ text_to_compress, context=context, question=question+ )+ compressed = result.compressed+ compressed_tokens = result.compressed_tokensexcept Exception as e:logger.warning("LLMLingua failed: %s", e)- return content, len(content.split())+ if compressed is None:+ return content, len(content.split())++ # Restore protected tag blocks into the compressed text+ if protected:+ compressed = restore_tags(compressed, protected)+ compressed_tokens = len(compressed.split())++ return compressed, compressed_tokens or len(compressed.split())# Backwards compatibility_try_llmlingua = _try_ml_compressordiff --git a/headroom/transforms/tag_protector.py b/headroom/transforms/tag_protector.pynew file mode 100644index 000000000..6d26afc58--- /dev/null+++ b/headroom/transforms/tag_protector.py@@ -0,0 +1,316 @@+"""Protect workflow/custom XML tags from text compression.++LLM workflows use XML-style tags (<system-reminder>, <tool_call>, <thinking>)+as structural markers. Text compressors (Kompress, LLMLingua) treat these as+droppable noise and silently remove them, breaking downstream tools.++This module detects custom tags (anything NOT standard HTML), replaces entire+blocks with placeholders before compression, and restores them after.++Standard HTML tags (<div>, <p>, <span>) are left alone — HTMLExtractor+handles those via trafilatura.++Usage:+ from headroom.transforms.tag_protector import protect_tags, restore_tags++ cleaned, protected = protect_tags(text)+ compressed = kompress.compress(cleaned)+ result = restore_tags(compressed, protected)+"""++from __future__ import annotations++import logging+import re++logger = logging.getLogger(__name__)++# ---------------------------------------------------------------------------+# HTML5 Living Standard element names (https://html.spec.whatwg.org/)+# These are the ONLY tags treated as "HTML". Everything else is a custom+# workflow tag and gets protected from compression.+# ---------------------------------------------------------------------------+KNOWN_HTML_TAGS: frozenset[str] = frozenset(+ {+ # Main root+ "html",+ # Document metadata+ "base",+ "head",+ "link",+ "meta",+ "style",+ "title",+ # Sectioning root+ "body",+ # Content sectioning+ "address",+ "article",+ "aside",+ "footer",+ "h1",+ "h2",+ "h3",+ "h4",+ "h5",+ "h6",+ "header",+ "hgroup",+ "main",+ "nav",+ "section",+ "search",+ # Text content+ "blockquote",+ "dd",+ "div",+ "dl",+ "dt",+ "figcaption",+ "figure",+ "hr",+ "li",+ "menu",+ "ol",+ "p",+ "pre",+ "ul",+ # Inline text semantics+ "a",+ "abbr",+ "b",+ "bdi",+ "bdo",+ "br",+ "cite",+ "code",+ "data",+ "dfn",+ "em",+ "i",+ "kbd",+ "mark",+ "q",+ "rp",+ "rt",+ "ruby",+ "s",+ "samp",+ "small",+ "span",+ "strong",+ "sub",+ "sup",+ "time",+ "u",+ "var",+ "wbr",+ # Image and multimedia+ "area",+ "audio",+ "img",+ "map",+ "track",+ "video",+ # Embedded content+ "embed",+ "iframe",+ "object",+ "param",+ "picture",+ "portal",+ "source",+ # SVG and MathML+ "svg",+ "math",+ # Scripting+ "canvas",+ "noscript",+ "script",+ # Demarcating edits+ "del",+ "ins",+ # Table content+ "caption",+ "col",+ "colgroup",+ "table",+ "tbody",+ "td",+ "tfoot",+ "th",+ "thead",+ "tr",+ # Forms+ "button",+ "datalist",+ "fieldset",+ "form",+ "input",+ "label",+ "legend",+ "meter",+ "optgroup",+ "option",+ "output",+ "progress",+ "select",+ "textarea",+ # Interactive+ "details",+ "dialog",+ "summary",+ # Web Components+ "slot",+ "template",+ }+)++# Placeholder format — unlikely to appear in real content+_PLACEHOLDER_PREFIX = "{{HEADROOM_TAG_"+_PLACEHOLDER_SUFFIX = "}}"++# Match opening tags: <tag>, <tag attr="val">, and self-closing <tag/>+_OPEN_TAG_RE = re.compile(r"<([a-zA-Z_][\w.-]*)((?:\s+[^>]*?)?)(/?)>")++# Match closing tags: </tag>+_CLOSE_TAG_RE = re.compile(r"</([a-zA-Z_][\w.-]*)>")+++def _is_html_tag(tag_name: str) -> bool:+ """Check if a tag name is a known HTML element (case-insensitive)."""+ return tag_name.lower() in KNOWN_HTML_TAGS+++def protect_tags(+ text: str,+ compress_tagged_content: bool = False,+) -> tuple[str, list[tuple[str, str]]]:+ """Protect custom/workflow XML tags from text compression.++ Scans for XML-style tags, classifies each as HTML or custom.+ Custom tags (and optionally their content) are replaced with+ placeholders that survive compression.++ Args:+ text: Input text potentially containing XML tags.+ compress_tagged_content: If False (default), protect entire+ ``<tag>content</tag>`` block verbatim. If True, only+ protect the tag markers; content between them can be+ compressed.++ Returns:+ Tuple of (cleaned_text, protected_blocks) where protected_blocks+ is a list of (placeholder, original_text) pairs for restoration.+ """+ if not text or "<" not in text:+ return text, []++ protected: list[tuple[str, str]] = []+ counter = 0++ if not compress_tagged_content:+ # Protect entire <tag>...</tag> blocks for custom tags.+ # Process from innermost out by iterating until stable.+ result = text+ changed = True+ max_iterations = 50 # Safety limit for deeply nested tags++ while changed and max_iterations > 0:+ changed = False+ max_iterations -= 1++ # Find blocks: <tag ...>...</tag> where tag is NOT known HTML+ # Use non-greedy match for content between tags+ pattern = re.compile(+ r"<([a-zA-Z_][\w.-]*)((?:\s+[^>]*?)?)>"+ r"(.*?)"+ r"</\1>",+ re.DOTALL,+ )++ for match in pattern.finditer(result):+ tag_name = match.group(1)+ if _is_html_tag(tag_name):+ continue++ original_block = match.group(0)+ placeholder = f"{_PLACEHOLDER_PREFIX}{counter}{_PLACEHOLDER_SUFFIX}"+ protected.append((placeholder, original_block))+ result = result.replace(original_block, placeholder, 1)+ counter += 1+ changed = True+ break # Restart iteration after each replacement++ # Also protect self-closing custom tags: <tag/> <tag attr="x"/>+ for match in _OPEN_TAG_RE.finditer(result):+ tag_name = match.group(1)+ is_self_closing = match.group(3) == "/"+ if is_self_closing and not _is_html_tag(tag_name):+ original = match.group(0)+ if original in result: # Not already replaced+ placeholder = f"{_PLACEHOLDER_PREFIX}{counter}{_PLACEHOLDER_SUFFIX}"+ protected.append((placeholder, original))+ result = result.replace(original, placeholder, 1)+ counter += 1++ return result, protected++ else:+ # compress_tagged_content=True: only protect tag markers, not content+ result = text++ # Protect opening custom tags+ for match in _OPEN_TAG_RE.finditer(text):+ tag_name = match.group(1)+ if not _is_html_tag(tag_name):+ original = match.group(0)+ placeholder = f"{_PLACEHOLDER_PREFIX}{counter}{_PLACEHOLDER_SUFFIX}"+ protected.append((placeholder, original))+ result = result.replace(original, placeholder, 1)+ counter += 1++ # Protect closing custom tags+ for match in _CLOSE_TAG_RE.finditer(text):+ tag_name = match.group(1)+ if not _is_html_tag(tag_name):+ original = match.group(0)+ if original in result:+ placeholder = f"{_PLACEHOLDER_PREFIX}{counter}{_PLACEHOLDER_SUFFIX}"+ protected.append((placeholder, original))+ result = result.replace(original, placeholder, 1)+ counter += 1++ return result, protected+++def restore_tags(+ text: str,+ protected_blocks: list[tuple[str, str]],+) -> str:+ """Restore protected tag blocks after compression.++ Args:+ text: Compressed text with placeholders.+ protected_blocks: List of (placeholder, original_text) pairs+ from ``protect_tags()``.++ Returns:+ Text with all placeholders replaced by original blocks.+ """+ if not protected_blocks:+ return text++ result = text+ for placeholder, original in protected_blocks:+ if placeholder in result:+ result = result.replace(placeholder, original)+ else:+ # Placeholder was lost during compression — append the block+ logger.warning(+ "Tag placeholder lost during compression, appending: %s",+ original[:80],+ )+ result = result + "\n" + original++ return resultdiff --git a/tests/test_tag_protection_integration.py b/tests/test_tag_protection_integration.pynew file mode 100644index 000000000..ba58ecb1d--- /dev/null+++ b/tests/test_tag_protection_integration.py@@ -0,0 +1,184 @@+"""Integration tests: verify tags survive the full compress() pipeline."""++from __future__ import annotations++import os++import pytest+++class TestCompressPreservesTags:+ """End-to-end tests: compress() with tagged content."""++ def test_system_reminder_survives(self):+ """<system-reminder> tags in tool output survive compression."""+ from headroom import compress++ messages = [+ {"role": "user", "content": "What are the rules?"},+ {+ "role": "tool",+ "tool_call_id": "call_1",+ "content": (+ "Here is a very long and verbose explanation of the system rules "+ "that contains a lot of unnecessary filler words and repetitive "+ "phrasing that could be compressed significantly. "+ "<system-reminder>You must always validate input before processing. "+ "Never skip authentication checks.</system-reminder> "+ "The rest of this text is also quite verbose and contains additional "+ "unnecessary details that add tokens without adding information value "+ "to the overall response that the language model needs to generate."+ ),+ },+ ]+ result = compress(messages, model="claude-sonnet-4-5-20250929")+ output = str(result.messages[-1].get("content", ""))++ assert "<system-reminder>" in output+ assert "</system-reminder>" in output+ assert "validate input" in output+ assert "authentication checks" in output++ def test_tool_call_tags_survive(self):+ """<tool_call> tags survive compression."""+ from headroom import compress++ messages = [+ {"role": "user", "content": "Search for results"},+ {+ "role": "tool",+ "tool_call_id": "call_1",+ "content": (+ "Processing the search request with extensive verbose output "+ "that includes many redundant descriptions and unnecessary detail. "+ '<tool_call>{"name": "search", "args": {"query": "test"}}</tool_call> '+ "Additional verbose context that repeats information already stated "+ "in the previous paragraphs about the search functionality and its "+ "various capabilities and features that are not directly relevant."+ ),+ },+ ]+ result = compress(messages, model="claude-sonnet-4-5-20250929")+ output = str(result.messages[-1].get("content", ""))++ assert "<tool_call>" in output+ assert "</tool_call>" in output+ assert '"name": "search"' in output++ def test_thinking_tags_survive(self):+ """<thinking> tags survive compression."""+ from headroom import compress++ messages = [+ {"role": "user", "content": "Analyze this data"},+ {+ "role": "tool",+ "tool_call_id": "call_1",+ "content": (+ "The analysis produced extensive results with much verbose "+ "explanatory text that describes methodology in great detail. "+ "<thinking>Step 1: Parse input. Step 2: Validate schema. "+ "Step 3: Run inference.</thinking> "+ "Further verbose explanation of the analytical process and its "+ "various stages and intermediate results and observations that "+ "could be expressed much more concisely without losing meaning."+ ),+ },+ ]+ result = compress(messages, model="claude-sonnet-4-5-20250929")+ output = str(result.messages[-1].get("content", ""))++ assert "<thinking>" in output+ assert "</thinking>" in output+ assert "Step 1: Parse input" in output++ def test_html_tags_still_compressible(self):+ """Standard HTML tags are NOT protected — they're just text to the compressor."""+ from headroom.transforms.tag_protector import protect_tags++ html_text = "<div>Some content</div> <span>More content</span>"+ cleaned, protected = protect_tags(html_text)++ # HTML tags should NOT be protected+ assert protected == []+ assert "<div>" in cleaned++ def test_multiple_custom_tags_in_messages(self):+ """Multiple different custom tags all survive."""+ from headroom import compress++ messages = [+ {"role": "user", "content": "What should I do?"},+ {+ "role": "tool",+ "tool_call_id": "call_1",+ "content": (+ "Very verbose introductory text with many unnecessary words. "+ "<system-reminder>Rule 1: Always validate</system-reminder> "+ "More verbose middle text repeating previous information. "+ "<context>session_id=abc-123</context> "+ "Additional verbose concluding text with redundant information. "+ "<IMPORTANT>Never expose API keys</IMPORTANT> "+ "Final paragraph of verbose text."+ ),+ },+ ]+ result = compress(messages, model="claude-sonnet-4-5-20250929")+ output = str(result.messages[-1].get("content", ""))++ assert "<system-reminder>" in output+ assert "<context>" in output+ assert "<IMPORTANT>" in output+++ not os.environ.get("ANTHROPIC_API_KEY"),+ reason="ANTHROPIC_API_KEY not set",+)+class TestRealAPIWithTags:+ """Real API integration: verify Claude can use tagged content after compression."""++ def test_compressed_tags_usable_by_llm(self):+ """Send compressed content with tags to Claude, verify it references them."""+ from anthropic import Anthropic++ from headroom import compress++ client = Anthropic()++ messages = [+ {"role": "user", "content": "What is the secret code in the system reminder?"},+ {+ "role": "assistant",+ "content": "Let me check the system configuration.",+ },+ {+ "role": "user",+ "content": (+ "Here is a very long and detailed configuration document with "+ "extensive verbose descriptions of various system parameters and "+ "their default values and recommended settings for production "+ "deployment scenarios across different cloud providers. "+ "<system-reminder>The secret code is ALPHA-7742-BRAVO</system-reminder> "+ "Additional verbose documentation about system architecture and "+ "deployment patterns and scaling strategies and monitoring setup "+ "and alerting configuration and backup procedures."+ ),+ },+ ]++ result = compress(messages, model="claude-sonnet-4-5-20250929")++ # Verify tags survived compression+ last_content = str(result.messages[-1].get("content", ""))+ assert "<system-reminder>" in last_content+ assert "ALPHA-7742-BRAVO" in last_content++ # Send to Claude and verify it can read the tagged content+ response = client.messages.create(+ model="claude-sonnet-4-5-20250929",+ max_tokens=200,+ messages=result.messages,+ )+ answer = response.content[0].text.lower()+ assert "alpha-7742-bravo" in answer or "alpha" in answerdiff --git a/tests/test_transforms/test_tag_protector.py b/tests/test_transforms/test_tag_protector.pynew file mode 100644index 000000000..9b0a1de6b--- /dev/null+++ b/tests/test_transforms/test_tag_protector.py@@ -0,0 +1,183 @@+"""Tests for tag_protector: protect custom/workflow XML tags from compression."""++from __future__ import annotations++from headroom.transforms.tag_protector import (+ KNOWN_HTML_TAGS,+ _is_html_tag,+ protect_tags,+ restore_tags,+)+++class TestKnownHTMLTags:+ def test_common_tags_are_html(self):+ for tag in ["div", "span", "p", "a", "table", "tr", "td", "h1", "body", "html"]:+ assert _is_html_tag(tag), f"{tag} should be recognized as HTML"++ def test_case_insensitive(self):+ assert _is_html_tag("DIV")+ assert _is_html_tag("Span")+ assert _is_html_tag("TABLE")++ def test_custom_tags_not_html(self):+ for tag in [+ "system-reminder",+ "tool_call",+ "thinking",+ "EXTREMELY_IMPORTANT",+ "context",+ "user-prompt-submit-hook",+ "result",+ "anthrpoic_thinking",+ ]:+ assert not _is_html_tag(tag), f"{tag} should NOT be recognized as HTML"++ def test_tag_set_not_empty(self):+ assert len(KNOWN_HTML_TAGS) > 100+++class TestProtectTags:+ def test_no_tags_passthrough(self):+ text = "Just plain text with no tags at all."+ cleaned, protected = protect_tags(text)+ assert cleaned == text+ assert protected == []++ def test_no_angle_brackets_fast_path(self):+ text = "No angle brackets here"+ cleaned, protected = protect_tags(text)+ assert cleaned == text+ assert protected == []++ def test_html_tags_not_protected(self):+ text = "<div>Some content</div>"+ cleaned, protected = protect_tags(text)+ assert cleaned == text+ assert protected == []++ def test_custom_tag_protected(self):+ text = "Before <system-reminder>Important rule</system-reminder> After"+ cleaned, protected = protect_tags(text)+ assert "<system-reminder>" not in cleaned+ assert "Important rule" not in cleaned+ assert "Before" in cleaned+ assert "After" in cleaned+ assert len(protected) == 1+ assert protected[0][1] == "<system-reminder>Important rule</system-reminder>"++ def test_multiple_custom_tags(self):+ text = "<thinking>Step 1</thinking> middle <context>data</context>"+ cleaned, protected = protect_tags(text)+ assert len(protected) == 2+ assert "<thinking>" not in cleaned+ assert "<context>" not in cleaned++ def test_custom_tag_with_attributes(self):+ text = '<context key="session" type="persistent">user data</context>'+ cleaned, protected = protect_tags(text)+ assert len(protected) == 1+ assert 'key="session"' in protected[0][1]++ def test_self_closing_custom_tag(self):+ text = "Text <marker/> more text"+ cleaned, protected = protect_tags(text)+ assert len(protected) == 1+ assert protected[0][1] == "<marker/>"++ def test_self_closing_html_tag_not_protected(self):+ text = "Text <br/> more <hr/> text"+ cleaned, protected = protect_tags(text)+ assert cleaned == text+ assert protected == []++ def test_mixed_html_and_custom(self):+ text = "<div>HTML content</div> <system-reminder>Rule</system-reminder> <p>More HTML</p>"+ cleaned, protected = protect_tags(text)+ assert "<div>" in cleaned+ assert "<p>" in cleaned+ assert "<system-reminder>" not in cleaned+ assert len(protected) == 1++ def test_nested_custom_tags(self):+ text = "<outer><inner>deep content</inner></outer>"+ cleaned, protected = protect_tags(text)+ # Both should be protected (inner first, then outer)+ assert "<outer>" not in cleaned+ assert "<inner>" not in cleaned+ assert len(protected) >= 1++ def test_real_workflow_tags(self):+ """Tags actually used in LLM workflows."""+ tags = [+ "<tool_call>search({query: 'test'})</tool_call>",+ "<thinking>Let me analyze this step by step</thinking>",+ "<EXTREMELY_IMPORTANT>Never skip validation</EXTREMELY_IMPORTANT>",+ "<user-prompt-submit-hook>check permissions</user-prompt-submit-hook>",+ "<system-reminder>Follow these rules exactly</system-reminder>",+ "<result>Success: 42 items processed</result>",+ ]+ for tag_text in tags:+ text = f"Before {tag_text} After"+ cleaned, protected = protect_tags(text)+ assert len(protected) == 1, f"Failed to protect: {tag_text}"+ assert protected[0][1] == tag_text++ def test_empty_string(self):+ cleaned, protected = protect_tags("")+ assert cleaned == ""+ assert protected == []+++class TestProtectTagsCompressContent:+ def test_compress_tagged_content_true(self):+ text = "Before <system-reminder>Compressible content here</system-reminder> After"+ cleaned, protected = protect_tags(text, compress_tagged_content=True)+ # Tags protected, but content between them is exposed for compression+ assert "<system-reminder>" not in cleaned+ assert "</system-reminder>" not in cleaned+ assert "Compressible content here" in cleaned+ assert len(protected) == 2 # Opening tag + closing tag++ def test_compress_tagged_content_false_default(self):+ text = "Before <system-reminder>Protected content</system-reminder> After"+ cleaned, protected = protect_tags(text)+ assert "Protected content" not in cleaned+ assert len(protected) == 1 # Entire block+++class TestRestoreTags:+ def test_basic_restore(self):+ original = "Before <system-reminder>Rule</system-reminder> After"+ cleaned, protected = protect_tags(original)+ restored = restore_tags(cleaned, protected)+ assert "<system-reminder>Rule</system-reminder>" in restored+ assert "Before" in restored+ assert "After" in restored++ def test_restore_empty_protected(self):+ text = "No tags here"+ assert restore_tags(text, []) == text++ def test_restore_multiple(self):+ original = "<thinking>A</thinking> gap <context>B</context>"+ cleaned, protected = protect_tags(original)+ restored = restore_tags(cleaned, protected)+ assert "<thinking>A</thinking>" in restored+ assert "<context>B</context>" in restored++ def test_lost_placeholder_appended(self):+ """If compression removes a placeholder, the block is appended."""+ protected = [("{{HEADROOM_TAG_0}}", "<tag>data</tag>")]+ result = restore_tags("text without placeholder", protected)+ assert "<tag>data</tag>" in result++ def test_roundtrip_preserves_content(self):+ original = (+ "Start <system-reminder>Rule 1: always validate</system-reminder> "+ "middle <tool_call>search(q='test')</tool_call> end"+ )+ cleaned, protected = protect_tags(original)+ restored = restore_tags(cleaned, protected)+ assert "<system-reminder>Rule 1: always validate</system-reminder>" in restored+ assert "<tool_call>search(q='test')</tool_call>" in restored