Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

Protect workflow XML tags from text compression

chopratejas7a79aa0792b0
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py
index 20a9d8fd6..d77f658ac 100644
--- a/headroom/transforms/content_router.py
+++ b/headroom/transforms/content_router.py
@@ -416,6 +416,12 @@ class ContentRouterConfig:
ccr_enabled: bool = True # Enable CCR marker injection for reversible compression
ccr_inject_marker: bool = True # Add retrieval markers to compressed content
+ # Tag protection: preserve custom/workflow XML tags from text compression.
+ # When False (default), entire <custom-tag>content</custom-tag> blocks are
+ # protected verbatim. When True, only the tag markers are protected and
+ # the content between them can be compressed.
+ compress_tagged_content: bool = False
+
# Tools to exclude from compression (output passed through unmodified)
# Set to None to use DEFAULT_EXCLUDE_TOOLS, or provide custom set
exclude_tools: set[str] | None = None
@@ -1042,6 +1048,10 @@ class ContentRouter(Transform):
Kompress (ModernBERT, trained on 330K structured tool outputs)
auto-downloads from HuggingFace on first use. No heuristic fallback.
+ Custom/workflow XML tags (<system-reminder>, <tool_call>, <thinking>)
+ are protected before compression and restored after. Standard HTML
+ tags are left alone (HTMLExtractor handles those separately).
+
Args:
content: Content to compress.
context: User context.
@@ -1050,27 +1060,58 @@ class ContentRouter(Transform):
Returns:
Tuple of (compressed, token_count).
"""
+ from .tag_protector import protect_tags, restore_tags
+
+ # Protect custom tags before any ML compression
+ cleaned, protected = protect_tags(
+ content,
+ compress_tagged_content=self.config.compress_tagged_content,
+ )
+
+ # If the entire content is custom tags with nothing to compress
+ if protected and not cleaned.strip():
+ return content, len(content.split())
+
+ # Use the cleaned (tag-free) text for compression
+ text_to_compress = cleaned if protected else content
+ compressed: str | None = None
+ compressed_tokens: int | None = None
+
# Primary: Kompress — downloads from chopratejas/kompress-base on first use
if self.config.enable_kompress:
compressor = self._get_kompress()
if compressor:
try:
- result = compressor.compress(content, context=context, question=question)
- return result.compressed, result.compressed_tokens
+ result = compressor.compress(
+ text_to_compress, context=context, question=question
+ )
+ compressed = result.compressed
+ compressed_tokens = result.compressed_tokens
except Exception as e:
logger.warning("Kompress failed: %s", e)
# Fallback: LLMLingua (only if Kompress not installed)
- if self.config.enable_llmlingua:
+ if compressed is None and self.config.enable_llmlingua:
compressor = self._get_llmlingua()
if compressor:
try:
- result = compressor.compress(content, context=context, question=question)
- return result.compressed, result.compressed_tokens
+ result = compressor.compress(
+ text_to_compress, context=context, question=question
+ )
+ compressed = result.compressed
+ compressed_tokens = result.compressed_tokens
except Exception as e:
logger.warning("LLMLingua failed: %s", e)
- return content, len(content.split())
+ if compressed is None:
+ return content, len(content.split())
+
+ # Restore protected tag blocks into the compressed text
+ if protected:
+ compressed = restore_tags(compressed, protected)
+ compressed_tokens = len(compressed.split())
+
+ return compressed, compressed_tokens or len(compressed.split())
# Backwards compatibility
_try_llmlingua = _try_ml_compressor
diff --git a/headroom/transforms/tag_protector.py b/headroom/transforms/tag_protector.py
new file mode 100644
index 000000000..6d26afc58
--- /dev/null
+++ b/headroom/transforms/tag_protector.py
@@ -0,0 +1,316 @@
+"""Protect workflow/custom XML tags from text compression.
+
+LLM workflows use XML-style tags (<system-reminder>, <tool_call>, <thinking>)
+as structural markers. Text compressors (Kompress, LLMLingua) treat these as
+droppable noise and silently remove them, breaking downstream tools.
+
+This module detects custom tags (anything NOT standard HTML), replaces entire
+blocks with placeholders before compression, and restores them after.
+
+Standard HTML tags (<div>, <p>, <span>) are left alone — HTMLExtractor
+handles those via trafilatura.
+
+Usage:
+ from headroom.transforms.tag_protector import protect_tags, restore_tags
+
+ cleaned, protected = protect_tags(text)
+ compressed = kompress.compress(cleaned)
+ result = restore_tags(compressed, protected)
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# HTML5 Living Standard element names (https://html.spec.whatwg.org/)
+# These are the ONLY tags treated as "HTML". Everything else is a custom
+# workflow tag and gets protected from compression.
+# ---------------------------------------------------------------------------
+KNOWN_HTML_TAGS: frozenset[str] = frozenset(
+ {
+ # Main root
+ "html",
+ # Document metadata
+ "base",
+ "head",
+ "link",
+ "meta",
+ "style",
+ "title",
+ # Sectioning root
+ "body",
+ # Content sectioning
+ "address",
+ "article",
+ "aside",
+ "footer",
+ "h1",
+ "h2",
+ "h3",
+ "h4",
+ "h5",
+ "h6",
+ "header",
+ "hgroup",
+ "main",
+ "nav",
+ "section",
+ "search",
+ # Text content
+ "blockquote",
+ "dd",
+ "div",
+ "dl",
+ "dt",
+ "figcaption",
+ "figure",
+ "hr",
+ "li",
+ "menu",
+ "ol",
+ "p",
+ "pre",
+ "ul",
+ # Inline text semantics
+ "a",
+ "abbr",
+ "b",
+ "bdi",
+ "bdo",
+ "br",
+ "cite",
+ "code",
+ "data",
+ "dfn",
+ "em",
+ "i",
+ "kbd",
+ "mark",
+ "q",
+ "rp",
+ "rt",
+ "ruby",
+ "s",
+ "samp",
+ "small",
+ "span",
+ "strong",
+ "sub",
+ "sup",
+ "time",
+ "u",
+ "var",
+ "wbr",
+ # Image and multimedia
+ "area",
+ "audio",
+ "img",
+ "map",
+ "track",
+ "video",
+ # Embedded content
+ "embed",
+ "iframe",
+ "object",
+ "param",
+ "picture",
+ "portal",
+ "source",
+ # SVG and MathML
+ "svg",
+ "math",
+ # Scripting
+ "canvas",
+ "noscript",
+ "script",
+ # Demarcating edits
+ "del",
+ "ins",
+ # Table content
+ "caption",
+ "col",
+ "colgroup",
+ "table",
+ "tbody",
+ "td",
+ "tfoot",
+ "th",
+ "thead",
+ "tr",
+ # Forms
+ "button",
+ "datalist",
+ "fieldset",
+ "form",
+ "input",
+ "label",
+ "legend",
+ "meter",
+ "optgroup",
+ "option",
+ "output",
+ "progress",
+ "select",
+ "textarea",
+ # Interactive
+ "details",
+ "dialog",
+ "summary",
+ # Web Components
+ "slot",
+ "template",
+ }
+)
+
+# Placeholder format — unlikely to appear in real content
+_PLACEHOLDER_PREFIX = "{{HEADROOM_TAG_"
+_PLACEHOLDER_SUFFIX = "}}"
+
+# Match opening tags: <tag>, <tag attr="val">, and self-closing <tag/>
+_OPEN_TAG_RE = re.compile(r"<([a-zA-Z_][\w.-]*)((?:\s+[^>]*?)?)(/?)>")
+
+# Match closing tags: </tag>
+_CLOSE_TAG_RE = re.compile(r"</([a-zA-Z_][\w.-]*)>")
+
+
+def _is_html_tag(tag_name: str) -> bool:
+ """Check if a tag name is a known HTML element (case-insensitive)."""
+ return tag_name.lower() in KNOWN_HTML_TAGS
+
+
+def protect_tags(
+ text: str,
+ compress_tagged_content: bool = False,
+) -> tuple[str, list[tuple[str, str]]]:
+ """Protect custom/workflow XML tags from text compression.
+
+ Scans for XML-style tags, classifies each as HTML or custom.
+ Custom tags (and optionally their content) are replaced with
+ placeholders that survive compression.
+
+ Args:
+ text: Input text potentially containing XML tags.
+ compress_tagged_content: If False (default), protect entire
+ ``<tag>content</tag>`` block verbatim. If True, only
+ protect the tag markers; content between them can be
+ compressed.
+
+ Returns:
+ Tuple of (cleaned_text, protected_blocks) where protected_blocks
+ is a list of (placeholder, original_text) pairs for restoration.
+ """
+ if not text or "<" not in text:
+ return text, []
+
+ protected: list[tuple[str, str]] = []
+ counter = 0
+
+ if not compress_tagged_content:
+ # Protect entire <tag>...</tag> blocks for custom tags.
+ # Process from innermost out by iterating until stable.
+ result = text
+ changed = True
+ max_iterations = 50 # Safety limit for deeply nested tags
+
+ while changed and max_iterations > 0:
+ changed = False
+ max_iterations -= 1
+
+ # Find blocks: <tag ...>...</tag> where tag is NOT known HTML
+ # Use non-greedy match for content between tags
+ pattern = re.compile(
+ r"<([a-zA-Z_][\w.-]*)((?:\s+[^>]*?)?)>"
+ r"(.*?)"
+ r"</\1>",
+ re.DOTALL,
+ )
+
+ for match in pattern.finditer(result):
+ tag_name = match.group(1)
+ if _is_html_tag(tag_name):
+ continue
+
+ original_block = match.group(0)
+ placeholder = f"{_PLACEHOLDER_PREFIX}{counter}{_PLACEHOLDER_SUFFIX}"
+ protected.append((placeholder, original_block))
+ result = result.replace(original_block, placeholder, 1)
+ counter += 1
+ changed = True
+ break # Restart iteration after each replacement
+
+ # Also protect self-closing custom tags: <tag/> <tag attr="x"/>
+ for match in _OPEN_TAG_RE.finditer(result):
+ tag_name = match.group(1)
+ is_self_closing = match.group(3) == "/"
+ if is_self_closing and not _is_html_tag(tag_name):
+ original = match.group(0)
+ if original in result: # Not already replaced
+ placeholder = f"{_PLACEHOLDER_PREFIX}{counter}{_PLACEHOLDER_SUFFIX}"
+ protected.append((placeholder, original))
+ result = result.replace(original, placeholder, 1)
+ counter += 1
+
+ return result, protected
+
+ else:
+ # compress_tagged_content=True: only protect tag markers, not content
+ result = text
+
+ # Protect opening custom tags
+ for match in _OPEN_TAG_RE.finditer(text):
+ tag_name = match.group(1)
+ if not _is_html_tag(tag_name):
+ original = match.group(0)
+ placeholder = f"{_PLACEHOLDER_PREFIX}{counter}{_PLACEHOLDER_SUFFIX}"
+ protected.append((placeholder, original))
+ result = result.replace(original, placeholder, 1)
+ counter += 1
+
+ # Protect closing custom tags
+ for match in _CLOSE_TAG_RE.finditer(text):
+ tag_name = match.group(1)
+ if not _is_html_tag(tag_name):
+ original = match.group(0)
+ if original in result:
+ placeholder = f"{_PLACEHOLDER_PREFIX}{counter}{_PLACEHOLDER_SUFFIX}"
+ protected.append((placeholder, original))
+ result = result.replace(original, placeholder, 1)
+ counter += 1
+
+ return result, protected
+
+
+def restore_tags(
+ text: str,
+ protected_blocks: list[tuple[str, str]],
+) -> str:
+ """Restore protected tag blocks after compression.
+
+ Args:
+ text: Compressed text with placeholders.
+ protected_blocks: List of (placeholder, original_text) pairs
+ from ``protect_tags()``.
+
+ Returns:
+ Text with all placeholders replaced by original blocks.
+ """
+ if not protected_blocks:
+ return text
+
+ result = text
+ for placeholder, original in protected_blocks:
+ if placeholder in result:
+ result = result.replace(placeholder, original)
+ else:
+ # Placeholder was lost during compression — append the block
+ logger.warning(
+ "Tag placeholder lost during compression, appending: %s",
+ original[:80],
+ )
+ result = result + "\n" + original
+
+ return result
diff --git a/tests/test_tag_protection_integration.py b/tests/test_tag_protection_integration.py
new file mode 100644
index 000000000..ba58ecb1d
--- /dev/null
+++ b/tests/test_tag_protection_integration.py
@@ -0,0 +1,184 @@
+"""Integration tests: verify tags survive the full compress() pipeline."""
+
+from __future__ import annotations
+
+import os
+
+import pytest
+
+
+class TestCompressPreservesTags:
+ """End-to-end tests: compress() with tagged content."""
+
+ def test_system_reminder_survives(self):
+ """<system-reminder> tags in tool output survive compression."""
+ from headroom import compress
+
+ messages = [
+ {"role": "user", "content": "What are the rules?"},
+ {
+ "role": "tool",
+ "tool_call_id": "call_1",
+ "content": (
+ "Here is a very long and verbose explanation of the system rules "
+ "that contains a lot of unnecessary filler words and repetitive "
+ "phrasing that could be compressed significantly. "
+ "<system-reminder>You must always validate input before processing. "
+ "Never skip authentication checks.</system-reminder> "
+ "The rest of this text is also quite verbose and contains additional "
+ "unnecessary details that add tokens without adding information value "
+ "to the overall response that the language model needs to generate."
+ ),
+ },
+ ]
+ result = compress(messages, model="claude-sonnet-4-5-20250929")
+ output = str(result.messages[-1].get("content", ""))
+
+ assert "<system-reminder>" in output
+ assert "</system-reminder>" in output
+ assert "validate input" in output
+ assert "authentication checks" in output
+
+ def test_tool_call_tags_survive(self):
+ """<tool_call> tags survive compression."""
+ from headroom import compress
+
+ messages = [
+ {"role": "user", "content": "Search for results"},
+ {
+ "role": "tool",
+ "tool_call_id": "call_1",
+ "content": (
+ "Processing the search request with extensive verbose output "
+ "that includes many redundant descriptions and unnecessary detail. "
+ '<tool_call>{"name": "search", "args": {"query": "test"}}</tool_call> '
+ "Additional verbose context that repeats information already stated "
+ "in the previous paragraphs about the search functionality and its "
+ "various capabilities and features that are not directly relevant."
+ ),
+ },
+ ]
+ result = compress(messages, model="claude-sonnet-4-5-20250929")
+ output = str(result.messages[-1].get("content", ""))
+
+ assert "<tool_call>" in output
+ assert "</tool_call>" in output
+ assert '"name": "search"' in output
+
+ def test_thinking_tags_survive(self):
+ """<thinking> tags survive compression."""
+ from headroom import compress
+
+ messages = [
+ {"role": "user", "content": "Analyze this data"},
+ {
+ "role": "tool",
+ "tool_call_id": "call_1",
+ "content": (
+ "The analysis produced extensive results with much verbose "
+ "explanatory text that describes methodology in great detail. "
+ "<thinking>Step 1: Parse input. Step 2: Validate schema. "
+ "Step 3: Run inference.</thinking> "
+ "Further verbose explanation of the analytical process and its "
+ "various stages and intermediate results and observations that "
+ "could be expressed much more concisely without losing meaning."
+ ),
+ },
+ ]
+ result = compress(messages, model="claude-sonnet-4-5-20250929")
+ output = str(result.messages[-1].get("content", ""))
+
+ assert "<thinking>" in output
+ assert "</thinking>" in output
+ assert "Step 1: Parse input" in output
+
+ def test_html_tags_still_compressible(self):
+ """Standard HTML tags are NOT protected — they're just text to the compressor."""
+ from headroom.transforms.tag_protector import protect_tags
+
+ html_text = "<div>Some content</div> <span>More content</span>"
+ cleaned, protected = protect_tags(html_text)
+
+ # HTML tags should NOT be protected
+ assert protected == []
+ assert "<div>" in cleaned
+
+ def test_multiple_custom_tags_in_messages(self):
+ """Multiple different custom tags all survive."""
+ from headroom import compress
+
+ messages = [
+ {"role": "user", "content": "What should I do?"},
+ {
+ "role": "tool",
+ "tool_call_id": "call_1",
+ "content": (
+ "Very verbose introductory text with many unnecessary words. "
+ "<system-reminder>Rule 1: Always validate</system-reminder> "
+ "More verbose middle text repeating previous information. "
+ "<context>session_id=abc-123</context> "
+ "Additional verbose concluding text with redundant information. "
+ "<IMPORTANT>Never expose API keys</IMPORTANT> "
+ "Final paragraph of verbose text."
+ ),
+ },
+ ]
+ result = compress(messages, model="claude-sonnet-4-5-20250929")
+ output = str(result.messages[-1].get("content", ""))
+
+ assert "<system-reminder>" in output
+ assert "<context>" in output
+ assert "<IMPORTANT>" in output
+
+
+ not os.environ.get("ANTHROPIC_API_KEY"),
+ reason="ANTHROPIC_API_KEY not set",
+)
+class TestRealAPIWithTags:
+ """Real API integration: verify Claude can use tagged content after compression."""
+
+ def test_compressed_tags_usable_by_llm(self):
+ """Send compressed content with tags to Claude, verify it references them."""
+ from anthropic import Anthropic
+
+ from headroom import compress
+
+ client = Anthropic()
+
+ messages = [
+ {"role": "user", "content": "What is the secret code in the system reminder?"},
+ {
+ "role": "assistant",
+ "content": "Let me check the system configuration.",
+ },
+ {
+ "role": "user",
+ "content": (
+ "Here is a very long and detailed configuration document with "
+ "extensive verbose descriptions of various system parameters and "
+ "their default values and recommended settings for production "
+ "deployment scenarios across different cloud providers. "
+ "<system-reminder>The secret code is ALPHA-7742-BRAVO</system-reminder> "
+ "Additional verbose documentation about system architecture and "
+ "deployment patterns and scaling strategies and monitoring setup "
+ "and alerting configuration and backup procedures."
+ ),
+ },
+ ]
+
+ result = compress(messages, model="claude-sonnet-4-5-20250929")
+
+ # Verify tags survived compression
+ last_content = str(result.messages[-1].get("content", ""))
+ assert "<system-reminder>" in last_content
+ assert "ALPHA-7742-BRAVO" in last_content
+
+ # Send to Claude and verify it can read the tagged content
+ response = client.messages.create(
+ model="claude-sonnet-4-5-20250929",
+ max_tokens=200,
+ messages=result.messages,
+ )
+ answer = response.content[0].text.lower()
+ assert "alpha-7742-bravo" in answer or "alpha" in answer
diff --git a/tests/test_transforms/test_tag_protector.py b/tests/test_transforms/test_tag_protector.py
new file mode 100644
index 000000000..9b0a1de6b
--- /dev/null
+++ b/tests/test_transforms/test_tag_protector.py
@@ -0,0 +1,183 @@
+"""Tests for tag_protector: protect custom/workflow XML tags from compression."""
+
+from __future__ import annotations
+
+from headroom.transforms.tag_protector import (
+ KNOWN_HTML_TAGS,
+ _is_html_tag,
+ protect_tags,
+ restore_tags,
+)
+
+
+class TestKnownHTMLTags:
+ def test_common_tags_are_html(self):
+ for tag in ["div", "span", "p", "a", "table", "tr", "td", "h1", "body", "html"]:
+ assert _is_html_tag(tag), f"{tag} should be recognized as HTML"
+
+ def test_case_insensitive(self):
+ assert _is_html_tag("DIV")
+ assert _is_html_tag("Span")
+ assert _is_html_tag("TABLE")
+
+ def test_custom_tags_not_html(self):
+ for tag in [
+ "system-reminder",
+ "tool_call",
+ "thinking",
+ "EXTREMELY_IMPORTANT",
+ "context",
+ "user-prompt-submit-hook",
+ "result",
+ "anthrpoic_thinking",
+ ]:
+ assert not _is_html_tag(tag), f"{tag} should NOT be recognized as HTML"
+
+ def test_tag_set_not_empty(self):
+ assert len(KNOWN_HTML_TAGS) > 100
+
+
+class TestProtectTags:
+ def test_no_tags_passthrough(self):
+ text = "Just plain text with no tags at all."
+ cleaned, protected = protect_tags(text)
+ assert cleaned == text
+ assert protected == []
+
+ def test_no_angle_brackets_fast_path(self):
+ text = "No angle brackets here"
+ cleaned, protected = protect_tags(text)
+ assert cleaned == text
+ assert protected == []
+
+ def test_html_tags_not_protected(self):
+ text = "<div>Some content</div>"
+ cleaned, protected = protect_tags(text)
+ assert cleaned == text
+ assert protected == []
+
+ def test_custom_tag_protected(self):
+ text = "Before <system-reminder>Important rule</system-reminder> After"
+ cleaned, protected = protect_tags(text)
+ assert "<system-reminder>" not in cleaned
+ assert "Important rule" not in cleaned
+ assert "Before" in cleaned
+ assert "After" in cleaned
+ assert len(protected) == 1
+ assert protected[0][1] == "<system-reminder>Important rule</system-reminder>"
+
+ def test_multiple_custom_tags(self):
+ text = "<thinking>Step 1</thinking> middle <context>data</context>"
+ cleaned, protected = protect_tags(text)
+ assert len(protected) == 2
+ assert "<thinking>" not in cleaned
+ assert "<context>" not in cleaned
+
+ def test_custom_tag_with_attributes(self):
+ text = '<context key="session" type="persistent">user data</context>'
+ cleaned, protected = protect_tags(text)
+ assert len(protected) == 1
+ assert 'key="session"' in protected[0][1]
+
+ def test_self_closing_custom_tag(self):
+ text = "Text <marker/> more text"
+ cleaned, protected = protect_tags(text)
+ assert len(protected) == 1
+ assert protected[0][1] == "<marker/>"
+
+ def test_self_closing_html_tag_not_protected(self):
+ text = "Text <br/> more <hr/> text"
+ cleaned, protected = protect_tags(text)
+ assert cleaned == text
+ assert protected == []
+
+ def test_mixed_html_and_custom(self):
+ text = "<div>HTML content</div> <system-reminder>Rule</system-reminder> <p>More HTML</p>"
+ cleaned, protected = protect_tags(text)
+ assert "<div>" in cleaned
+ assert "<p>" in cleaned
+ assert "<system-reminder>" not in cleaned
+ assert len(protected) == 1
+
+ def test_nested_custom_tags(self):
+ text = "<outer><inner>deep content</inner></outer>"
+ cleaned, protected = protect_tags(text)
+ # Both should be protected (inner first, then outer)
+ assert "<outer>" not in cleaned
+ assert "<inner>" not in cleaned
+ assert len(protected) >= 1
+
+ def test_real_workflow_tags(self):
+ """Tags actually used in LLM workflows."""
+ tags = [
+ "<tool_call>search({query: 'test'})</tool_call>",
+ "<thinking>Let me analyze this step by step</thinking>",
+ "<EXTREMELY_IMPORTANT>Never skip validation</EXTREMELY_IMPORTANT>",
+ "<user-prompt-submit-hook>check permissions</user-prompt-submit-hook>",
+ "<system-reminder>Follow these rules exactly</system-reminder>",
+ "<result>Success: 42 items processed</result>",
+ ]
+ for tag_text in tags:
+ text = f"Before {tag_text} After"
+ cleaned, protected = protect_tags(text)
+ assert len(protected) == 1, f"Failed to protect: {tag_text}"
+ assert protected[0][1] == tag_text
+
+ def test_empty_string(self):
+ cleaned, protected = protect_tags("")
+ assert cleaned == ""
+ assert protected == []
+
+
+class TestProtectTagsCompressContent:
+ def test_compress_tagged_content_true(self):
+ text = "Before <system-reminder>Compressible content here</system-reminder> After"
+ cleaned, protected = protect_tags(text, compress_tagged_content=True)
+ # Tags protected, but content between them is exposed for compression
+ assert "<system-reminder>" not in cleaned
+ assert "</system-reminder>" not in cleaned
+ assert "Compressible content here" in cleaned
+ assert len(protected) == 2 # Opening tag + closing tag
+
+ def test_compress_tagged_content_false_default(self):
+ text = "Before <system-reminder>Protected content</system-reminder> After"
+ cleaned, protected = protect_tags(text)
+ assert "Protected content" not in cleaned
+ assert len(protected) == 1 # Entire block
+
+
+class TestRestoreTags:
+ def test_basic_restore(self):
+ original = "Before <system-reminder>Rule</system-reminder> After"
+ cleaned, protected = protect_tags(original)
+ restored = restore_tags(cleaned, protected)
+ assert "<system-reminder>Rule</system-reminder>" in restored
+ assert "Before" in restored
+ assert "After" in restored
+
+ def test_restore_empty_protected(self):
+ text = "No tags here"
+ assert restore_tags(text, []) == text
+
+ def test_restore_multiple(self):
+ original = "<thinking>A</thinking> gap <context>B</context>"
+ cleaned, protected = protect_tags(original)
+ restored = restore_tags(cleaned, protected)
+ assert "<thinking>A</thinking>" in restored
+ assert "<context>B</context>" in restored
+
+ def test_lost_placeholder_appended(self):
+ """If compression removes a placeholder, the block is appended."""
+ protected = [("{{HEADROOM_TAG_0}}", "<tag>data</tag>")]
+ result = restore_tags("text without placeholder", protected)
+ assert "<tag>data</tag>" in result
+
+ def test_roundtrip_preserves_content(self):
+ original = (
+ "Start <system-reminder>Rule 1: always validate</system-reminder> "
+ "middle <tool_call>search(q='test')</tool_call> end"
+ )
+ cleaned, protected = protect_tags(original)
+ restored = restore_tags(cleaned, protected)
+ assert "<system-reminder>Rule 1: always validate</system-reminder>" in restored
+ assert "<tool_call>search(q='test')</tool_call>" in restored