Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(smart-crusher): match audit-safe protected_patterns against unescaped text (#4294)

Rudy Mizrahi Celeklie11afdbdf8a7
diff --git a/headroom/transforms/smart_crusher.py b/headroom/transforms/smart_crusher.py
index 12033aada..8d57ba85b 100644
--- a/headroom/transforms/smart_crusher.py
+++ b/headroom/transforms/smart_crusher.py
@@ -600,9 +600,10 @@ class SmartCrusher(Transform):
Used both for protected-pattern matching and for identity
comparison across the crush boundary — kept rows are
re-serialized by Rust, so rows are matched by content, not by
- Python object identity.
+ Python object identity. ``ensure_ascii=False`` so a pattern such
+ as ``Kraków`` sees the characters, not ``\\u00f3`` escapes.
"""
- return json.dumps(item, sort_keys=True, default=str)
+ return json.dumps(item, sort_keys=True, default=str, ensure_ascii=False)
def _row_matches_protected(self, item: Any) -> bool:
text = self._canon(item)
diff --git a/tests/test_transforms/test_smart_crusher_audit_safe.py b/tests/test_transforms/test_smart_crusher_audit_safe.py
index bd53a13ff..e38520685 100644
--- a/tests/test_transforms/test_smart_crusher_audit_safe.py
+++ b/tests/test_transforms/test_smart_crusher_audit_safe.py
@@ -399,3 +399,29 @@ def test_content_protection_no_op_when_pattern_count_preserved() -> None:
assert out_text == crushed_with_marker
assert was_modified is True
assert info == "lossless:table"
+
+
[email protected]("pattern", ["Kraków", "错误", "Zürich"])
+def test_protected_patterns_match_non_ascii_text(pattern: str) -> None:
+ """Patterns are matched against the characters, not `\\uXXXX` escapes.
+ Tool output keeps UTF-8, so a non-ASCII pattern must find its row, and
+ the row must survive a real lossy crush that samples it out otherwise."""
+ from headroom.transforms.smart_crusher import SmartCrusher, SmartCrusherConfig
+
+ cities = ["Berlin", "Paris", "Madrid", "Rome", "Vienna", "Oslo", "Lisbon", "Prague"]
+ items = [{"id": i, "city": cities[i % 8], "ms": (i * 37) % 999} for i in range(600)]
+ items[437]["city"] = pattern
+ content = json.dumps(items, ensure_ascii=False)
+
+ crusher = SmartCrusher(
+ SmartCrusherConfig(audit_safe=True, protected_patterns=[pattern]),
+ with_compaction=False,
+ )
+ assert crusher._scan_protected_rows(content) == [items[437]]
+
+ crushed, was_modified, _info = crusher._smart_crush_content(content)
+
+ assert was_modified
+ kept = json.loads(crushed)
+ assert items[437] in kept
+ assert len(kept) < len(items)