Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(smart-crusher): match audit-safe protected_patterns against unescaped text (#4294)
Rudy Mizrahi Celeklie11afdbdf8a7
diff --git a/headroom/transforms/smart_crusher.py b/headroom/transforms/smart_crusher.pyindex 12033aada..8d57ba85b 100644--- a/headroom/transforms/smart_crusher.py+++ b/headroom/transforms/smart_crusher.py@@ -600,9 +600,10 @@ class SmartCrusher(Transform):Used both for protected-pattern matching and for identitycomparison across the crush boundary — kept rows arere-serialized by Rust, so rows are matched by content, not by- Python object identity.+ Python object identity. ``ensure_ascii=False`` so a pattern such+ as ``Kraków`` sees the characters, not ``\\u00f3`` escapes."""- return json.dumps(item, sort_keys=True, default=str)+ return json.dumps(item, sort_keys=True, default=str, ensure_ascii=False)def _row_matches_protected(self, item: Any) -> bool:text = self._canon(item)diff --git a/tests/test_transforms/test_smart_crusher_audit_safe.py b/tests/test_transforms/test_smart_crusher_audit_safe.pyindex bd53a13ff..e38520685 100644--- a/tests/test_transforms/test_smart_crusher_audit_safe.py+++ b/tests/test_transforms/test_smart_crusher_audit_safe.py@@ -399,3 +399,29 @@ def test_content_protection_no_op_when_pattern_count_preserved() -> None:assert out_text == crushed_with_markerassert was_modified is Trueassert info == "lossless:table"++[email protected]("pattern", ["Kraków", "错误", "Zürich"])+def test_protected_patterns_match_non_ascii_text(pattern: str) -> None:+ """Patterns are matched against the characters, not `\\uXXXX` escapes.+ Tool output keeps UTF-8, so a non-ASCII pattern must find its row, and+ the row must survive a real lossy crush that samples it out otherwise."""+ from headroom.transforms.smart_crusher import SmartCrusher, SmartCrusherConfig++ cities = ["Berlin", "Paris", "Madrid", "Rome", "Vienna", "Oslo", "Lisbon", "Prague"]+ items = [{"id": i, "city": cities[i % 8], "ms": (i * 37) % 999} for i in range(600)]+ items[437]["city"] = pattern+ content = json.dumps(items, ensure_ascii=False)++ crusher = SmartCrusher(+ SmartCrusherConfig(audit_safe=True, protected_patterns=[pattern]),+ with_compaction=False,+ )+ assert crusher._scan_protected_rows(content) == [items[437]]++ crushed, was_modified, _info = crusher._smart_crush_content(content)++ assert was_modified+ kept = json.loads(crushed)+ assert items[437] in kept+ assert len(kept) < len(items)