Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(code): quarantine Perl parser from code-aware compression (#2204)
Rod Boev8522fcbc40b7
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex 149fc76b3..897318b5c 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -1385,6 +1385,7 @@ class AnthropicHandlerMixin:else "deferred:dropped"]timing = {}+ waste_signals = Noneresult = _DeferredCompressionResult()else:diff --git a/headroom/transforms/code_compressor.py b/headroom/transforms/code_compressor.pyindex e9872d936..1023d1d12 100644--- a/headroom/transforms/code_compressor.py+++ b/headroom/transforms/code_compressor.py@@ -102,6 +102,9 @@ def _tree_sitter_importable() -> bool:return False+_UNSAFE_TREE_SITTER_LANGUAGES: frozenset[str] = frozenset({"perl"})++def _get_parser(language: str) -> Any:"""Get a tree-sitter parser for the given language.@@ -129,6 +132,8 @@ def _get_parser(language: str) -> Any:ImportError: If tree-sitter is not installed.ValueError: If language is not supported."""+ if language in _UNSAFE_TREE_SITTER_LANGUAGES:+ raise ValueError(f"Language '{language}' is quarantined for code-aware compression.")# NOTE: guard on importability (not _check_tree_sitter_available), because# _check_tree_sitter_available now performs a real end-to-end parse via# _get_parser; guarding on it here would recurse.@@ -161,7 +166,7 @@ def _get_parser(language: str) -> Any:except Exception as e:raise ValueError(f"Language '{language}' is not supported by tree-sitter. "- f"Supported: python, javascript, typescript, go, rust, java, c, cpp, csharp, perl. "+ f"Supported: python, javascript, typescript, go, rust, java, c, cpp, csharp. "f"Error: {e}") from e@@ -714,6 +719,16 @@ def detect_language(code: str) -> tuple[CodeLanguage, float]:if candidates[CodeLanguage.CPP] >= 2:candidates[CodeLanguage.C] = 0+ perl_score = candidates.get(CodeLanguage.PERL, 0)+ if perl_score > 0:+ best_non_perl = max(+ (score for lang, score in candidates.items() if lang != CodeLanguage.PERL),+ default=0,+ )+ if perl_score > best_non_perl:+ return CodeLanguage.UNKNOWN, 0.0+ candidates.pop(CodeLanguage.PERL, None)+# Phase 2: If tree-sitter available, parse with candidates and pick fewest errorsif _check_tree_sitter_available():best_lang = CodeLanguage.UNKNOWN@@ -1142,6 +1157,19 @@ class CodeAwareCompressor(Transform):language_confidence=0.0,syntax_valid=True,)+ if detected_lang == CodeLanguage.PERL:+ if self.config.fallback_to_kompress:+ return self._fallback_compress(code, original_tokens)+ return CodeCompressionResult(+ compressed=code,+ original=code,+ original_tokens=original_tokens,+ compressed_tokens=original_tokens,+ compression_ratio=1.0,+ language=detected_lang,+ language_confidence=confidence,+ syntax_valid=True,+ )# Check if tree-sitter is availableif not _check_tree_sitter_available():diff --git a/server.json b/server.jsonindex c660bf2fe..20fa7726d 100644--- a/server.json+++ b/server.json@@ -9,13 +9,13 @@"source": "github","id": "1129940957"},- "version": "0.27.0",+ "version": "0.31.0","packages": [{"registryType": "pypi","registryBaseUrl": "https://pypi.org","identifier": "headroom-ai",- "version": "0.27.0",+ "version": "0.31.0","runtimeHint": "uvx","runtimeArguments": [{diff --git a/tests/test_cli/test_wrap_zcode.py b/tests/test_cli/test_wrap_zcode.pyindex be20cf9f4..17b7a18ad 100644--- a/tests/test_cli/test_wrap_zcode.py+++ b/tests/test_cli/test_wrap_zcode.py@@ -121,7 +121,7 @@ def test_wrap_prints_proxy_urls(def fake_watcher(**kwargs): # noqa: ANN003print_fn = kwargs.get("print_setup_lines")if callable(print_fn):- print_fn()+ print_fn(kwargs["port"])with patch.object(wrap_mod, "_ensure_rtk_binary", return_value=fake_rtk):with patch.object(wrap_mod, "_run_proxy_only_watcher", side_effect=fake_watcher):diff --git a/tests/test_cold_start_fast_pass.py b/tests/test_cold_start_fast_pass.pyindex c90e915aa..6c6baa4c6 100644--- a/tests/test_cold_start_fast_pass.py+++ b/tests/test_cold_start_fast_pass.py@@ -38,6 +38,9 @@ class _DummyMetrics:async def record_failed(self, **kwargs):return None+ def record_compression_failed(self, reason: str) -> None:+ return None+async def record_rate_limited(self, **kwargs):return Nonediff --git a/tests/test_perl_scanner_safety.py b/tests/test_perl_scanner_safety.pynew file mode 100644index 000000000..ecce4b5c3--- /dev/null+++ b/tests/test_perl_scanner_safety.py@@ -0,0 +1,173 @@+from unittest.mock import patch++import pytest++import headroom.transforms.code_compressor as cc+from headroom.transforms.code_compressor import (+ CodeAwareCompressor,+ CodeCompressionResult,+ CodeCompressorConfig,+ CodeLanguage,+ unload_tree_sitter,+)+from headroom.transforms.content_router import ContentRouter, ContentRouterConfig++[email protected](autouse=True)+def _reset_tree_sitter():+ unload_tree_sitter()+ yield+ unload_tree_sitter()+++def _compressor(**overrides) -> CodeAwareCompressor:+ defaults = {+ "min_tokens_for_compression": 1,+ "fallback_to_kompress": False,+ "enable_ccr": False,+ }+ defaults.update(overrides)+ return CodeAwareCompressor(CodeCompressorConfig(**defaults))+++def _record_parser_calls(monkeypatch):+ calls: list[str] = []+ real_get_parser = cc._get_parser++ def spy(language: str):+ calls.append(language)+ return real_get_parser(language)++ monkeypatch.setattr(cc, "_get_parser", spy)+ return calls+++def test_typescript_with_decorators_stays_typescript_without_perl(monkeypatch):+ tree_sitter_installed = cc._tree_sitter_importable()+ calls = _record_parser_calls(monkeypatch)+ code = """+import { Component, Input } from "@angular/core";++@Component({+ selector: "demo-card",+ template: "<div>{{ title }}</div>",+})+export class DemoCardComponent {+ @Input() title: string = "";++ render(items: string[]): string {+ return items.map((item) => item.trim()).join(",");+ }+}+""".strip()++ result = _compressor().compress(code)++ assert result.language == CodeLanguage.TYPESCRIPT+ assert "perl" not in calls+ if tree_sitter_installed:+ assert "typescript" in calls+++def test_explicit_perl_uses_existing_fallback_without_requesting_parser(monkeypatch):+ calls = _record_parser_calls(monkeypatch)+ compressor = _compressor(fallback_to_kompress=True)+ code = "use strict;\nsub demo {\n my $value = shift;\n return $value;\n}\n"+ sentinel = CodeCompressionResult(+ compressed="kompress sentinel",+ original=code,+ original_tokens=10,+ compressed_tokens=3,+ compression_ratio=0.3,+ language=CodeLanguage.UNKNOWN,+ language_confidence=0.0,+ syntax_valid=False,+ )++ with patch.object(compressor, "_fallback_compress", return_value=sentinel) as fallback:+ result = compressor.compress(code, language="perl")++ assert result is sentinel+ fallback.assert_called_once()+ assert "perl" not in calls++[email protected]("hint", ["perl", "pl"])+def test_explicit_perl_passthrough_keeps_original_without_parser(monkeypatch, hint):+ calls = _record_parser_calls(monkeypatch)+ code = "use strict;\nsub demo {\n my $value = shift;\n return $value;\n}\n"++ result = _compressor().compress(code, language=hint)++ assert result.compressed == code+ assert result.language == CodeLanguage.PERL+ assert "perl" not in calls+++def test_real_perl_detects_unknown_without_requesting_parser(monkeypatch):+ calls = _record_parser_calls(monkeypatch)+ code = """+package Demo;+use strict;+use warnings;++sub greet {+ my ($name) = @_;+ return "hello $name";+}++1;+""".strip()++ result = _compressor().compress(code)++ assert result.language == CodeLanguage.UNKNOWN+ assert result.compressed == code+ assert "perl" not in calls+++def test_fenced_perl_router_path_never_requests_parser(monkeypatch):+ calls = _record_parser_calls(monkeypatch)+ router = ContentRouter(ContentRouterConfig(enable_code_aware=True, min_section_tokens=1))+ content = """+```perl+use strict;+use warnings;++sub greet {+ my ($name) = @_;+ return "hello $name";+}+```+""".strip()++ result = router.compress(content)++ assert "```perl" in result.compressed+ assert "perl" not in calls+++def test_go_still_uses_non_perl_parser(monkeypatch):+ tree_sitter_installed = cc._tree_sitter_importable()+ calls = _record_parser_calls(monkeypatch)+ code = """+package main++import "fmt"++func main() {+ fmt.Println("hello")+}+""".strip()++ result = _compressor().compress(code)++ assert result.language == CodeLanguage.GO+ assert "perl" not in calls+ if tree_sitter_installed:+ assert "go" in calls+++def test_get_parser_refuses_quarantined_perl():+ with pytest.raises(ValueError, match="quarantined"):+ cc._get_parser("perl")