Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(code): quarantine Perl parser from code-aware compression (#2204)

Rod Boev8522fcbc40b7
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index 149fc76b3..897318b5c 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -1385,6 +1385,7 @@ class AnthropicHandlerMixin:
else "deferred:dropped"
]
timing = {}
+ waste_signals = None
result = _DeferredCompressionResult()
else:
diff --git a/headroom/transforms/code_compressor.py b/headroom/transforms/code_compressor.py
index e9872d936..1023d1d12 100644
--- a/headroom/transforms/code_compressor.py
+++ b/headroom/transforms/code_compressor.py
@@ -102,6 +102,9 @@ def _tree_sitter_importable() -> bool:
return False
+_UNSAFE_TREE_SITTER_LANGUAGES: frozenset[str] = frozenset({"perl"})
+
+
def _get_parser(language: str) -> Any:
"""Get a tree-sitter parser for the given language.
@@ -129,6 +132,8 @@ def _get_parser(language: str) -> Any:
ImportError: If tree-sitter is not installed.
ValueError: If language is not supported.
"""
+ if language in _UNSAFE_TREE_SITTER_LANGUAGES:
+ raise ValueError(f"Language '{language}' is quarantined for code-aware compression.")
# NOTE: guard on importability (not _check_tree_sitter_available), because
# _check_tree_sitter_available now performs a real end-to-end parse via
# _get_parser; guarding on it here would recurse.
@@ -161,7 +166,7 @@ def _get_parser(language: str) -> Any:
except Exception as e:
raise ValueError(
f"Language '{language}' is not supported by tree-sitter. "
- f"Supported: python, javascript, typescript, go, rust, java, c, cpp, csharp, perl. "
+ f"Supported: python, javascript, typescript, go, rust, java, c, cpp, csharp. "
f"Error: {e}"
) from e
@@ -714,6 +719,16 @@ def detect_language(code: str) -> tuple[CodeLanguage, float]:
if candidates[CodeLanguage.CPP] >= 2:
candidates[CodeLanguage.C] = 0
+ perl_score = candidates.get(CodeLanguage.PERL, 0)
+ if perl_score > 0:
+ best_non_perl = max(
+ (score for lang, score in candidates.items() if lang != CodeLanguage.PERL),
+ default=0,
+ )
+ if perl_score > best_non_perl:
+ return CodeLanguage.UNKNOWN, 0.0
+ candidates.pop(CodeLanguage.PERL, None)
+
# Phase 2: If tree-sitter available, parse with candidates and pick fewest errors
if _check_tree_sitter_available():
best_lang = CodeLanguage.UNKNOWN
@@ -1142,6 +1157,19 @@ class CodeAwareCompressor(Transform):
language_confidence=0.0,
syntax_valid=True,
)
+ if detected_lang == CodeLanguage.PERL:
+ if self.config.fallback_to_kompress:
+ return self._fallback_compress(code, original_tokens)
+ return CodeCompressionResult(
+ compressed=code,
+ original=code,
+ original_tokens=original_tokens,
+ compressed_tokens=original_tokens,
+ compression_ratio=1.0,
+ language=detected_lang,
+ language_confidence=confidence,
+ syntax_valid=True,
+ )
# Check if tree-sitter is available
if not _check_tree_sitter_available():
diff --git a/server.json b/server.json
index c660bf2fe..20fa7726d 100644
--- a/server.json
+++ b/server.json
@@ -9,13 +9,13 @@
"source": "github",
"id": "1129940957"
},
- "version": "0.27.0",
+ "version": "0.31.0",
"packages": [
{
"registryType": "pypi",
"registryBaseUrl": "https://pypi.org",
"identifier": "headroom-ai",
- "version": "0.27.0",
+ "version": "0.31.0",
"runtimeHint": "uvx",
"runtimeArguments": [
{
diff --git a/tests/test_cli/test_wrap_zcode.py b/tests/test_cli/test_wrap_zcode.py
index be20cf9f4..17b7a18ad 100644
--- a/tests/test_cli/test_wrap_zcode.py
+++ b/tests/test_cli/test_wrap_zcode.py
@@ -121,7 +121,7 @@ def test_wrap_prints_proxy_urls(
def fake_watcher(**kwargs): # noqa: ANN003
print_fn = kwargs.get("print_setup_lines")
if callable(print_fn):
- print_fn()
+ print_fn(kwargs["port"])
with patch.object(wrap_mod, "_ensure_rtk_binary", return_value=fake_rtk):
with patch.object(wrap_mod, "_run_proxy_only_watcher", side_effect=fake_watcher):
diff --git a/tests/test_cold_start_fast_pass.py b/tests/test_cold_start_fast_pass.py
index c90e915aa..6c6baa4c6 100644
--- a/tests/test_cold_start_fast_pass.py
+++ b/tests/test_cold_start_fast_pass.py
@@ -38,6 +38,9 @@ class _DummyMetrics:
async def record_failed(self, **kwargs):
return None
+ def record_compression_failed(self, reason: str) -> None:
+ return None
+
async def record_rate_limited(self, **kwargs):
return None
diff --git a/tests/test_perl_scanner_safety.py b/tests/test_perl_scanner_safety.py
new file mode 100644
index 000000000..ecce4b5c3
--- /dev/null
+++ b/tests/test_perl_scanner_safety.py
@@ -0,0 +1,173 @@
+from unittest.mock import patch
+
+import pytest
+
+import headroom.transforms.code_compressor as cc
+from headroom.transforms.code_compressor import (
+ CodeAwareCompressor,
+ CodeCompressionResult,
+ CodeCompressorConfig,
+ CodeLanguage,
+ unload_tree_sitter,
+)
+from headroom.transforms.content_router import ContentRouter, ContentRouterConfig
+
+
[email protected](autouse=True)
+def _reset_tree_sitter():
+ unload_tree_sitter()
+ yield
+ unload_tree_sitter()
+
+
+def _compressor(**overrides) -> CodeAwareCompressor:
+ defaults = {
+ "min_tokens_for_compression": 1,
+ "fallback_to_kompress": False,
+ "enable_ccr": False,
+ }
+ defaults.update(overrides)
+ return CodeAwareCompressor(CodeCompressorConfig(**defaults))
+
+
+def _record_parser_calls(monkeypatch):
+ calls: list[str] = []
+ real_get_parser = cc._get_parser
+
+ def spy(language: str):
+ calls.append(language)
+ return real_get_parser(language)
+
+ monkeypatch.setattr(cc, "_get_parser", spy)
+ return calls
+
+
+def test_typescript_with_decorators_stays_typescript_without_perl(monkeypatch):
+ tree_sitter_installed = cc._tree_sitter_importable()
+ calls = _record_parser_calls(monkeypatch)
+ code = """
+import { Component, Input } from "@angular/core";
+
+@Component({
+ selector: "demo-card",
+ template: "<div>{{ title }}</div>",
+})
+export class DemoCardComponent {
+ @Input() title: string = "";
+
+ render(items: string[]): string {
+ return items.map((item) => item.trim()).join(",");
+ }
+}
+""".strip()
+
+ result = _compressor().compress(code)
+
+ assert result.language == CodeLanguage.TYPESCRIPT
+ assert "perl" not in calls
+ if tree_sitter_installed:
+ assert "typescript" in calls
+
+
+def test_explicit_perl_uses_existing_fallback_without_requesting_parser(monkeypatch):
+ calls = _record_parser_calls(monkeypatch)
+ compressor = _compressor(fallback_to_kompress=True)
+ code = "use strict;\nsub demo {\n my $value = shift;\n return $value;\n}\n"
+ sentinel = CodeCompressionResult(
+ compressed="kompress sentinel",
+ original=code,
+ original_tokens=10,
+ compressed_tokens=3,
+ compression_ratio=0.3,
+ language=CodeLanguage.UNKNOWN,
+ language_confidence=0.0,
+ syntax_valid=False,
+ )
+
+ with patch.object(compressor, "_fallback_compress", return_value=sentinel) as fallback:
+ result = compressor.compress(code, language="perl")
+
+ assert result is sentinel
+ fallback.assert_called_once()
+ assert "perl" not in calls
+
+
[email protected]("hint", ["perl", "pl"])
+def test_explicit_perl_passthrough_keeps_original_without_parser(monkeypatch, hint):
+ calls = _record_parser_calls(monkeypatch)
+ code = "use strict;\nsub demo {\n my $value = shift;\n return $value;\n}\n"
+
+ result = _compressor().compress(code, language=hint)
+
+ assert result.compressed == code
+ assert result.language == CodeLanguage.PERL
+ assert "perl" not in calls
+
+
+def test_real_perl_detects_unknown_without_requesting_parser(monkeypatch):
+ calls = _record_parser_calls(monkeypatch)
+ code = """
+package Demo;
+use strict;
+use warnings;
+
+sub greet {
+ my ($name) = @_;
+ return "hello $name";
+}
+
+1;
+""".strip()
+
+ result = _compressor().compress(code)
+
+ assert result.language == CodeLanguage.UNKNOWN
+ assert result.compressed == code
+ assert "perl" not in calls
+
+
+def test_fenced_perl_router_path_never_requests_parser(monkeypatch):
+ calls = _record_parser_calls(monkeypatch)
+ router = ContentRouter(ContentRouterConfig(enable_code_aware=True, min_section_tokens=1))
+ content = """
+```perl
+use strict;
+use warnings;
+
+sub greet {
+ my ($name) = @_;
+ return "hello $name";
+}
+```
+""".strip()
+
+ result = router.compress(content)
+
+ assert "```perl" in result.compressed
+ assert "perl" not in calls
+
+
+def test_go_still_uses_non_perl_parser(monkeypatch):
+ tree_sitter_installed = cc._tree_sitter_importable()
+ calls = _record_parser_calls(monkeypatch)
+ code = """
+package main
+
+import "fmt"
+
+func main() {
+ fmt.Println("hello")
+}
+""".strip()
+
+ result = _compressor().compress(code)
+
+ assert result.language == CodeLanguage.GO
+ assert "perl" not in calls
+ if tree_sitter_installed:
+ assert "go" in calls
+
+
+def test_get_parser_refuses_quarantined_perl():
+ with pytest.raises(ValueError, match="quarantined"):
+ cc._get_parser("perl")