Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(transforms/content-router): route grep/log output away from HTML extractor (#1719)
Tejas Chopra0d18ef26f4d1
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.pyindex aa6c61a6a..191b1ec1b 100644--- a/headroom/transforms/content_router.py+++ b/headroom/transforms/content_router.py@@ -58,7 +58,7 @@ from ..config import (from ..parser import CCR_RETRIEVAL_MARKER_REfrom ..tokenizer import Tokenizerfrom .base import Transform-from .content_detector import ContentType, DetectionResult+from .content_detector import ContentType, DetectionResult, _try_detect_log, _try_detect_searchfrom .content_detector import detect_content_type as _regex_detect_content_typefrom .error_detection import content_has_strong_error_indicators@@ -302,6 +302,18 @@ def _detect_content(content: str) -> DetectionResult:)return _regex_detect_content_type(content)+ # HTML misroute guard (native/magika path): dense punctuation in grep+ # output and build logs (file paths, </>, brackets) can read as markup, so+ # the native detector tags real search results / logs as HTML. Routing those+ # to the HTML article-extractor is lossy — it strips code and identifiers.+ # When the structural log/search detectors positively claim the payload,+ # trust them over the HTML verdict: tracebacks/build output win as LOG+ # (checked first), path:line grep output routes to SEARCH.+ if content_type is ContentType.HTML:+ override = _try_detect_log(content) or _try_detect_search(content)+ if override is not None:+ return override+if content_type is ContentType.PLAIN_TEXT:regex_result = _regex_detect_content_type(content)if regex_result.content_type is not ContentType.PLAIN_TEXT:diff --git a/tests/test_transforms_content_router.py b/tests/test_transforms_content_router.pyindex 7a62873d8..02498bbac 100644--- a/tests/test_transforms_content_router.py+++ b/tests/test_transforms_content_router.py@@ -1024,3 +1024,48 @@ def test_detect_content_sees_through_tool_output_envelope() -> None:wrapped = f"<returncode>0</returncode>\n<output>\n{code}\n</output>"assert _detect_content(wrapped).content_type is ContentType.SOURCE_CODEassert _detect_content(wrapped).content_type is _detect_content(code).content_type+++def test_detect_content_overrides_html_misroute_for_grep_and_logs(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Regression: the native detector (magika) tags dense grep output and+ build logs as HTML because file paths and </> read as markup. Routing those+ to the HTML article-extractor is lossy (it strips code + identifiers). When+ the structural log/search detectors positively claim the payload they+ override the HTML verdict (log checked first so tracebacks win); genuine+ HTML with no such structure is left as HTML."""+ import headroom._core as _core++ monkeypatch.setenv("HEADROOM_DETECT_BACKEND", "rust")+ monkeypatch.setattr(+ _core,+ "detect_content_type",+ lambda content: SimpleNamespace(content_type="html", confidence=1.0, metadata={}),+ )++ # grep over HTML template files: native says html, but it is search results.+ grep = "\n".join(+ f'templates/pages/dashboard_{i}.html:{10 + i}: <div class="card" data-id="{i}">'+ for i in range(6)+ )+ assert _detect_content(grep).content_type is ContentType.SEARCH_RESULTS++ # build/error log misread as html -> LOG wins (checked before search).+ build_log = "\n".join(+ [+ "ERROR failed to compile module widget",+ "WARNING deprecated call near <template>",+ "Traceback (most recent call last):",+ "ERROR build aborted after 2 retries",+ ]+ )+ assert _detect_content(build_log).content_type is ContentType.BUILD_OUTPUT++ # genuine HTML article: no grep/log structure -> override does not fire.+ html = (+ "<!DOCTYPE html>\n<html><head><title>x</title></head>"+ "<body><main><section><p>An article about widgets and gadgets.</p>"+ "</section></main></body></html>"+ )+ assert _detect_content(html).content_type is ContentType.HTML