Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(transforms/content-router): route grep/log output away from HTML extractor (#1719)

Tejas Chopra0d18ef26f4d1
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py
index aa6c61a6a..191b1ec1b 100644
--- a/headroom/transforms/content_router.py
+++ b/headroom/transforms/content_router.py
@@ -58,7 +58,7 @@ from ..config import (
from ..parser import CCR_RETRIEVAL_MARKER_RE
from ..tokenizer import Tokenizer
from .base import Transform
-from .content_detector import ContentType, DetectionResult
+from .content_detector import ContentType, DetectionResult, _try_detect_log, _try_detect_search
from .content_detector import detect_content_type as _regex_detect_content_type
from .error_detection import content_has_strong_error_indicators
@@ -302,6 +302,18 @@ def _detect_content(content: str) -> DetectionResult:
)
return _regex_detect_content_type(content)
+ # HTML misroute guard (native/magika path): dense punctuation in grep
+ # output and build logs (file paths, </>, brackets) can read as markup, so
+ # the native detector tags real search results / logs as HTML. Routing those
+ # to the HTML article-extractor is lossy — it strips code and identifiers.
+ # When the structural log/search detectors positively claim the payload,
+ # trust them over the HTML verdict: tracebacks/build output win as LOG
+ # (checked first), path:line grep output routes to SEARCH.
+ if content_type is ContentType.HTML:
+ override = _try_detect_log(content) or _try_detect_search(content)
+ if override is not None:
+ return override
+
if content_type is ContentType.PLAIN_TEXT:
regex_result = _regex_detect_content_type(content)
if regex_result.content_type is not ContentType.PLAIN_TEXT:
diff --git a/tests/test_transforms_content_router.py b/tests/test_transforms_content_router.py
index 7a62873d8..02498bbac 100644
--- a/tests/test_transforms_content_router.py
+++ b/tests/test_transforms_content_router.py
@@ -1024,3 +1024,48 @@ def test_detect_content_sees_through_tool_output_envelope() -> None:
wrapped = f"<returncode>0</returncode>\n<output>\n{code}\n</output>"
assert _detect_content(wrapped).content_type is ContentType.SOURCE_CODE
assert _detect_content(wrapped).content_type is _detect_content(code).content_type
+
+
+def test_detect_content_overrides_html_misroute_for_grep_and_logs(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Regression: the native detector (magika) tags dense grep output and
+ build logs as HTML because file paths and </> read as markup. Routing those
+ to the HTML article-extractor is lossy (it strips code + identifiers). When
+ the structural log/search detectors positively claim the payload they
+ override the HTML verdict (log checked first so tracebacks win); genuine
+ HTML with no such structure is left as HTML."""
+ import headroom._core as _core
+
+ monkeypatch.setenv("HEADROOM_DETECT_BACKEND", "rust")
+ monkeypatch.setattr(
+ _core,
+ "detect_content_type",
+ lambda content: SimpleNamespace(content_type="html", confidence=1.0, metadata={}),
+ )
+
+ # grep over HTML template files: native says html, but it is search results.
+ grep = "\n".join(
+ f'templates/pages/dashboard_{i}.html:{10 + i}: <div class="card" data-id="{i}">'
+ for i in range(6)
+ )
+ assert _detect_content(grep).content_type is ContentType.SEARCH_RESULTS
+
+ # build/error log misread as html -> LOG wins (checked before search).
+ build_log = "\n".join(
+ [
+ "ERROR failed to compile module widget",
+ "WARNING deprecated call near <template>",
+ "Traceback (most recent call last):",
+ "ERROR build aborted after 2 retries",
+ ]
+ )
+ assert _detect_content(build_log).content_type is ContentType.BUILD_OUTPUT
+
+ # genuine HTML article: no grep/log structure -> override does not fire.
+ html = (
+ "<!DOCTYPE html>\n<html><head><title>x</title></head>"
+ "<body><main><section><p>An article about widgets and gadgets.</p>"
+ "</section></main></body></html>"
+ )
+ assert _detect_content(html).content_type is ContentType.HTML