Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(transforms): stop embedded JSON pre-pass short-circuiting HTML extraction (#3620)

Xmon Dai195910b218da
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py
index 5be6ff74f..e00900eb7 100644
--- a/headroom/transforms/content_router.py
+++ b/headroom/transforms/content_router.py
@@ -3515,22 +3515,20 @@ class ContentRouter(Transform):
# call is a one-shot re-entrancy guard (NOT a depth cap). Deterministic +
# benefit-gated (no size/min thresholds) → prefix-cache- and CCR-store-
# stable, and a strict no-op when the block has no embedded JSON.
- if _allow_embedded:
- from headroom.transforms.recursive_json import route_embedded_json
-
- def _dispatch_span(span: str) -> str | None:
- strat = self._strategy_from_detection_type(_detect_content(span).content_type)
- text, _t, _c = self._apply_strategy_to_content(
- span,
- strat,
- context,
- question=question,
- bias=bias,
- _allow_embedded=False,
- )
- return text if text != span else None
-
- routed = route_embedded_json(content, _dispatch_span, tok=_estimate_tokens)
+ #
+ # HTML blocks are the one exception (#3609): whole-document extraction
+ # routinely beats a local JSON minify by orders of magnitude (the report
+ # measured a 2% splice win pre-empting a 95% extraction), so the splice
+ # must not short-circuit the parent strategy. Defer it: run the HTML
+ # dispatch below first and compute the splice in the passthrough
+ # fallback only when extraction found nothing. Lossless mode never
+ # reaches that dispatch, so it keeps the immediate-return behavior
+ # (JSON minification is itself byte-lossless).
+ _defer_embedded_for_html = (
+ _allow_embedded and strategy is CompressionStrategy.HTML and not self.config.lossless
+ )
+ if _allow_embedded and not _defer_embedded_for_html:
+ routed = self._embedded_json_route(content, context, question, bias)
if routed is not None:
return routed, _estimate_tokens(routed), ["embedded_json"]
@@ -3830,9 +3828,15 @@ class ContentRouter(Transform):
# ``result.extracted is None`` path (chain
# ``[html, passthrough]``). A real extraction is
# byte-identical to the historical ``result.extracted``.
+ # An EMPTY extraction (trafilatura found no article text)
+ # collapses to None too: a blank page is "nothing
+ # extracts", and returning "" here would discard the
+ # whole block instead of falling through (#3609).
output = self._registry_compress("html", strategy, content, context, bias)
compressed = (
- output.content if output is not None and output.compressed else None
+ output.content
+ if output is not None and output.compressed and output.content.strip()
+ else None
)
# A page that is all <script>/<style> extracts to "".
# That is "nothing extracted" (fall through to the
@@ -4067,6 +4071,16 @@ class ContentRouter(Transform):
)
return compressed, compressed_tokens, strategy_chain
+ # #3609: deferred embedded-JSON splice. The HTML strategy ran above and
+ # extracted nothing (``compressed`` is still None), so the local JSON
+ # win — held back at the top to give whole-document extraction first
+ # refusal — is now the best available outcome.
+ if _defer_embedded_for_html:
+ routed = self._embedded_json_route(content, context, question, bias)
+ if routed is not None:
+ strategy_chain.append("embedded_json")
+ return routed, _estimate_tokens(routed), strategy_chain
+
# Fallback: return unchanged
if self.config.enable_dense_line_elision:
dense = self._elide_dense(content, context)
@@ -4140,6 +4154,32 @@ class ContentRouter(Transform):
return None
return elided, _estimate_tokens(elided)
+ def _embedded_json_route(
+ self, content: str, context: str, question: str | None, bias: float
+ ) -> str | None:
+ """Route embedded JSON spans inside ``content`` through the JSON compressors.
+
+ Extracted from the ``_apply_strategy_to_content`` pre-pass so the HTML
+ strategy can defer the splice (#3609) and retry it in the passthrough
+ fallback. Returns the spliced block, or ``None`` when nothing
+ safe/smaller applied.
+ """
+ from headroom.transforms.recursive_json import route_embedded_json
+
+ def _dispatch_span(span: str) -> str | None:
+ strat = self._strategy_from_detection_type(_detect_content(span).content_type)
+ text, _t, _c = self._apply_strategy_to_content(
+ span,
+ strat,
+ context,
+ question=question,
+ bias=bias,
+ _allow_embedded=False,
+ )
+ return text if text != span else None
+
+ return route_embedded_json(content, _dispatch_span, tok=_estimate_tokens)
+
def _try_ml_compressor(
self,
content: str,
diff --git a/tests/test_transforms_content_router.py b/tests/test_transforms_content_router.py
index 7b0211c64..973cb843d 100644
--- a/tests/test_transforms_content_router.py
+++ b/tests/test_transforms_content_router.py
@@ -2151,3 +2151,102 @@ def test_prompt_text_compresses_without_replay_guarantee(monkeypatch: pytest.Mon
min_tokens_to_compress=1,
)
assert _TASK in calls
+
+
+def _html_doc_with_embedded_json(*, padded: bool = True) -> str:
+ """A whole HTML document with one embedded JSON array-of-objects.
+
+ The JSON sits in a single-line ``<script>`` so the mixed-content regex
+ heuristics do not fire: the block detects as pure HTML and takes the
+ ``_compress_pure`` path. ``padded`` keeps the default ``json.dumps``
+ separators so the array is minifiable — a compact array gives the JSON
+ route no benefit and both passes no-op.
+ """
+ rows = [
+ {
+ "id": i,
+ "name": f"item-{i}",
+ "tags": ["alpha", "beta"],
+ "score": i * 3,
+ "description": "a routine record with repeated descriptive fields",
+ }
+ for i in range(60)
+ ]
+ embedded = json.dumps(rows) if padded else json.dumps(rows, separators=(",", ":"))
+ paras = "".join(
+ f"<p>Paragraph {i}: distributed systems rely on consensus protocols to "
+ f"agree on state. Raft separates leader election from log replication, "
+ f"which makes it easier to reason about.</p>\n"
+ for i in range(30)
+ )
+ return (
+ "<!DOCTYPE html>\n<html><head><title>Raft Notes</title></head>\n<body>\n"
+ "<article><h1>Raft consensus notes</h1>\n"
+ + paras
+ + '<script type="application/json">'
+ + embedded
+ + "</script>\n"
+ + paras
+ + "</article>\n</body></html>\n"
+ )
+
+
+def test_html_with_embedded_json_runs_extractor_not_json_shortcut() -> None:
+ """#3609: the embedded-JSON pre-pass must not short-circuit HTML extraction.
+
+ Before the fix, a small local JSON minify (~10% win) returned immediately
+ from ``_apply_strategy_to_content`` and whole-document extraction (~36%
+ win on this document) never ran; the issue reported the same shape at
+ 2% vs 95.5%. The splice is now deferred for HTML: extraction gets first
+ refusal and the JSON route only fires when extraction finds nothing.
+ """
+ pytest.importorskip("trafilatura")
+ doc = _html_doc_with_embedded_json()
+ assert _detect_content(doc).content_type is ContentType.HTML
+ assert not is_mixed_content(doc)
+
+ result = ContentRouter().compress(doc)
+
+ assert result.strategy_used is CompressionStrategy.HTML
+ assert result.strategy_chain == ["html"]
+ # extraction ran: markup is gone and the document collapsed hard
+ # (the pre-fix JSON shortcut left <html> in place at ~10% reduction)
+ assert "<html>" not in result.compressed
+ assert len(result.compressed) < len(doc) * 3 // 5
+
+
+def test_html_extraction_miss_still_gets_deferred_json_splice() -> None:
+ """Deferring must not LOSE the embedded-JSON win when extraction finds nothing.
+
+ A shell-only HTML document makes the extractor come back empty; the
+ deferred splice then still compresses the embedded array. Before the fix
+ this shape short-circuited at the top (chain ``["embedded_json"]``); now
+ the block first proves the HTML strategy is a miss (chain
+ ``["html", "embedded_json"]``) with the same spliced output.
+ """
+ pytest.importorskip("trafilatura")
+ embedded = json.dumps(
+ [
+ {"id": i, "name": f"item-{i}", "tags": ["alpha", "beta"], "score": i * 3}
+ for i in range(40)
+ ]
+ )
+ shell = (
+ "<!DOCTYPE html>\n<html><head><title>t</title></head>\n<body>\n"
+ '<script type="application/json">' + embedded + "</script>\n"
+ "</body></html>\n"
+ )
+ router = ContentRouter()
+ # sanity: extraction finds nothing on this shell (empty or None)
+ extractor = router._get_html_extractor()
+ assert extractor is not None
+ assert not (extractor.extract(shell).extracted or "").strip()
+
+ text, _tokens, chain = router._apply_strategy_to_content(
+ shell, CompressionStrategy.HTML, "", None, 1.0
+ )
+
+ assert chain == ["html", "embedded_json"]
+ assert len(text) < len(shell)
+ json_marker = '"id":'
+ assert json_marker not in text # the array was compressed away