Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(transforms): keep record-bearing JSON out of Kompress (#3673) (#3880)

afernandez-stratio8dbbd1d8967f
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py
index d2452c235..682f0deda 100644
--- a/headroom/transforms/content_router.py
+++ b/headroom/transforms/content_router.py
@@ -91,7 +91,7 @@ from .lossless_provider import (
get_lossless_verifier,
)
from .mixed_content import ContentSection, mixed_content_indicators
-from .relevance_split import build_relevance_query, plan_relevance_split
+from .relevance_split import build_relevance_query, plan_relevance_split, segment
logger = logging.getLogger(__name__)
@@ -895,6 +895,73 @@ def _content_is_valid_json(content: str) -> bool:
return True
+def _protected_json_spans(content: str) -> list[tuple[int, int]]:
+ """JSON spans of ``content`` that a prose model must not be handed (#3673).
+
+ Kompress has no notion of JSON grammar, so JSON bytes in its input are at
+ risk of a destructive span around ``},{``. The scan is the deterministic
+ balanced-span walk embedded-JSON routing already uses, so prefixed tool
+ output and relevance fragments are seen too. Two shapes qualify:
+
+ * a span carrying an array of objects — deleting the ``},{`` between two
+ records leaves VALID JSON, so the loss is invisible to every parser, log
+ line and model downstream, which is what made #3673 invisible;
+ * a block that is one JSON document end to end, which covers welding two
+ keys of a lone object by the same argument.
+
+ A stream of SEPARATE top-level values (search hits as space-joined objects,
+ JSONL) is deliberately not protected. Welding two of those yields an
+ unparseable record rather than a silently shorter document, so the
+ invisible-loss argument does not apply — and protecting them costs a large,
+ legitimate savings class: it turns forced Kompress into a no-op on a
+ 160-object search payload (see
+ ``test_force_kompress_routes_anthropic_tool_result_to_targeted_kompress``).
+ """
+ from .recursive_json import carries_record_array, scan_json_documents
+
+ spans, complete = scan_json_documents(content)
+ if not complete:
+ # The scan stopped at its budget (input built to defeat the linear walk),
+ # so a record array past that point cannot be ruled out. Protect the
+ # whole block: declining a prose compression is recoverable, a silently
+ # deleted record is not.
+ return [(0, len(content))]
+ if not spans:
+ return []
+ if (
+ len(spans) == 1
+ and not content[: spans[0][0]].strip()
+ and not content[spans[0][1] :].strip()
+ ):
+ return spans
+ return [(a, b) for a, b in spans if carries_record_array(content[a:b])]
+
+
+def _contains_protected_json(content: str) -> bool:
+ """True when ``content`` is, or contains, a JSON span worth protecting."""
+ return bool(_protected_json_spans(content))
+
+
+def _json_straddles_segments(content: str) -> bool:
+ """True when a protected JSON span crosses a boundary ``segment()`` cuts at.
+
+ The relevance split windows by line with no JSON awareness, so a payload
+ misclassified as LOG/SEARCH can be cut mid-structure and the pieces handed
+ to Kompress as fragments no span check can recognise. Declining only for a
+ straddling span keeps the split working on line-contained JSON (JSONL
+ logs), which is the shape it exists for.
+ """
+ spans = _protected_json_spans(content)
+ if not spans:
+ return False
+ cuts: list[int] = []
+ offset = 0
+ for piece in segment(content)[:-1]:
+ offset += len(piece)
+ cuts.append(offset)
+ return any(a < cut < b for a, b in spans for cut in cuts)
+
+
def _mixed_indicators(content: str) -> dict[str, bool]:
return mixed_content_indicators(content)
@@ -4290,6 +4357,14 @@ class ContentRouter(Transform):
Returns:
Tuple of (compressed, token_count).
"""
+ # Kompress is a prose model: dropping a span around ``},{`` can remove
+ # an entire JSON record while leaving the remaining document valid
+ # (#3673). This boundary sees the final Kompress input, which may be a
+ # prefixed block or a fragment rather than a whole JSON document, so
+ # guard every parseable JSON span before the prose model runs.
+ if _contains_protected_json(content):
+ return content, _estimate_tokens(content)
+
from .tag_protector import protect_tags, restore_tags
# Protect custom tags before any ML compression
@@ -4632,6 +4707,11 @@ class ContentRouter(Transform):
normal path when the scorer is unavailable, the query is empty, nothing
is dropped, or the split doesn't beat plain compaction. Never raises.
+ JSON the windows would cut is excluded from splitting. Without that
+ route guard, a payload misclassified as LOG/SEARCH is cut at arbitrary
+ offsets before the JSON guard sees it (#3673); line-contained JSON
+ (JSONL) still splits, which is the shape this exists for.
+
Embedding cost is bounded two ways: the model is pre-warmed off the
request thread (BM25 until it's ready, see _get_relevance_scorer) and
outputs segmenting into more than ``relevance_max_records`` records skip
@@ -4640,6 +4720,8 @@ class ContentRouter(Transform):
scorer = self._get_relevance_scorer()
if scorer is None or not query.strip():
return None
+ if _json_straddles_segments(content):
+ return None
from .lossless_compaction import compact_lossless
try:
diff --git a/headroom/transforms/recursive_json.py b/headroom/transforms/recursive_json.py
index e6d7b1665..eb29b9e8c 100644
--- a/headroom/transforms/recursive_json.py
+++ b/headroom/transforms/recursive_json.py
@@ -70,26 +70,124 @@ def _match_span(text: str, start: int) -> int | None:
return None
-def _spans(text: str) -> list[tuple[int, int]]:
- """Deterministic list of ``(start, end)`` for top-level balanced JSON spans.
- Nested spans are not returned separately — the dispatch handles depth."""
+#: Backstop for ``_scan_spans``: characters it may walk, per character of input,
+#: before it stops looking for further spans. The memoized walk below is linear
+#: on real payloads (source, logs, JSON, prose), so this only bounds inputs built
+#: to defeat the memo. Callers that must not miss a span check ``complete``.
+_SCAN_BUDGET_PER_CHAR = 4
+_SCAN_BUDGET_FLOOR = 4096
+
+
+def _scan_from(text: str, start: int, known: dict[int, int | None]) -> tuple[int | None, int]:
+ """``_match_span(text, start)``, recording every nested container's verdict.
+
+ A walk from ``start`` also decides the result of ``_match_span`` for every
+ ``[``/``{`` it pushes outside a string: from that position the walk's string
+ state is the same (not in a string), its stack is the suffix above that
+ bracket, so it pops, hits a mismatch or runs off the end exactly when this
+ walk does. Those verdicts go into ``known`` so the caller never walks the same
+ bytes again for them — which is what made unmatched ``{`` in source and logs
+ quadratic. Returns ``(end or None, characters walked)``.
+ """
+ stack: list[int] = []
+ in_str = esc = False
+ n = len(text)
+ for j in range(start, n):
+ ch = text[j]
+ if in_str:
+ if esc:
+ esc = False
+ elif ch == "\\":
+ esc = True
+ elif ch == '"':
+ in_str = False
+ continue
+ if ch == '"':
+ in_str = True
+ elif ch in _OPEN:
+ stack.append(j)
+ elif ch in _CLOSE:
+ if not stack or text[stack[-1]] != _PAIR[ch]:
+ for p in stack:
+ known[p] = None
+ return None, j - start + 1
+ known[stack.pop()] = j + 1
+ if not stack:
+ return j + 1, j - start + 1
+ for p in stack:
+ known[p] = None
+ return None, n - start
+
+
+def _scan_spans(text: str) -> tuple[list[tuple[int, int]], bool]:
+ """``(spans, complete)``: the top-level balanced spans ``_spans`` returns, and
+ whether the walk covered the whole text within its budget.
+
+ Same left-to-right semantics as calling ``_match_span`` at every opening
+ bracket, but each bracket's verdict is computed at most once (see
+ ``_scan_from``), so a block full of unmatched ``{`` costs one pass instead of
+ one pass per brace.
+ """
out: list[tuple[int, int]] = []
- i, n = 0, len(text)
+ known: dict[int, int | None] = {}
+ n = len(text)
+ budget = _SCAN_BUDGET_PER_CHAR * n + _SCAN_BUDGET_FLOOR
+ spent = 0
+ i = 0
while i < n:
if text[i] in _OPEN:
- end = _match_span(text, i)
+ if i in known:
+ end = known[i]
+ elif spent >= budget:
+ return out, False
+ else:
+ end, walked = _scan_from(text, i, known)
+ spent += walked
if end is not None:
out.append((i, end))
i = end
continue
i += 1
- return out
+ return out, True
+
+
+def _spans(text: str) -> list[tuple[int, int]]:
+ """Deterministic list of ``(start, end)`` for top-level balanced JSON spans.
+ Nested spans are not returned separately — the dispatch handles depth."""
+ return _scan_spans(text)[0]
+
+
+def json_document_spans(text: str) -> list[tuple[int, int]]:
+ """``(start, end)`` of every top-level balanced span of ``text`` that parses
+ as a JSON object or array, left to right. Scalars are not documents; a
+ whole-document ``text`` yields one span covering it (modulo whitespace)."""
+ return scan_json_documents(text)[0]
+
+
+def scan_json_documents(text: str) -> tuple[list[tuple[int, int]], bool]:
+ """``json_document_spans`` plus whether the scan covered all of ``text``.
+
+ ``complete`` is only ever False past the scan budget, i.e. on input built to
+ defeat the linear walk; a caller that must not miss a document treats that
+ as "may contain one".
+ """
+ spans, complete = _scan_spans(text)
+ out: list[tuple[int, int]] = []
+ for a, b in spans:
+ try:
+ parsed = json.loads(text[a:b])
+ except (ValueError, TypeError, RecursionError):
+ continue
+ if isinstance(parsed, dict | list):
+ out.append((a, b))
+ return out, complete
-def _has_routable_json(span: str) -> bool:
+def carries_record_array(span: str) -> bool:
"""True if ``span`` parses and contains an array of objects somewhere — the
- shape the JSON compressors actually act on. Cheap structural check, no size
- threshold."""
+ shape the JSON compressors act on, and the one whose record delimiters a
+ prose model can delete while leaving valid JSON behind (#3673). Cheap
+ structural check, no size threshold."""
try:
v = json.loads(span)
except (ValueError, TypeError):
@@ -141,7 +239,7 @@ def route_embedded_json(
chunk = content[a:b]
if "<<ccr:" in chunk: # R1: already compressed — never re-route
continue
- if not _has_routable_json(chunk):
+ if not carries_record_array(chunk):
continue
out = dispatch(chunk)
if out is None or out == chunk:
diff --git a/tests/test_recursive_json.py b/tests/test_recursive_json.py
index 32e8709e6..53a8ff8cb 100644
--- a/tests/test_recursive_json.py
+++ b/tests/test_recursive_json.py
@@ -6,7 +6,11 @@ from __future__ import annotations
import json
-from headroom.transforms.recursive_json import route_embedded_json
+from headroom.transforms.recursive_json import (
+ carries_record_array,
+ json_document_spans,
+ route_embedded_json,
+)
def _upper_dispatch(span: str) -> str | None:
@@ -65,3 +69,121 @@ def test_scalar_array_not_routed() -> None:
# array of scalars is not a "routable" JSON shape (no dict rows)
content = "nums: [1,2,3,4,5,6,7,8] done"
assert route_embedded_json(content, _upper_dispatch) is None
+
+
+def test_json_document_spans_finds_containers_anywhere() -> None:
+ doc = json.dumps({"domains": [{"name": "a"}, {"name": "b"}]})
+ arr = json.dumps([1, 2, 3])
+ text = "Tool result: " + doc + " and a list " + arr + " done"
+ assert [text[a:b] for a, b in json_document_spans(text)] == [doc, arr]
+
+
+def test_json_document_spans_ignores_scalars_prose_and_unbalanced_json() -> None:
+ assert json_document_spans('"just a quoted sentence"') == []
+ assert json_document_spans("42") == []
+ assert json_document_spans("prose with a [note] and {braces} but no JSON") == []
+ assert json_document_spans('{"truncated": [1, 2, 3') == []
+ assert json_document_spans("{{HEADROOM_TAG_0}}") == []
+ assert json_document_spans("") == []
+
+
+def test_json_document_spans_whole_document() -> None:
+ doc = json.dumps({"a": [1, 2]})
+ assert json_document_spans(doc) == [(0, len(doc))]
+ assert json_document_spans(" " + doc + "\n") == [(2, 2 + len(doc))]
+
+
+def test_carries_record_array_separates_the_two_shapes() -> None:
+ assert carries_record_array(json.dumps({"domains": [{"name": "a"}, {"name": "b"}]}))
+ assert carries_record_array(json.dumps([{"id": 1}, {"id": 2}]))
+ # A lone object, or an array of scalars, has no record delimiter whose
+ # deletion leaves a valid-but-shorter document.
+ assert not carries_record_array(json.dumps({"file": "src/mod.py", "line": 1}))
+ assert not carries_record_array(json.dumps([1, 2, 3]))
+ assert not carries_record_array(json.dumps([{"id": 1}]))
+
+
+# --- the span walk is linear on brace-heavy text -----------------------------
+#
+# ``_spans`` used to call ``_match_span`` at every opening bracket, and a ``{``
+# that never closes walks to the end of the text: source code and logs full of
+# unmatched braces cost one full pass per brace. ``_scan_spans`` memoizes each
+# bracket's verdict from the walk that already crossed it. These tests pin that
+# the result did not change and that the work did.
+
+
+def _reference_spans(text: str) -> list[tuple[int, int]]:
+ """The previous ``_spans``: ``_match_span`` at every opening bracket."""
+ from headroom.transforms.recursive_json import _match_span
+
+ out: list[tuple[int, int]] = []
+ i, n = 0, len(text)
+ while i < n:
+ if text[i] in "[{":
+ end = _match_span(text, i)
+ if end is not None:
+ out.append((i, end))
+ i = end
+ continue
+ i += 1
+ return out
+
+
+def test_span_walk_matches_the_previous_semantics_exactly() -> None:
+ import random
+
+ from headroom.transforms.recursive_json import _scan_spans
+
+ rng = random.Random(3673)
+ alphabet = '{}[]"\\ a,:\n'
+ samples = [
+ "".join(rng.choice(alphabet) for _ in range(rng.randint(0, 120))) for _ in range(4000)
+ ]
+ samples += [
+ 'noise {"a": [1, {"b": "}"}]} tail [1,2] {oops',
+ 'He said "a {quote" then {"k": "v"} and "another {one}"',
+ '{"esc": "a \\" } still string"} {x',
+ "[{]}] {[}] {{{{",
+ ]
+ for text in samples:
+ spans, complete = _scan_spans(text)
+ assert complete, text
+ assert spans == _reference_spans(text), text
+
+
+def test_span_walk_is_linear_on_unmatched_braces(monkeypatch) -> None:
+ import headroom.transforms.recursive_json as rj
+
+ line = ' if (flags & MASK) { log.debug("state={}", state); retry(ctx, {timeout: 30, max\n'
+ text = line * 800 # ~70 KB, about three unmatched `{` per line
+ walked = 0
+ real = rj._scan_from
+
+ def counting(t, start, known): # noqa: ANN001, ANN202
+ nonlocal walked
+ end, n = real(t, start, known)
+ walked += n
+ return end, n
+
+ monkeypatch.setattr(rj, "_scan_from", counting)
+ spans, complete = rj._scan_spans(text)
+ assert complete
+ # One pass, give or take the per-line restarts inside quoted strings. The
+ # previous walk re-read the rest of the text for every unmatched brace:
+ # hundreds of times the input size here.
+ assert walked <= 2 * len(text), (walked, len(text))
+ # Same answer as before on the same shape (a slice keeps the quadratic
+ # reference fast enough to run).
+ small = line * 40
+ assert rj._scan_spans(small)[0] == _reference_spans(small)
+
+
+def test_scan_stops_at_its_budget_and_says_so(monkeypatch) -> None:
+ import headroom.transforms.recursive_json as rj
+
+ monkeypatch.setattr(rj, "_SCAN_BUDGET_PER_CHAR", 0)
+ monkeypatch.setattr(rj, "_SCAN_BUDGET_FLOOR", 0)
+ text = 'prose {"a": 1} then [{"id": 1}, {"id": 2}]'
+ spans, complete = rj.scan_json_documents(text)
+ assert complete is False
+ assert spans == []
diff --git a/tests/test_transforms_content_router.py b/tests/test_transforms_content_router.py
index 946710f6b..f7e2aee9e 100644
--- a/tests/test_transforms_content_router.py
+++ b/tests/test_transforms_content_router.py
@@ -705,6 +705,378 @@ def test_smart_crusher_log_fallback_skipped_for_invalid_json(
assert CompressionStrategy.KOMPRESS.value in strategy_chain
+def test_smart_crusher_fallback_skips_kompress_for_valid_json_object(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Kompress must not delete records from parseable single-line JSON (#3673).
+
+ SmartCrusher passes a compact JSON object through when it has no applicable
+ array transform. The no-savings fallback used to hand the whole document to
+ Kompress; when Kompress dropped a span around ``},{``, the result remained
+ valid JSON but silently lost a record.
+ """
+ router = ContentRouter(ContentRouterConfig())
+ payload = json.dumps(
+ {
+ "domains": [
+ {"name": "first", "description": "First collection description"},
+ {"name": "second", "description": "Second collection description"},
+ {"name": "third", "description": "Third collection description"},
+ ]
+ },
+ separators=(",", ":"),
+ )
+ kompress_calls: list[str] = []
+
+ class NoopSmartCrusher:
+ def crush(self, content: str, query: str = "", bias: float = 1.0) -> SimpleNamespace:
+ return SimpleNamespace(compressed=content)
+
+ class RecordEatingKompress:
+ def is_ready(self) -> bool:
+ return True
+
+ def compress(self, content: str, **_kwargs: object) -> SimpleNamespace:
+ kompress_calls.append("kompress")
+ return SimpleNamespace(
+ compressed='{"domains":["record deleted"]}',
+ compressed_tokens=1,
+ )
+
+ monkeypatch.setattr(router, "_get_smart_crusher", lambda: NoopSmartCrusher())
+ monkeypatch.setattr(router, "_get_log_compressor", lambda: None)
+ monkeypatch.setattr(router, "_get_kompress", lambda: RecordEatingKompress())
+
+ result = router.compress(payload)
+
+ assert result.compressed == payload
+ assert kompress_calls == []
+
+
+def test_force_kompress_skips_valid_json_object(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The forced-Kompress fast path keeps the same parseable-JSON guard."""
+ router = ContentRouter(ContentRouterConfig(force_kompress_all=True))
+ router._runtime_force_kompress = True
+ payload = json.dumps({"items": [{"id": 1}, {"id": 2}]}, separators=(",", ":"))
+ calls: list[str] = []
+
+ class RecordEatingKompress:
+ def is_ready(self) -> bool:
+ return True
+
+ def compress(self, content: str, **_kwargs: object) -> SimpleNamespace:
+ calls.append("kompress")
+ return SimpleNamespace(compressed='{"items":[{"id":1}]}', compressed_tokens=1)
+
+ monkeypatch.setattr(router, "_get_kompress", lambda: RecordEatingKompress())
+
+ result = router.compress(payload)
+
+ assert calls == []
+ assert result.compressed == payload
+
+
+def test_prefixed_same_line_json_skips_kompress(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Embedded JSON must be guarded even when the block itself is not JSON.
+
+ The mixed splitter gives a JSON span its own section only when a line starts
+ with ``{``/``[``. With ``Tool result:`` on the same line, embedded routing
+ finds the span but SmartCrusher may produce no savings; the full block then
+ reaches the Kompress fallback. That final input still contains JSON and must
+ not reach the prose model (#3673).
+ """
+ router = ContentRouter(ContentRouterConfig())
+ payload = 'Tool result: {"items":[{"id":1},{"id":2},{"id":3}]}'
+ kompress_calls: list[str] = []
+
+ class NoopSmartCrusher:
+ def crush(self, content: str, query: str = "", bias: float = 1.0) -> SimpleNamespace:
+ return SimpleNamespace(compressed=content)
+
+ class RecordEatingKompress:
+ def is_ready(self) -> bool:
+ return True
+
+ def compress(self, content: str, **_kwargs: object) -> SimpleNamespace:
+ kompress_calls.append("kompress")
+ return SimpleNamespace(
+ compressed='Tool result: {"items":[{"id":1}]}',
+ compressed_tokens=1,
+ )
+
+ monkeypatch.setattr(router, "_get_smart_crusher", lambda: NoopSmartCrusher())
+ monkeypatch.setattr(router, "_get_log_compressor", lambda: None)
+ monkeypatch.setattr(router, "_get_kompress", lambda: RecordEatingKompress())
+
+ result = router.compress(payload)
+
+ assert kompress_calls == []
+ assert result.compressed == payload
+
+
+def test_record_stream_of_separate_objects_still_compresses(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Separate top-level objects (search hits, JSONL) are NOT protected.
+
+ Welding two of them yields an unparseable record rather than a silently
+ shorter document, so the #3673 invisible-loss argument does not apply.
+ Protecting them turns forced Kompress into a no-op on real search payloads
+ — ``test_force_kompress_routes_anthropic_tool_result_to_targeted_kompress``
+ in tests/test_transforms/test_content_router.py depends on this line.
+ """
+ router = ContentRouter(ContentRouterConfig())
+ stream = " ".join(
+ json.dumps({"file": f"src/mod_{i}.py", "line": i, "text": "repeated search payload"})
+ for i in range(160)
+ )
+ kompress_inputs: list[str] = []
+
+ class HalvingKompress:
+ def is_ready(self) -> bool:
+ return True
+
+ def compress(self, content: str, **_kwargs: object) -> SimpleNamespace:
+ kompress_inputs.append(content)
+ out = " ".join(content.split()[::2])
+ return SimpleNamespace(compressed=out, compressed_tokens=len(out.split()))
+
+ monkeypatch.setattr(router, "_get_kompress", lambda: HalvingKompress())
+
+ out, _tokens = router._try_ml_compressor(stream, context="")
+
+ assert kompress_inputs == [stream]
+ assert out != stream
+
+
+def test_record_array_is_protected_even_beside_a_lone_object(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The protected shape is the record array, at any offset in the block."""
+ router = ContentRouter(ContentRouterConfig())
+ document = json.dumps({"domains": [{"name": "a"}, {"name": "b"}, {"name": "c"}]})
+ block = (
+ "The listing call returned two payloads on this line: "
+ + json.dumps({"file": "src/mod.py", "line": 1})
+ + " and then "
+ + document
+ + " done."
+ )
+ kompress_inputs: list[str] = []
+
+ class RecordEatingKompress:
+ def is_ready(self) -> bool:
+ return True
+
+ def compress(self, content: str, **_kwargs: object) -> SimpleNamespace:
+ kompress_inputs.append(content)
+ return SimpleNamespace(compressed="eaten", compressed_tokens=1)
+
+ monkeypatch.setattr(router, "_get_kompress", lambda: RecordEatingKompress())
+
+ out, _tokens = router._try_ml_compressor(block, context="")
+
+ assert out == block
+ assert kompress_inputs == []
+
+
+def test_protection_scan_stays_linear_on_brace_heavy_text(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Source and logs are full of ``{`` that never close. The protection guard
+ scans at every Kompress boundary; it must not re-read the rest of the block
+ for each brace, or large tool output spends its compression time in scans."""
+ import headroom.transforms.recursive_json as rj
+
+ router = ContentRouter(ContentRouterConfig())
+ line = " if (flags & MASK) { log.debug(state); retry(ctx, {timeout: 30, max attempts\n"
+ block = line * 800 # ~65 KB, two unmatched `{` per line, no JSON at all
+ walked = 0
+ real = rj._scan_from
+
+ def counting(text, start, known): # noqa: ANN001, ANN202
+ nonlocal walked
+ end, n = real(text, start, known)
+ walked += n
+ return end, n
+
+ monkeypatch.setattr(rj, "_scan_from", counting)
+ kompress_inputs: list[str] = []
+
+ class HalvingKompress:
+ def is_ready(self) -> bool:
+ return True
+
+ def compress(self, content: str, **_kwargs: object) -> SimpleNamespace:
+ kompress_inputs.append(content)
+ out = " ".join(content.split()[::2])
+ return SimpleNamespace(compressed=out, compressed_tokens=len(out.split()))
+
+ monkeypatch.setattr(router, "_get_kompress", lambda: HalvingKompress())
+
+ out, _tokens = router._try_ml_compressor(block, context="")
+
+ # No JSON to protect, so prose compression still runs...
+ assert kompress_inputs
+ assert out != block
+ # ...and every scan of the call together stays a small multiple of one pass
+ # (the quadratic walk read hundreds of times the block here).
+ assert walked <= 8 * len(block), (walked, len(block))
+
+
+def test_unfinished_protection_scan_protects_the_whole_block(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """If the scan ever stops at its budget, a record array past that point
+ cannot be ruled out, so the block is kept away from the prose model."""
+ import headroom.transforms.recursive_json as rj
+
+ monkeypatch.setattr(rj, "_SCAN_BUDGET_PER_CHAR", 0)
+ monkeypatch.setattr(rj, "_SCAN_BUDGET_FLOOR", 0)
+ router = ContentRouter(ContentRouterConfig())
+ block = "Results below. " + json.dumps([{"id": i, "v": "x" * 20} for i in range(5)])
+ kompress_inputs: list[str] = []
+
+ class RecordEatingKompress:
+ def is_ready(self) -> bool:
+ return True
+
+ def compress(self, content: str, **_kwargs: object) -> SimpleNamespace:
+ kompress_inputs.append(content)
+ return SimpleNamespace(compressed="eaten", compressed_tokens=1)
+
+ monkeypatch.setattr(router, "_get_kompress", lambda: RecordEatingKompress())
+
+ out, _tokens = router._try_ml_compressor(block, context="")
+
+ assert out == block
+ assert kompress_inputs == []
+
+
+def test_relevance_split_still_runs_for_line_contained_json(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """JSONL keeps each object inside one line, so no window cuts a document and
+ the split — which exists for log payloads — proceeds as before."""
+ router = ContentRouter(ContentRouterConfig(relevance_split=True))
+ payload = "\n".join(
+ json.dumps({"level": "info", "msg": f"event {i} with a prose-like description"})
+ for i in range(40)
+ )
+ planned: list[str] = []
+
+ router._relevance_scorer = object()
+ router._relevance_scorer_tried = True
+ monkeypatch.setattr(
+ content_router_module,
+ "plan_relevance_split",
+ lambda content, query, scorer, **_kw: planned.append(content) or [(True, content)],
+ )
+
+ router._relevance_split_compress(payload, "log", "event 3")
+
+ assert planned == [payload]
+
+
+def test_relevance_split_runs_when_a_record_array_sits_inside_one_window(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Only a STRADDLING span blocks the split.
+
+ A log whose paragraphs happen to embed a small record array is windowed at
+ the blank lines, so the array stays whole inside one window. That window is
+ safe to score and drop: if it is dropped, the Kompress boundary sees the
+ intact array and passes it through. Declining here instead would disable
+ the relevance split for most log payloads.
+ """
+ router = ContentRouter(ContentRouterConfig(relevance_split=True))
+ array = json.dumps([{"id": 1, "state": "ok"}, {"id": 2, "state": "ok"}])
+ paragraphs = [
+ "worker started and connected to the queue, waiting for the first batch",
+ "batch 1 finished; per-item results were reported as " + array,
+ "worker idle for 30 seconds, then shut down cleanly on SIGTERM",
+ ]
+ payload = (chr(10) + chr(10)).join(paragraphs)
+ planned: list[str] = []
+ kompress_inputs: list[str] = []
+
+ class HalvingKompress:
+ def is_ready(self) -> bool:
+ return True
+
+ def compress(self, content: str, **_kwargs: object) -> SimpleNamespace:
+ kompress_inputs.append(content)
+ out = " ".join(content.split()[::2])
+ return SimpleNamespace(compressed=out, compressed_tokens=len(out.split()))
+
+ router._relevance_scorer = object()
+ router._relevance_scorer_tried = True
+ monkeypatch.setattr(router, "_get_kompress", lambda: HalvingKompress())
+ monkeypatch.setattr(
+ content_router_module,
+ "plan_relevance_split",
+ lambda content, query, scorer, **_kw: (
+ planned.append(content)
+ or [(False, piece) for piece in content_router_module.segment(content)]
+ ),
+ )
+
+ router._relevance_split_compress(payload, "log", "worker shutdown")
+
+ # The split ran...
+ assert planned == [payload]
+ # ...and the window carrying the array reached the Kompress boundary, which
+ # passed it through untouched rather than letting the prose model see it.
+ carrying = [text for text in kompress_inputs if array in text]
+ assert carrying == [], f"a record array reached the prose model: {carrying}"
+
+
+def test_relevance_split_does_not_fragment_json_payloads(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Relevance windowing must not cut JSON before the Kompress guard.
+
+ A detector can misclassify JSON as LOG/SEARCH. Windowing such a payload at
+ blank-line-free boundaries produces fragments that no longer parse, so a
+ whole-block JSON check cannot help. Skip the split and let the structural
+ JSON boundary preserve the document (#3673).
+ """
+ router = ContentRouter(ContentRouterConfig(relevance_split=True))
+ payload = json.dumps(
+ {
+ "items": [
+ {"id": index, "value": f"record {index} with prose-like description"}
+ for index in range(8)
+ ]
+ },
+ indent=2,
+ )
+ kompress_inputs: list[str] = []
+
+ class DeterministicScorer:
+ def score_batch(self, items: list[str], _context: str) -> list[SimpleNamespace]:
+ return [SimpleNamespace(score=0.0) for _ in items]
+
+ def record_kompress(
+ content: str,
+ _context: str,
+ question: str | None = None,
+ ) -> tuple[str, int]:
+ kompress_inputs.append(content)
+ return content, _estimate_tokens(content)
+
+ router._relevance_scorer = DeterministicScorer()
+ router._relevance_scorer_tried = True
+ monkeypatch.setattr(router, "_try_ml_compressor", record_kompress)
+
+ assert router._relevance_split_compress(payload, "log", "find record 3") is None
+ assert kompress_inputs == []
+
+
def test_smart_crusher_log_fallback_runs_for_valid_json(
monkeypatch: pytest.MonkeyPatch,
) -> None: