Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(transforms): keep record-bearing JSON out of Kompress (#3673) (#3880)
afernandez-stratio8dbbd1d8967f
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.pyindex d2452c235..682f0deda 100644--- a/headroom/transforms/content_router.py+++ b/headroom/transforms/content_router.py@@ -91,7 +91,7 @@ from .lossless_provider import (get_lossless_verifier,)from .mixed_content import ContentSection, mixed_content_indicators-from .relevance_split import build_relevance_query, plan_relevance_split+from .relevance_split import build_relevance_query, plan_relevance_split, segmentlogger = logging.getLogger(__name__)@@ -895,6 +895,73 @@ def _content_is_valid_json(content: str) -> bool:return True+def _protected_json_spans(content: str) -> list[tuple[int, int]]:+ """JSON spans of ``content`` that a prose model must not be handed (#3673).++ Kompress has no notion of JSON grammar, so JSON bytes in its input are at+ risk of a destructive span around ``},{``. The scan is the deterministic+ balanced-span walk embedded-JSON routing already uses, so prefixed tool+ output and relevance fragments are seen too. Two shapes qualify:++ * a span carrying an array of objects — deleting the ``},{`` between two+ records leaves VALID JSON, so the loss is invisible to every parser, log+ line and model downstream, which is what made #3673 invisible;+ * a block that is one JSON document end to end, which covers welding two+ keys of a lone object by the same argument.++ A stream of SEPARATE top-level values (search hits as space-joined objects,+ JSONL) is deliberately not protected. Welding two of those yields an+ unparseable record rather than a silently shorter document, so the+ invisible-loss argument does not apply — and protecting them costs a large,+ legitimate savings class: it turns forced Kompress into a no-op on a+ 160-object search payload (see+ ``test_force_kompress_routes_anthropic_tool_result_to_targeted_kompress``).+ """+ from .recursive_json import carries_record_array, scan_json_documents++ spans, complete = scan_json_documents(content)+ if not complete:+ # The scan stopped at its budget (input built to defeat the linear walk),+ # so a record array past that point cannot be ruled out. Protect the+ # whole block: declining a prose compression is recoverable, a silently+ # deleted record is not.+ return [(0, len(content))]+ if not spans:+ return []+ if (+ len(spans) == 1+ and not content[: spans[0][0]].strip()+ and not content[spans[0][1] :].strip()+ ):+ return spans+ return [(a, b) for a, b in spans if carries_record_array(content[a:b])]+++def _contains_protected_json(content: str) -> bool:+ """True when ``content`` is, or contains, a JSON span worth protecting."""+ return bool(_protected_json_spans(content))+++def _json_straddles_segments(content: str) -> bool:+ """True when a protected JSON span crosses a boundary ``segment()`` cuts at.++ The relevance split windows by line with no JSON awareness, so a payload+ misclassified as LOG/SEARCH can be cut mid-structure and the pieces handed+ to Kompress as fragments no span check can recognise. Declining only for a+ straddling span keeps the split working on line-contained JSON (JSONL+ logs), which is the shape it exists for.+ """+ spans = _protected_json_spans(content)+ if not spans:+ return False+ cuts: list[int] = []+ offset = 0+ for piece in segment(content)[:-1]:+ offset += len(piece)+ cuts.append(offset)+ return any(a < cut < b for a, b in spans for cut in cuts)++def _mixed_indicators(content: str) -> dict[str, bool]:return mixed_content_indicators(content)@@ -4290,6 +4357,14 @@ class ContentRouter(Transform):Returns:Tuple of (compressed, token_count)."""+ # Kompress is a prose model: dropping a span around ``},{`` can remove+ # an entire JSON record while leaving the remaining document valid+ # (#3673). This boundary sees the final Kompress input, which may be a+ # prefixed block or a fragment rather than a whole JSON document, so+ # guard every parseable JSON span before the prose model runs.+ if _contains_protected_json(content):+ return content, _estimate_tokens(content)+from .tag_protector import protect_tags, restore_tags# Protect custom tags before any ML compression@@ -4632,6 +4707,11 @@ class ContentRouter(Transform):normal path when the scorer is unavailable, the query is empty, nothingis dropped, or the split doesn't beat plain compaction. Never raises.+ JSON the windows would cut is excluded from splitting. Without that+ route guard, a payload misclassified as LOG/SEARCH is cut at arbitrary+ offsets before the JSON guard sees it (#3673); line-contained JSON+ (JSONL) still splits, which is the shape this exists for.+Embedding cost is bounded two ways: the model is pre-warmed off therequest thread (BM25 until it's ready, see _get_relevance_scorer) andoutputs segmenting into more than ``relevance_max_records`` records skip@@ -4640,6 +4720,8 @@ class ContentRouter(Transform):scorer = self._get_relevance_scorer()if scorer is None or not query.strip():return None+ if _json_straddles_segments(content):+ return Nonefrom .lossless_compaction import compact_losslesstry:diff --git a/headroom/transforms/recursive_json.py b/headroom/transforms/recursive_json.pyindex e6d7b1665..eb29b9e8c 100644--- a/headroom/transforms/recursive_json.py+++ b/headroom/transforms/recursive_json.py@@ -70,26 +70,124 @@ def _match_span(text: str, start: int) -> int | None:return None-def _spans(text: str) -> list[tuple[int, int]]:- """Deterministic list of ``(start, end)`` for top-level balanced JSON spans.- Nested spans are not returned separately — the dispatch handles depth."""+#: Backstop for ``_scan_spans``: characters it may walk, per character of input,+#: before it stops looking for further spans. The memoized walk below is linear+#: on real payloads (source, logs, JSON, prose), so this only bounds inputs built+#: to defeat the memo. Callers that must not miss a span check ``complete``.+_SCAN_BUDGET_PER_CHAR = 4+_SCAN_BUDGET_FLOOR = 4096+++def _scan_from(text: str, start: int, known: dict[int, int | None]) -> tuple[int | None, int]:+ """``_match_span(text, start)``, recording every nested container's verdict.++ A walk from ``start`` also decides the result of ``_match_span`` for every+ ``[``/``{`` it pushes outside a string: from that position the walk's string+ state is the same (not in a string), its stack is the suffix above that+ bracket, so it pops, hits a mismatch or runs off the end exactly when this+ walk does. Those verdicts go into ``known`` so the caller never walks the same+ bytes again for them — which is what made unmatched ``{`` in source and logs+ quadratic. Returns ``(end or None, characters walked)``.+ """+ stack: list[int] = []+ in_str = esc = False+ n = len(text)+ for j in range(start, n):+ ch = text[j]+ if in_str:+ if esc:+ esc = False+ elif ch == "\\":+ esc = True+ elif ch == '"':+ in_str = False+ continue+ if ch == '"':+ in_str = True+ elif ch in _OPEN:+ stack.append(j)+ elif ch in _CLOSE:+ if not stack or text[stack[-1]] != _PAIR[ch]:+ for p in stack:+ known[p] = None+ return None, j - start + 1+ known[stack.pop()] = j + 1+ if not stack:+ return j + 1, j - start + 1+ for p in stack:+ known[p] = None+ return None, n - start+++def _scan_spans(text: str) -> tuple[list[tuple[int, int]], bool]:+ """``(spans, complete)``: the top-level balanced spans ``_spans`` returns, and+ whether the walk covered the whole text within its budget.++ Same left-to-right semantics as calling ``_match_span`` at every opening+ bracket, but each bracket's verdict is computed at most once (see+ ``_scan_from``), so a block full of unmatched ``{`` costs one pass instead of+ one pass per brace.+ """out: list[tuple[int, int]] = []- i, n = 0, len(text)+ known: dict[int, int | None] = {}+ n = len(text)+ budget = _SCAN_BUDGET_PER_CHAR * n + _SCAN_BUDGET_FLOOR+ spent = 0+ i = 0while i < n:if text[i] in _OPEN:- end = _match_span(text, i)+ if i in known:+ end = known[i]+ elif spent >= budget:+ return out, False+ else:+ end, walked = _scan_from(text, i, known)+ spent += walkedif end is not None:out.append((i, end))i = endcontinuei += 1- return out+ return out, True+++def _spans(text: str) -> list[tuple[int, int]]:+ """Deterministic list of ``(start, end)`` for top-level balanced JSON spans.+ Nested spans are not returned separately — the dispatch handles depth."""+ return _scan_spans(text)[0]+++def json_document_spans(text: str) -> list[tuple[int, int]]:+ """``(start, end)`` of every top-level balanced span of ``text`` that parses+ as a JSON object or array, left to right. Scalars are not documents; a+ whole-document ``text`` yields one span covering it (modulo whitespace)."""+ return scan_json_documents(text)[0]+++def scan_json_documents(text: str) -> tuple[list[tuple[int, int]], bool]:+ """``json_document_spans`` plus whether the scan covered all of ``text``.++ ``complete`` is only ever False past the scan budget, i.e. on input built to+ defeat the linear walk; a caller that must not miss a document treats that+ as "may contain one".+ """+ spans, complete = _scan_spans(text)+ out: list[tuple[int, int]] = []+ for a, b in spans:+ try:+ parsed = json.loads(text[a:b])+ except (ValueError, TypeError, RecursionError):+ continue+ if isinstance(parsed, dict | list):+ out.append((a, b))+ return out, complete-def _has_routable_json(span: str) -> bool:+def carries_record_array(span: str) -> bool:"""True if ``span`` parses and contains an array of objects somewhere — the- shape the JSON compressors actually act on. Cheap structural check, no size- threshold."""+ shape the JSON compressors act on, and the one whose record delimiters a+ prose model can delete while leaving valid JSON behind (#3673). Cheap+ structural check, no size threshold."""try:v = json.loads(span)except (ValueError, TypeError):@@ -141,7 +239,7 @@ def route_embedded_json(chunk = content[a:b]if "<<ccr:" in chunk: # R1: already compressed — never re-routecontinue- if not _has_routable_json(chunk):+ if not carries_record_array(chunk):continueout = dispatch(chunk)if out is None or out == chunk:diff --git a/tests/test_recursive_json.py b/tests/test_recursive_json.pyindex 32e8709e6..53a8ff8cb 100644--- a/tests/test_recursive_json.py+++ b/tests/test_recursive_json.py@@ -6,7 +6,11 @@ from __future__ import annotationsimport json-from headroom.transforms.recursive_json import route_embedded_json+from headroom.transforms.recursive_json import (+ carries_record_array,+ json_document_spans,+ route_embedded_json,+)def _upper_dispatch(span: str) -> str | None:@@ -65,3 +69,121 @@ def test_scalar_array_not_routed() -> None:# array of scalars is not a "routable" JSON shape (no dict rows)content = "nums: [1,2,3,4,5,6,7,8] done"assert route_embedded_json(content, _upper_dispatch) is None+++def test_json_document_spans_finds_containers_anywhere() -> None:+ doc = json.dumps({"domains": [{"name": "a"}, {"name": "b"}]})+ arr = json.dumps([1, 2, 3])+ text = "Tool result: " + doc + " and a list " + arr + " done"+ assert [text[a:b] for a, b in json_document_spans(text)] == [doc, arr]+++def test_json_document_spans_ignores_scalars_prose_and_unbalanced_json() -> None:+ assert json_document_spans('"just a quoted sentence"') == []+ assert json_document_spans("42") == []+ assert json_document_spans("prose with a [note] and {braces} but no JSON") == []+ assert json_document_spans('{"truncated": [1, 2, 3') == []+ assert json_document_spans("{{HEADROOM_TAG_0}}") == []+ assert json_document_spans("") == []+++def test_json_document_spans_whole_document() -> None:+ doc = json.dumps({"a": [1, 2]})+ assert json_document_spans(doc) == [(0, len(doc))]+ assert json_document_spans(" " + doc + "\n") == [(2, 2 + len(doc))]+++def test_carries_record_array_separates_the_two_shapes() -> None:+ assert carries_record_array(json.dumps({"domains": [{"name": "a"}, {"name": "b"}]}))+ assert carries_record_array(json.dumps([{"id": 1}, {"id": 2}]))+ # A lone object, or an array of scalars, has no record delimiter whose+ # deletion leaves a valid-but-shorter document.+ assert not carries_record_array(json.dumps({"file": "src/mod.py", "line": 1}))+ assert not carries_record_array(json.dumps([1, 2, 3]))+ assert not carries_record_array(json.dumps([{"id": 1}]))+++# --- the span walk is linear on brace-heavy text -----------------------------+#+# ``_spans`` used to call ``_match_span`` at every opening bracket, and a ``{``+# that never closes walks to the end of the text: source code and logs full of+# unmatched braces cost one full pass per brace. ``_scan_spans`` memoizes each+# bracket's verdict from the walk that already crossed it. These tests pin that+# the result did not change and that the work did.+++def _reference_spans(text: str) -> list[tuple[int, int]]:+ """The previous ``_spans``: ``_match_span`` at every opening bracket."""+ from headroom.transforms.recursive_json import _match_span++ out: list[tuple[int, int]] = []+ i, n = 0, len(text)+ while i < n:+ if text[i] in "[{":+ end = _match_span(text, i)+ if end is not None:+ out.append((i, end))+ i = end+ continue+ i += 1+ return out+++def test_span_walk_matches_the_previous_semantics_exactly() -> None:+ import random++ from headroom.transforms.recursive_json import _scan_spans++ rng = random.Random(3673)+ alphabet = '{}[]"\\ a,:\n'+ samples = [+ "".join(rng.choice(alphabet) for _ in range(rng.randint(0, 120))) for _ in range(4000)+ ]+ samples += [+ 'noise {"a": [1, {"b": "}"}]} tail [1,2] {oops',+ 'He said "a {quote" then {"k": "v"} and "another {one}"',+ '{"esc": "a \\" } still string"} {x',+ "[{]}] {[}] {{{{",+ ]+ for text in samples:+ spans, complete = _scan_spans(text)+ assert complete, text+ assert spans == _reference_spans(text), text+++def test_span_walk_is_linear_on_unmatched_braces(monkeypatch) -> None:+ import headroom.transforms.recursive_json as rj++ line = ' if (flags & MASK) { log.debug("state={}", state); retry(ctx, {timeout: 30, max\n'+ text = line * 800 # ~70 KB, about three unmatched `{` per line+ walked = 0+ real = rj._scan_from++ def counting(t, start, known): # noqa: ANN001, ANN202+ nonlocal walked+ end, n = real(t, start, known)+ walked += n+ return end, n++ monkeypatch.setattr(rj, "_scan_from", counting)+ spans, complete = rj._scan_spans(text)+ assert complete+ # One pass, give or take the per-line restarts inside quoted strings. The+ # previous walk re-read the rest of the text for every unmatched brace:+ # hundreds of times the input size here.+ assert walked <= 2 * len(text), (walked, len(text))+ # Same answer as before on the same shape (a slice keeps the quadratic+ # reference fast enough to run).+ small = line * 40+ assert rj._scan_spans(small)[0] == _reference_spans(small)+++def test_scan_stops_at_its_budget_and_says_so(monkeypatch) -> None:+ import headroom.transforms.recursive_json as rj++ monkeypatch.setattr(rj, "_SCAN_BUDGET_PER_CHAR", 0)+ monkeypatch.setattr(rj, "_SCAN_BUDGET_FLOOR", 0)+ text = 'prose {"a": 1} then [{"id": 1}, {"id": 2}]'+ spans, complete = rj.scan_json_documents(text)+ assert complete is False+ assert spans == []diff --git a/tests/test_transforms_content_router.py b/tests/test_transforms_content_router.pyindex 946710f6b..f7e2aee9e 100644--- a/tests/test_transforms_content_router.py+++ b/tests/test_transforms_content_router.py@@ -705,6 +705,378 @@ def test_smart_crusher_log_fallback_skipped_for_invalid_json(assert CompressionStrategy.KOMPRESS.value in strategy_chain+def test_smart_crusher_fallback_skips_kompress_for_valid_json_object(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Kompress must not delete records from parseable single-line JSON (#3673).++ SmartCrusher passes a compact JSON object through when it has no applicable+ array transform. The no-savings fallback used to hand the whole document to+ Kompress; when Kompress dropped a span around ``},{``, the result remained+ valid JSON but silently lost a record.+ """+ router = ContentRouter(ContentRouterConfig())+ payload = json.dumps(+ {+ "domains": [+ {"name": "first", "description": "First collection description"},+ {"name": "second", "description": "Second collection description"},+ {"name": "third", "description": "Third collection description"},+ ]+ },+ separators=(",", ":"),+ )+ kompress_calls: list[str] = []++ class NoopSmartCrusher:+ def crush(self, content: str, query: str = "", bias: float = 1.0) -> SimpleNamespace:+ return SimpleNamespace(compressed=content)++ class RecordEatingKompress:+ def is_ready(self) -> bool:+ return True++ def compress(self, content: str, **_kwargs: object) -> SimpleNamespace:+ kompress_calls.append("kompress")+ return SimpleNamespace(+ compressed='{"domains":["record deleted"]}',+ compressed_tokens=1,+ )++ monkeypatch.setattr(router, "_get_smart_crusher", lambda: NoopSmartCrusher())+ monkeypatch.setattr(router, "_get_log_compressor", lambda: None)+ monkeypatch.setattr(router, "_get_kompress", lambda: RecordEatingKompress())++ result = router.compress(payload)++ assert result.compressed == payload+ assert kompress_calls == []+++def test_force_kompress_skips_valid_json_object(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The forced-Kompress fast path keeps the same parseable-JSON guard."""+ router = ContentRouter(ContentRouterConfig(force_kompress_all=True))+ router._runtime_force_kompress = True+ payload = json.dumps({"items": [{"id": 1}, {"id": 2}]}, separators=(",", ":"))+ calls: list[str] = []++ class RecordEatingKompress:+ def is_ready(self) -> bool:+ return True++ def compress(self, content: str, **_kwargs: object) -> SimpleNamespace:+ calls.append("kompress")+ return SimpleNamespace(compressed='{"items":[{"id":1}]}', compressed_tokens=1)++ monkeypatch.setattr(router, "_get_kompress", lambda: RecordEatingKompress())++ result = router.compress(payload)++ assert calls == []+ assert result.compressed == payload+++def test_prefixed_same_line_json_skips_kompress(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Embedded JSON must be guarded even when the block itself is not JSON.++ The mixed splitter gives a JSON span its own section only when a line starts+ with ``{``/``[``. With ``Tool result:`` on the same line, embedded routing+ finds the span but SmartCrusher may produce no savings; the full block then+ reaches the Kompress fallback. That final input still contains JSON and must+ not reach the prose model (#3673).+ """+ router = ContentRouter(ContentRouterConfig())+ payload = 'Tool result: {"items":[{"id":1},{"id":2},{"id":3}]}'+ kompress_calls: list[str] = []++ class NoopSmartCrusher:+ def crush(self, content: str, query: str = "", bias: float = 1.0) -> SimpleNamespace:+ return SimpleNamespace(compressed=content)++ class RecordEatingKompress:+ def is_ready(self) -> bool:+ return True++ def compress(self, content: str, **_kwargs: object) -> SimpleNamespace:+ kompress_calls.append("kompress")+ return SimpleNamespace(+ compressed='Tool result: {"items":[{"id":1}]}',+ compressed_tokens=1,+ )++ monkeypatch.setattr(router, "_get_smart_crusher", lambda: NoopSmartCrusher())+ monkeypatch.setattr(router, "_get_log_compressor", lambda: None)+ monkeypatch.setattr(router, "_get_kompress", lambda: RecordEatingKompress())++ result = router.compress(payload)++ assert kompress_calls == []+ assert result.compressed == payload+++def test_record_stream_of_separate_objects_still_compresses(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Separate top-level objects (search hits, JSONL) are NOT protected.++ Welding two of them yields an unparseable record rather than a silently+ shorter document, so the #3673 invisible-loss argument does not apply.+ Protecting them turns forced Kompress into a no-op on real search payloads+ — ``test_force_kompress_routes_anthropic_tool_result_to_targeted_kompress``+ in tests/test_transforms/test_content_router.py depends on this line.+ """+ router = ContentRouter(ContentRouterConfig())+ stream = " ".join(+ json.dumps({"file": f"src/mod_{i}.py", "line": i, "text": "repeated search payload"})+ for i in range(160)+ )+ kompress_inputs: list[str] = []++ class HalvingKompress:+ def is_ready(self) -> bool:+ return True++ def compress(self, content: str, **_kwargs: object) -> SimpleNamespace:+ kompress_inputs.append(content)+ out = " ".join(content.split()[::2])+ return SimpleNamespace(compressed=out, compressed_tokens=len(out.split()))++ monkeypatch.setattr(router, "_get_kompress", lambda: HalvingKompress())++ out, _tokens = router._try_ml_compressor(stream, context="")++ assert kompress_inputs == [stream]+ assert out != stream+++def test_record_array_is_protected_even_beside_a_lone_object(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The protected shape is the record array, at any offset in the block."""+ router = ContentRouter(ContentRouterConfig())+ document = json.dumps({"domains": [{"name": "a"}, {"name": "b"}, {"name": "c"}]})+ block = (+ "The listing call returned two payloads on this line: "+ + json.dumps({"file": "src/mod.py", "line": 1})+ + " and then "+ + document+ + " done."+ )+ kompress_inputs: list[str] = []++ class RecordEatingKompress:+ def is_ready(self) -> bool:+ return True++ def compress(self, content: str, **_kwargs: object) -> SimpleNamespace:+ kompress_inputs.append(content)+ return SimpleNamespace(compressed="eaten", compressed_tokens=1)++ monkeypatch.setattr(router, "_get_kompress", lambda: RecordEatingKompress())++ out, _tokens = router._try_ml_compressor(block, context="")++ assert out == block+ assert kompress_inputs == []+++def test_protection_scan_stays_linear_on_brace_heavy_text(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Source and logs are full of ``{`` that never close. The protection guard+ scans at every Kompress boundary; it must not re-read the rest of the block+ for each brace, or large tool output spends its compression time in scans."""+ import headroom.transforms.recursive_json as rj++ router = ContentRouter(ContentRouterConfig())+ line = " if (flags & MASK) { log.debug(state); retry(ctx, {timeout: 30, max attempts\n"+ block = line * 800 # ~65 KB, two unmatched `{` per line, no JSON at all+ walked = 0+ real = rj._scan_from++ def counting(text, start, known): # noqa: ANN001, ANN202+ nonlocal walked+ end, n = real(text, start, known)+ walked += n+ return end, n++ monkeypatch.setattr(rj, "_scan_from", counting)+ kompress_inputs: list[str] = []++ class HalvingKompress:+ def is_ready(self) -> bool:+ return True++ def compress(self, content: str, **_kwargs: object) -> SimpleNamespace:+ kompress_inputs.append(content)+ out = " ".join(content.split()[::2])+ return SimpleNamespace(compressed=out, compressed_tokens=len(out.split()))++ monkeypatch.setattr(router, "_get_kompress", lambda: HalvingKompress())++ out, _tokens = router._try_ml_compressor(block, context="")++ # No JSON to protect, so prose compression still runs...+ assert kompress_inputs+ assert out != block+ # ...and every scan of the call together stays a small multiple of one pass+ # (the quadratic walk read hundreds of times the block here).+ assert walked <= 8 * len(block), (walked, len(block))+++def test_unfinished_protection_scan_protects_the_whole_block(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """If the scan ever stops at its budget, a record array past that point+ cannot be ruled out, so the block is kept away from the prose model."""+ import headroom.transforms.recursive_json as rj++ monkeypatch.setattr(rj, "_SCAN_BUDGET_PER_CHAR", 0)+ monkeypatch.setattr(rj, "_SCAN_BUDGET_FLOOR", 0)+ router = ContentRouter(ContentRouterConfig())+ block = "Results below. " + json.dumps([{"id": i, "v": "x" * 20} for i in range(5)])+ kompress_inputs: list[str] = []++ class RecordEatingKompress:+ def is_ready(self) -> bool:+ return True++ def compress(self, content: str, **_kwargs: object) -> SimpleNamespace:+ kompress_inputs.append(content)+ return SimpleNamespace(compressed="eaten", compressed_tokens=1)++ monkeypatch.setattr(router, "_get_kompress", lambda: RecordEatingKompress())++ out, _tokens = router._try_ml_compressor(block, context="")++ assert out == block+ assert kompress_inputs == []+++def test_relevance_split_still_runs_for_line_contained_json(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """JSONL keeps each object inside one line, so no window cuts a document and+ the split — which exists for log payloads — proceeds as before."""+ router = ContentRouter(ContentRouterConfig(relevance_split=True))+ payload = "\n".join(+ json.dumps({"level": "info", "msg": f"event {i} with a prose-like description"})+ for i in range(40)+ )+ planned: list[str] = []++ router._relevance_scorer = object()+ router._relevance_scorer_tried = True+ monkeypatch.setattr(+ content_router_module,+ "plan_relevance_split",+ lambda content, query, scorer, **_kw: planned.append(content) or [(True, content)],+ )++ router._relevance_split_compress(payload, "log", "event 3")++ assert planned == [payload]+++def test_relevance_split_runs_when_a_record_array_sits_inside_one_window(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Only a STRADDLING span blocks the split.++ A log whose paragraphs happen to embed a small record array is windowed at+ the blank lines, so the array stays whole inside one window. That window is+ safe to score and drop: if it is dropped, the Kompress boundary sees the+ intact array and passes it through. Declining here instead would disable+ the relevance split for most log payloads.+ """+ router = ContentRouter(ContentRouterConfig(relevance_split=True))+ array = json.dumps([{"id": 1, "state": "ok"}, {"id": 2, "state": "ok"}])+ paragraphs = [+ "worker started and connected to the queue, waiting for the first batch",+ "batch 1 finished; per-item results were reported as " + array,+ "worker idle for 30 seconds, then shut down cleanly on SIGTERM",+ ]+ payload = (chr(10) + chr(10)).join(paragraphs)+ planned: list[str] = []+ kompress_inputs: list[str] = []++ class HalvingKompress:+ def is_ready(self) -> bool:+ return True++ def compress(self, content: str, **_kwargs: object) -> SimpleNamespace:+ kompress_inputs.append(content)+ out = " ".join(content.split()[::2])+ return SimpleNamespace(compressed=out, compressed_tokens=len(out.split()))++ router._relevance_scorer = object()+ router._relevance_scorer_tried = True+ monkeypatch.setattr(router, "_get_kompress", lambda: HalvingKompress())+ monkeypatch.setattr(+ content_router_module,+ "plan_relevance_split",+ lambda content, query, scorer, **_kw: (+ planned.append(content)+ or [(False, piece) for piece in content_router_module.segment(content)]+ ),+ )++ router._relevance_split_compress(payload, "log", "worker shutdown")++ # The split ran...+ assert planned == [payload]+ # ...and the window carrying the array reached the Kompress boundary, which+ # passed it through untouched rather than letting the prose model see it.+ carrying = [text for text in kompress_inputs if array in text]+ assert carrying == [], f"a record array reached the prose model: {carrying}"+++def test_relevance_split_does_not_fragment_json_payloads(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Relevance windowing must not cut JSON before the Kompress guard.++ A detector can misclassify JSON as LOG/SEARCH. Windowing such a payload at+ blank-line-free boundaries produces fragments that no longer parse, so a+ whole-block JSON check cannot help. Skip the split and let the structural+ JSON boundary preserve the document (#3673).+ """+ router = ContentRouter(ContentRouterConfig(relevance_split=True))+ payload = json.dumps(+ {+ "items": [+ {"id": index, "value": f"record {index} with prose-like description"}+ for index in range(8)+ ]+ },+ indent=2,+ )+ kompress_inputs: list[str] = []++ class DeterministicScorer:+ def score_batch(self, items: list[str], _context: str) -> list[SimpleNamespace]:+ return [SimpleNamespace(score=0.0) for _ in items]++ def record_kompress(+ content: str,+ _context: str,+ question: str | None = None,+ ) -> tuple[str, int]:+ kompress_inputs.append(content)+ return content, _estimate_tokens(content)++ router._relevance_scorer = DeterministicScorer()+ router._relevance_scorer_tried = True+ monkeypatch.setattr(router, "_try_ml_compressor", record_kompress)++ assert router._relevance_split_compress(payload, "log", "find record 3") is None+ assert kompress_inputs == []++def test_smart_crusher_log_fallback_runs_for_valid_json(monkeypatch: pytest.MonkeyPatch,) -> None: