Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(content-detector): detect and compress space-separated JSON objects (#1742)

Rohan Richard5194bdc5a6e5
diff --git a/headroom/transforms/content_detector.py b/headroom/transforms/content_detector.py
index 78ab52fb5..35bb77a20 100644
--- a/headroom/transforms/content_detector.py
+++ b/headroom/transforms/content_detector.py
@@ -181,16 +181,57 @@ def detect_content_type(content: str) -> DetectionResult:
return DetectionResult(ContentType.PLAIN_TEXT, 0.5, {})
+def _decode_concatenated_json(content: str) -> list | None:
+ """Decode a run of whitespace-separated top-level JSON values.
+
+ Web search tools (SerpAPI, Tavily, custom backends) commonly emit
+ back-to-back JSON objects separated only by whitespace rather than a real
+ array: ``{"title": ...} {"title": ...} {"title": ...}``. Returns the list
+ of decoded values, or None if the text isn't a clean run of JSON values
+ separated only by whitespace.
+ """
+ decoder = json.JSONDecoder()
+ idx, length = 0, len(content)
+ items: list = []
+ while idx < length:
+ while idx < length and content[idx].isspace():
+ idx += 1
+ if idx >= length:
+ break
+ try:
+ value, idx = decoder.raw_decode(content, idx)
+ except ValueError:
+ return None
+ items.append(value)
+ return items or None
+
+
+def normalize_concatenated_json(content: str) -> str | None:
+ """Convert whitespace-separated JSON objects into a canonical JSON array.
+
+ SmartCrusher only compresses JSON arrays, so this rewrites the
+ space-separated web_search shape (``{...} {...} {...}``) into
+ ``[{...}, {...}, {...}]``. Returns None unless the content is two or more
+ whitespace-separated JSON objects.
+ """
+ stripped = content.strip()
+ if not stripped.startswith("{"):
+ return None
+ items = _decode_concatenated_json(stripped)
+ if items and len(items) >= 2 and all(isinstance(item, dict) for item in items):
+ return json.dumps(items)
+ return None
+
+
def _try_detect_json(content: str) -> DetectionResult | None:
"""Try to detect JSON array content."""
content = content.strip()
- # Quick check: must start with [ for array
- if not content.startswith("["):
- return None
-
- try:
- parsed = json.loads(content)
+ if content.startswith("["):
+ try:
+ parsed = json.loads(content)
+ except json.JSONDecodeError:
+ return None
if isinstance(parsed, list):
# Check if it's a list of dicts (SmartCrusher compatible)
if parsed and all(isinstance(item, dict) for item in parsed):
@@ -205,8 +246,20 @@ def _try_detect_json(content: str) -> DetectionResult | None:
0.8,
{"item_count": len(parsed), "is_dict_array": False},
)
- except json.JSONDecodeError:
- pass
+ return None
+
+ # Space-separated JSON objects (typical web_search output) aren't a valid
+ # array, so they'd fall through to PLAIN_TEXT and skip SmartCrusher at 0%
+ # compression. SmartCrusher normalizes this shape to a real array before
+ # crushing (#1741).
+ if content.startswith("{"):
+ items = _decode_concatenated_json(content)
+ if items and len(items) >= 2 and all(isinstance(item, dict) for item in items):
+ return DetectionResult(
+ ContentType.JSON_ARRAY,
+ 1.0,
+ {"item_count": len(items), "is_dict_array": True, "concatenated": True},
+ )
return None
diff --git a/headroom/transforms/smart_crusher.py b/headroom/transforms/smart_crusher.py
index bf8c1ed8f..249e37d34 100644
--- a/headroom/transforms/smart_crusher.py
+++ b/headroom/transforms/smart_crusher.py
@@ -54,6 +54,7 @@ from ..config import CCRConfig, TransformResult
from ..tokenizer import Tokenizer
from ..utils import compute_short_hash, create_tool_digest_marker, deep_copy_messages
from .base import Transform
+from .content_detector import normalize_concatenated_json
logger = logging.getLogger(__name__)
@@ -446,6 +447,13 @@ class SmartCrusher(Transform):
opaque-blob offload) leaves the content uncompacted instead.
`None` (default) uses the instance's configured value.
"""
+ # Web search tools often return space-separated JSON objects
+ # (``{...} {...} {...}``) rather than a real array. The Rust crusher
+ # only compresses JSON arrays, so normalize that shape first —
+ # otherwise it passes through at 0% compression (#1741).
+ normalized = normalize_concatenated_json(content)
+ if normalized is not None:
+ content = normalized
rust = (
self._rust
if lossless_only is None or bool(lossless_only) == self._lossless_only
diff --git a/tests/test_transforms_content_detection.py b/tests/test_transforms_content_detection.py
index d08a8d127..d209ecabc 100644
--- a/tests/test_transforms_content_detection.py
+++ b/tests/test_transforms_content_detection.py
@@ -1,5 +1,7 @@
from __future__ import annotations
+import json
+
from headroom.transforms.content_detector import (
ContentType,
_try_detect_code,
@@ -10,6 +12,7 @@ from headroom.transforms.content_detector import (
_try_detect_search,
detect_content_type,
is_json_array_of_dicts,
+ normalize_concatenated_json,
)
from headroom.transforms.error_detection import (
ERROR_INDICATOR_KEYWORDS,
@@ -59,6 +62,46 @@ def test_json_detection_distinguishes_dict_arrays_and_other_lists() -> None:
assert is_json_array_of_dicts('["value"]') is False
+def test_space_separated_json_objects_detected_as_array() -> None:
+ # Typical web_search output: back-to-back JSON objects, no array brackets.
+ content = " ".join(
+ json.dumps({"title": f"Result {i}", "url": f"http://example.com/{i}"}) for i in range(3)
+ )
+ result = _try_detect_json(content)
+ assert result is not None
+ assert result.content_type is ContentType.JSON_ARRAY
+ assert result.confidence == 1.0
+ assert result.metadata == {"item_count": 3, "is_dict_array": True, "concatenated": True}
+
+ # Reaches the same verdict through the top-level detector (not PLAIN_TEXT).
+ assert detect_content_type(content).content_type is ContentType.JSON_ARRAY
+ assert is_json_array_of_dicts(content) is True
+
+ # Newline separation is just as common and must also be recognized.
+ newline_sep = "\n".join(json.dumps({"id": i, "snippet": "x"}) for i in range(2))
+ assert _try_detect_json(newline_sep).content_type is ContentType.JSON_ARRAY
+
+
+def test_space_separated_json_detection_is_conservative() -> None:
+ # A single object is not an array — must not be claimed.
+ assert _try_detect_json('{"id": 1}') is None
+ # Objects interleaved with prose are not clean concatenated JSON.
+ assert _try_detect_json('{"id": 1} then some prose {"id": 2}') is None
+ # Scalars/strings between objects disqualify the run of dicts.
+ assert _try_detect_json('{"id": 1} "loose string"') is None
+
+
+def test_normalize_concatenated_json_roundtrips_to_array() -> None:
+ content = '{"a": 1} {"b": 2}'
+ normalized = normalize_concatenated_json(content)
+ assert normalized is not None
+ assert json.loads(normalized) == [{"a": 1}, {"b": 2}]
+
+ # Already-valid arrays and single objects are left for the caller as-is.
+ assert normalize_concatenated_json('[{"a": 1}]') is None
+ assert normalize_concatenated_json('{"a": 1}') is None
+
+
def test_diff_detection_tracks_headers_and_changes() -> None:
diff = "\n".join(
[