Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix: ignore brackets inside JSON strings when splitting mixed content (#553)
Mubashir Rbdcfc322da0c
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.pyindex 8c02c7753..6dc6f0203 100644--- a/headroom/transforms/content_router.py+++ b/headroom/transforms/content_router.py@@ -658,13 +658,38 @@ def _extract_json_block(lines: list[str], start: int) -> tuple[str | None, int]:bracket_count = 0brace_count = 0json_lines = []+ in_string = False+ escaped = Falsefor i in range(start, len(lines)):line = lines[i]json_lines.append(line)- bracket_count += line.count("[") - line.count("]")- brace_count += line.count("{") - line.count("}")+ # Count brackets/braces, but ignore any that appear inside a JSON+ # string literal — a naive line.count() treats e.g. the "]" in+ # {"path": "a]b"} as a closing bracket and terminates the block+ # early, splitting one array across multiple sections.+ for ch in line:+ if escaped:+ escaped = False+ continue+ if ch == "\\":+ if in_string:+ escaped = True+ continue+ if ch == '"':+ in_string = not in_string+ continue+ if in_string:+ continue+ if ch == "[":+ bracket_count += 1+ elif ch == "]":+ bracket_count -= 1+ elif ch == "{":+ brace_count += 1+ elif ch == "}":+ brace_count -= 1if bracket_count <= 0 and brace_count <= 0 and json_lines:return "\n".join(json_lines), idiff --git a/tests/test_transforms_content_router.py b/tests/test_transforms_content_router.pyindex 8ee570979..01c3732c6 100644--- a/tests/test_transforms_content_router.py+++ b/tests/test_transforms_content_router.py@@ -171,6 +171,66 @@ def test_mixed_content_section_splitting_and_json_extraction() -> None:assert _extract_json_block(["{", '"a": 1'], 0) == (None, 0)+def test_extract_json_block_ignores_brackets_inside_strings() -> None:+ """Brackets/braces inside JSON string values must not end the block early.++ Regression: counting raw ``[``/``]``/``{``/``}`` per line treated the+ ``]`` inside ``{"path": "a]b"}`` as a closing bracket, so the array was+ truncated mid-way and the remaining rows leaked into later sections.+ """+ import json as _json++ lines = [+ "[",+ ' {"path": "a]b"},',+ ' {"path": "c"}',+ "]",+ ]+ block, end_idx = _extract_json_block(lines, 0)+ assert end_idx == 3+ assert block is not None+ parsed = _json.loads(block)+ assert parsed == [{"path": "a]b"}, {"path": "c"}]++ # Braces inside a string value must likewise be ignored.+ obj_lines = [+ "{",+ ' "msg": "use {curly} and [square]",',+ ' "n": 1',+ "}",+ ]+ obj_block, obj_end = _extract_json_block(obj_lines, 0)+ assert obj_end == 3+ assert obj_block is not None+ assert _json.loads(obj_block) == {"msg": "use {curly} and [square]", "n": 1}+++def test_split_into_sections_keeps_json_array_with_bracket_in_string() -> None:+ """A JSON array embedded in prose stays one JSON section, not fragments.++ With the bracket-in-string bug, the array below split into a truncated+ JSON section plus a stray ``]`` glued onto the trailing prose.+ """+ import json as _json++ content = "\n".join(+ [+ "prose line here that is long enough to matter",+ "[",+ ' {"path": "a]b"},',+ ' {"path": "c"}',+ "]",+ "trailing prose",+ ]+ )++ sections = split_into_sections(content)+ json_sections = [s for s in sections if s.content_type == ContentType.JSON_ARRAY]+ assert len(json_sections) == 1+ parsed = _json.loads(json_sections[0].content)+ assert parsed == [{"path": "a]b"}, {"path": "c"}]++def test_content_router_strategy_and_compress_paths(monkeypatch: pytest.MonkeyPatch) -> None:router = ContentRouter(ContentRouterConfig(prefer_code_aware_for_code=False))