Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(compression): correct JSON array item counting and entropy gate (#887)

Ashishd6f0f0f64269
diff --git a/headroom/compression/handlers/json_handler.py b/headroom/compression/handlers/json_handler.py
index 941ef2b87..67e41af37 100644
--- a/headroom/compression/handlers/json_handler.py
+++ b/headroom/compression/handlers/json_handler.py
@@ -138,30 +138,44 @@ class JSONStructureHandler(BaseStructureHandler):
# Build character-level mask
mask = [False] * len(content)
- # Track array depth for selective preservation
- array_depth = 0
- array_item_counts: dict[int, int] = {} # depth -> count
+ # Track containers so commas inside objects are not counted as
+ # array item separators — only commas whose immediate enclosing
+ # container is an array advance that array's item index. Values
+ # nested in an object that is itself an array item inherit the
+ # innermost enclosing array's index via array_item_stack[-1].
+ container_stack: list[str] = [] # "[" or "{" per open container
+ array_item_stack: list[int] = [] # item count per open array
for token in json_tokens:
- # Track array items
+ # Track containers
if token.token_type == JSONTokenType.BRACKET:
- if token.text == "[":
- array_depth += 1
- array_item_counts[array_depth] = 0
+ if token.text in "{[":
+ container_stack.append(token.text)
+ if token.text == "[":
+ array_item_stack.append(0)
+ elif token.text == "}":
+ if container_stack and container_stack[-1] == "{":
+ container_stack.pop()
elif token.text == "]":
- if array_depth in array_item_counts:
- del array_item_counts[array_depth]
- array_depth = max(0, array_depth - 1)
-
- # Count array items at commas
- if token.token_type == JSONTokenType.COMMA and array_depth > 0:
- array_item_counts[array_depth] = array_item_counts.get(array_depth, 0) + 1
+ if container_stack and container_stack[-1] == "[":
+ container_stack.pop()
+ if array_item_stack:
+ array_item_stack.pop()
+
+ # Count array items only at the array's own commas
+ if (
+ token.token_type == JSONTokenType.COMMA
+ and container_stack
+ and container_stack[-1] == "["
+ and array_item_stack
+ ):
+ array_item_stack[-1] += 1
# Determine if this token should be preserved
preserve = self._should_preserve_token(
token,
- array_depth,
- array_item_counts.get(array_depth, 0),
+ len(array_item_stack),
+ array_item_stack[-1] if array_item_stack else 0,
)
# Mark in mask
@@ -225,9 +239,13 @@ class JSONStructureHandler(BaseStructureHandler):
if self.preserve_high_entropy:
# Strip quotes for entropy calculation
value = token.text.strip('"')
- score = EntropyScore.compute(value, self.entropy_threshold)
- if score.should_preserve:
- return True
+ # Entropy targets identifiers (UUIDs, hashes, API keys).
+ # Self-normalized entropy also scores English prose >0.85,
+ # so gate on the cheapest identifier signal: no spaces.
+ if " " not in value:
+ score = EntropyScore.compute(value, self.entropy_threshold)
+ if score.should_preserve:
+ return True
return False
diff --git a/tests/test_compression/test_json_handler.py b/tests/test_compression/test_json_handler.py
index ed30f07aa..c86b7e20f 100644
--- a/tests/test_compression/test_json_handler.py
+++ b/tests/test_compression/test_json_handler.py
@@ -153,6 +153,75 @@ class TestJSONStructureHandler:
assert result.mask.mask[first_id] is True
assert result.mask.mask[second_id] is True
+ def test_object_commas_do_not_advance_array_item_index(self):
+ """Commas between keys inside an object must not count as array
+ item separators.
+
+ Regression: depth-keyed comma counting treated every comma under
+ array_depth > 0 as an item boundary, so an array's FIRST object
+ exhausted max_array_items_full within its own keys and later
+ values were dropped despite being in item 0.
+ """
+ handler = JSONStructureHandler(
+ preserve_short_values=True,
+ short_value_threshold=20,
+ max_array_items_full=3,
+ )
+ # Single array item (index 0) with 5 short values — all must
+ # stay eligible for preservation.
+ content = '[{"a": "valuea", "b": "valueb", "c": "valuec", "d": "valued", "e": "valuee"}]'
+ result = handler.get_mask(content)
+
+ for marker in ('"valuea"', '"valueb"', '"valuec"', '"valued"', '"valuee"'):
+ start = content.index(marker)
+ for i in range(start, start + len(marker)):
+ assert result.mask.mask[i] is True, f"{marker} in array item 0 should be preserved"
+
+ def test_array_items_past_threshold_compressed(self):
+ """Items at index >= max_array_items_full are still compressed."""
+ handler = JSONStructureHandler(
+ preserve_short_values=True,
+ short_value_threshold=20,
+ max_array_items_full=2,
+ )
+ content = '[{"v": "itemzero"}, {"v": "itemone"}, {"v": "itemtwo"}]'
+ result = handler.get_mask(content)
+
+ # Items 0 and 1 preserved
+ for marker in ('"itemzero"', '"itemone"'):
+ start = content.index(marker)
+ assert result.mask.mask[start + 1] is True, f"{marker} should be preserved"
+
+ # Item 2 is past the threshold — value chars compressed
+ start = content.index('"itemtwo"')
+ inner = range(start + 1, start + len('"itemtwo"') - 1)
+ assert not any(result.mask.mask[i] for i in inner), (
+ "values in array items past max_array_items_full should be compressible"
+ )
+
+ def test_prose_not_preserved_by_entropy(self):
+ """Long natural-language strings must not pass the entropy gate.
+
+ Regression: self-normalized Shannon entropy scores English prose
+ above the 0.85 threshold, so every description survived
+ compression. Identifiers (no spaces) are still preserved.
+ """
+ handler = JSONStructureHandler(short_value_threshold=20)
+ prose = "Context optimization layer for LLM applications with caching"
+ uuid = "8f14e45f-ceea-4123-8f14-e45fceea4123"
+ content = f'{{"description": "{prose}", "id": "{uuid}"}}'
+ result = handler.get_mask(content)
+
+ prose_start = content.index(prose)
+ assert not any(result.mask.mask[i] for i in range(prose_start, prose_start + len(prose))), (
+ "long prose value should be compressible"
+ )
+
+ uuid_start = content.index(uuid)
+ assert all(result.mask.mask[i] for i in range(uuid_start, uuid_start + len(uuid))), (
+ "UUID value should be preserved via entropy"
+ )
+
def test_metadata_contains_key_count(self, handler):
"""Test that metadata includes key count."""
content = '{"a": 1, "b": 2, "c": 3}'