Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(compression): correct JSON array item counting and entropy gate (#887)
Ashishd6f0f0f64269
diff --git a/headroom/compression/handlers/json_handler.py b/headroom/compression/handlers/json_handler.pyindex 941ef2b87..67e41af37 100644--- a/headroom/compression/handlers/json_handler.py+++ b/headroom/compression/handlers/json_handler.py@@ -138,30 +138,44 @@ class JSONStructureHandler(BaseStructureHandler):# Build character-level maskmask = [False] * len(content)- # Track array depth for selective preservation- array_depth = 0- array_item_counts: dict[int, int] = {} # depth -> count+ # Track containers so commas inside objects are not counted as+ # array item separators — only commas whose immediate enclosing+ # container is an array advance that array's item index. Values+ # nested in an object that is itself an array item inherit the+ # innermost enclosing array's index via array_item_stack[-1].+ container_stack: list[str] = [] # "[" or "{" per open container+ array_item_stack: list[int] = [] # item count per open arrayfor token in json_tokens:- # Track array items+ # Track containersif token.token_type == JSONTokenType.BRACKET:- if token.text == "[":- array_depth += 1- array_item_counts[array_depth] = 0+ if token.text in "{[":+ container_stack.append(token.text)+ if token.text == "[":+ array_item_stack.append(0)+ elif token.text == "}":+ if container_stack and container_stack[-1] == "{":+ container_stack.pop()elif token.text == "]":- if array_depth in array_item_counts:- del array_item_counts[array_depth]- array_depth = max(0, array_depth - 1)-- # Count array items at commas- if token.token_type == JSONTokenType.COMMA and array_depth > 0:- array_item_counts[array_depth] = array_item_counts.get(array_depth, 0) + 1+ if container_stack and container_stack[-1] == "[":+ container_stack.pop()+ if array_item_stack:+ array_item_stack.pop()++ # Count array items only at the array's own commas+ if (+ token.token_type == JSONTokenType.COMMA+ and container_stack+ and container_stack[-1] == "["+ and array_item_stack+ ):+ array_item_stack[-1] += 1# Determine if this token should be preservedpreserve = self._should_preserve_token(token,- array_depth,- array_item_counts.get(array_depth, 0),+ len(array_item_stack),+ array_item_stack[-1] if array_item_stack else 0,)# Mark in mask@@ -225,9 +239,13 @@ class JSONStructureHandler(BaseStructureHandler):if self.preserve_high_entropy:# Strip quotes for entropy calculationvalue = token.text.strip('"')- score = EntropyScore.compute(value, self.entropy_threshold)- if score.should_preserve:- return True+ # Entropy targets identifiers (UUIDs, hashes, API keys).+ # Self-normalized entropy also scores English prose >0.85,+ # so gate on the cheapest identifier signal: no spaces.+ if " " not in value:+ score = EntropyScore.compute(value, self.entropy_threshold)+ if score.should_preserve:+ return Truereturn Falsediff --git a/tests/test_compression/test_json_handler.py b/tests/test_compression/test_json_handler.pyindex ed30f07aa..c86b7e20f 100644--- a/tests/test_compression/test_json_handler.py+++ b/tests/test_compression/test_json_handler.py@@ -153,6 +153,75 @@ class TestJSONStructureHandler:assert result.mask.mask[first_id] is Trueassert result.mask.mask[second_id] is True+ def test_object_commas_do_not_advance_array_item_index(self):+ """Commas between keys inside an object must not count as array+ item separators.++ Regression: depth-keyed comma counting treated every comma under+ array_depth > 0 as an item boundary, so an array's FIRST object+ exhausted max_array_items_full within its own keys and later+ values were dropped despite being in item 0.+ """+ handler = JSONStructureHandler(+ preserve_short_values=True,+ short_value_threshold=20,+ max_array_items_full=3,+ )+ # Single array item (index 0) with 5 short values — all must+ # stay eligible for preservation.+ content = '[{"a": "valuea", "b": "valueb", "c": "valuec", "d": "valued", "e": "valuee"}]'+ result = handler.get_mask(content)++ for marker in ('"valuea"', '"valueb"', '"valuec"', '"valued"', '"valuee"'):+ start = content.index(marker)+ for i in range(start, start + len(marker)):+ assert result.mask.mask[i] is True, f"{marker} in array item 0 should be preserved"++ def test_array_items_past_threshold_compressed(self):+ """Items at index >= max_array_items_full are still compressed."""+ handler = JSONStructureHandler(+ preserve_short_values=True,+ short_value_threshold=20,+ max_array_items_full=2,+ )+ content = '[{"v": "itemzero"}, {"v": "itemone"}, {"v": "itemtwo"}]'+ result = handler.get_mask(content)++ # Items 0 and 1 preserved+ for marker in ('"itemzero"', '"itemone"'):+ start = content.index(marker)+ assert result.mask.mask[start + 1] is True, f"{marker} should be preserved"++ # Item 2 is past the threshold — value chars compressed+ start = content.index('"itemtwo"')+ inner = range(start + 1, start + len('"itemtwo"') - 1)+ assert not any(result.mask.mask[i] for i in inner), (+ "values in array items past max_array_items_full should be compressible"+ )++ def test_prose_not_preserved_by_entropy(self):+ """Long natural-language strings must not pass the entropy gate.++ Regression: self-normalized Shannon entropy scores English prose+ above the 0.85 threshold, so every description survived+ compression. Identifiers (no spaces) are still preserved.+ """+ handler = JSONStructureHandler(short_value_threshold=20)+ prose = "Context optimization layer for LLM applications with caching"+ uuid = "8f14e45f-ceea-4123-8f14-e45fceea4123"+ content = f'{{"description": "{prose}", "id": "{uuid}"}}'+ result = handler.get_mask(content)++ prose_start = content.index(prose)+ assert not any(result.mask.mask[i] for i in range(prose_start, prose_start + len(prose))), (+ "long prose value should be compressible"+ )++ uuid_start = content.index(uuid)+ assert all(result.mask.mask[i] for i in range(uuid_start, uuid_start + len(uuid))), (+ "UUID value should be preserved via entropy"+ )+def test_metadata_contains_key_count(self, handler):"""Test that metadata includes key count."""content = '{"a": 1, "b": 2, "c": 3}'