Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(compression): repair entropy preservation + JSON-safe truncation fallback (#1536)
Tejas Chopra840871cb9652
diff --git a/headroom/compression/masks.py b/headroom/compression/masks.pyindex c333e1108..adce4c6fe 100644--- a/headroom/compression/masks.py+++ b/headroom/compression/masks.py@@ -12,9 +12,17 @@ The mask is content-agnostic - it's just a boolean array aligned to tokens.from __future__ import annotations+import refrom collections.abc import Callable, Sequencefrom dataclasses import dataclass, field+# Minimum word length for entropy-based secret preservation. Normalized Shannon+# entropy alone cannot tell a 40-char API key from an 8-char diverse word, so a+# length floor carries that signal. Matches the entropy-detection floor used by+# secret scanners (trufflehog, detect-secrets); see+# compute_entropy_mask_for_content.+SECRET_ENTROPY_MIN_LENGTH = 20+@dataclassclass StructureMask:@@ -343,3 +351,56 @@ def compute_entropy_mask(mask=mask,metadata={"source": "entropy", "threshold": threshold},)+++def compute_entropy_mask_for_content(+ content: str,+ threshold: float = 0.85,+ min_token_length: int = SECRET_ENTROPY_MIN_LENGTH,+) -> StructureMask:+ """Create a character-indexed mask preserving high-entropy *words*.++ Unlike :func:`compute_entropy_mask`, which scores whatever token sequence+ it is handed, this scans ``content`` for whitespace-delimited words and+ marks the character ranges of high-entropy words (API keys, OAuth tokens,+ UUIDs, hashes, private keys) for preservation. The returned mask is aligned+ to ``list(content)``, so it can be unioned with a character-level structure+ mask.++ This exists because character-level tokenization (``list(content)``) yields+ single-character tokens that never reach ``min_token_length`` — which makes+ :func:`compute_entropy_mask` a silent no-op on plain text, defeating the+ purpose of entropy preservation. Mapping word-level scores back to character+ positions restores it.++ The default ``min_token_length`` is a length floor, not a magic number:+ *normalized* Shannon entropy rates short but character-diverse English words+ (e.g. "detailed") nearly as high as a real secret, so length is the signal+ that separates the two. ``SECRET_ENTROPY_MIN_LENGTH`` matches the entropy+ floor used by secret scanners (trufflehog, detect-secrets); shorter+ high-entropy strings would over-preserve ordinary prose and defeat+ compression.++ Args:+ content: Raw content to scan.+ threshold: Entropy threshold (0.0-1.0). Higher = more selective.+ min_token_length: Only score words this long or longer.++ Returns:+ StructureMask aligned to ``list(content)`` with high-entropy word+ spans marked for preservation.+ """+ mask = [False] * len(content)+ for match in re.finditer(r"\S+", content):+ word = match.group()+ if len(word) < min_token_length:+ continue+ if EntropyScore.compute(word, threshold).should_preserve:+ for i in range(match.start(), match.end()):+ mask[i] = True++ return StructureMask(+ tokens=list(content),+ mask=mask,+ metadata={"source": "entropy", "threshold": threshold, "granularity": "word"},+ )diff --git a/headroom/compression/universal.py b/headroom/compression/universal.pyindex 529e83280..e384826ea 100644--- a/headroom/compression/universal.py+++ b/headroom/compression/universal.py@@ -38,7 +38,7 @@ from headroom.compression.handlers.code_handler import CodeStructureHandlerfrom headroom.compression.handlers.json_handler import JSONStructureHandlerfrom headroom.compression.masks import (StructureMask,- compute_entropy_mask,+ compute_entropy_mask_for_content,mask_to_spans,)@@ -209,11 +209,13 @@ class UniversalCompressor:if len(text) <= target_len:return text- # Keep first and last portions+ # Keep first and last portions. The separator must contain no control+ # characters (RFC 8259 §7): this fallback can run on a span inside a+ # JSON string value, and a raw newline there produces invalid JSON.keep_start = target_len * 2 // 3keep_end = target_len // 3- return text[:keep_start] + "\n...[compressed]...\n" + text[-keep_end:]+ return text[:keep_start] + " ...[compressed]... " + text[-keep_end:]def compress(self,@@ -266,10 +268,13 @@ class UniversalCompressor:handler_result = handler.get_mask(content, tokens, **kwargs)structure_mask = handler_result.mask- # Optionally add entropy-based preservation+ # Optionally add entropy-based preservation. Score whole words (mapped+ # back to character positions) rather than the character-level `tokens`+ # above — single-character tokens never reach the min length, which+ # would make entropy preservation a silent no-op on plain text.if self.config.use_entropy_preservation:- entropy_mask = compute_entropy_mask(- tokens,+ entropy_mask = compute_entropy_mask_for_content(+ content,threshold=self.config.entropy_threshold,)# Union: preserve if either mask says preservediff --git a/tests/test_compression/test_masks.py b/tests/test_compression/test_masks.pyindex 4573454ac..25f746833 100644--- a/tests/test_compression/test_masks.py+++ b/tests/test_compression/test_masks.py@@ -8,6 +8,7 @@ from headroom.compression.masks import (StructureMask,apply_mask_to_text,compute_entropy_mask,+ compute_entropy_mask_for_content,mask_to_spans,)@@ -232,6 +233,47 @@ class TestComputeEntropyMask:assert mask.metadata["threshold"] == 0.9+class TestComputeEntropyMaskForContent:+ """Tests for compute_entropy_mask_for_content (SEC-01 regression).++ The character-level path (`compute_entropy_mask(list(content))`) is a silent+ no-op on plain text because every single-character token is below+ min_token_length. The content-level helper must restore preservation by+ scoring whole words and mapping them back to character positions.+ """++ def test_char_level_tokenization_is_inert(self):+ """Regression: char tokens never reach min length -> nothing preserved."""+ secret = "Zx9Kq3Wm7Pv2Lr8Nt4Bc6Df1Gh5Jy" # gitleaks:allow synthetic test fixture+ char_mask = compute_entropy_mask(list(f"k={secret}"), threshold=0.85)+ # This is the bug the fix routes around: zero preservation.+ assert sum(char_mask.mask) == 0++ def test_high_entropy_word_char_range_preserved(self):+ """The full character span of a high-entropy word is marked True."""+ secret = "Zx9Kq3Wm7Pv2Lr8Nt4Bc6Df1Gh5Jy" # gitleaks:allow synthetic test fixture+ content = f"prefix {secret} suffix"+ mask = compute_entropy_mask_for_content(content, threshold=0.85)++ start = content.index(secret)+ end = start + len(secret)+ assert all(mask.mask[start:end]) # secret preserved+ assert not any(mask.mask[:start]) # ordinary words not preserved+ assert not any(mask.mask[end:]) # trailing words not preserved+ assert len(mask.mask) == len(content) # char-aligned++ def test_short_words_not_preserved(self):+ """Short words are not scored regardless of entropy."""+ mask = compute_entropy_mask_for_content("a b cd ef", threshold=0.5)+ assert sum(mask.mask) == 0++ def test_metadata_marks_word_granularity(self):+ mask = compute_entropy_mask_for_content("plain words only", threshold=0.9)+ assert mask.metadata["source"] == "entropy"+ assert mask.metadata["threshold"] == 0.9+ assert mask.metadata["granularity"] == "word"++class TestApplyMaskToText:"""Tests for apply_mask_to_text function."""diff --git a/tests/test_compression/test_universal.py b/tests/test_compression/test_universal.pyindex bd190d8db..7c2602a00 100644--- a/tests/test_compression/test_universal.py+++ b/tests/test_compression/test_universal.py@@ -338,3 +338,62 @@ class ShoppingCart:# Should achieve some compressionassert result.tokens_after < result.tokens_beforeassert result.compression_ratio < 1.0+++class TestSecurityAuditRegressions:+ """End-to-end regressions for the June 2026 security audit findings."""++ def test_sec01_entropy_preservation_keeps_high_entropy_secret(self):+ """SEC-01: an enabled entropy pass must preserve high-entropy strings.++ With the character-level tokenization bug, use_entropy_preservation was+ a silent no-op, so a secret buried in a compressible region was dropped.+ Uses the deterministic truncation fallback (use_kompress=False) so the+ middle of the content is genuinely removed when preservation is off.+ """+ secret = "Zx9Kq3Wm7Pv2Lr8Nt4Bc6Df1Gh5Jy" # gitleaks:allow synthetic test fixture+ filler = "the quick brown fox jumps over the lazy dog and runs on. " * 6+ content = filler + " " + secret + " " + filler++ off = UniversalCompressor(+ config=UniversalCompressorConfig(+ use_magika=False,+ use_kompress=False,+ use_entropy_preservation=False,+ ccr_enabled=False,+ )+ ).compress(content, content_type=ContentType.TEXT)+ on = UniversalCompressor(+ config=UniversalCompressorConfig(+ use_magika=False,+ use_kompress=False,+ use_entropy_preservation=True,+ ccr_enabled=False,+ )+ ).compress(content, content_type=ContentType.TEXT)++ assert secret not in off.compressed # dropped without preservation+ assert secret in on.compressed # preserved with the fix++ def test_sec02_json_fallback_stays_valid_json(self):+ """SEC-02: the truncation fallback must not emit control chars into JSON.++ The old separator embedded raw newlines, producing invalid JSON inside a+ string value when Kompress was unavailable. Validate both paths.+ """+ payload = json.dumps(+ {+ "service": "api",+ "payload": "This is a fictional long string value " * 4 + "END",+ }+ )+ for use_kompress in (False, True):+ res = UniversalCompressor(+ config=UniversalCompressorConfig(+ use_magika=False,+ use_kompress=use_kompress,+ ccr_enabled=False,+ )+ ).compress(payload, content_type=ContentType.JSON)+ assert "\n" not in res.compressed, f"raw newline leaked (use_kompress={use_kompress})"+ json.loads(res.compressed) # must not raise