Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(compression): repair entropy preservation + JSON-safe truncation fallback (#1536)

Tejas Chopra840871cb9652
diff --git a/headroom/compression/masks.py b/headroom/compression/masks.py
index c333e1108..adce4c6fe 100644
--- a/headroom/compression/masks.py
+++ b/headroom/compression/masks.py
@@ -12,9 +12,17 @@ The mask is content-agnostic - it's just a boolean array aligned to tokens.
from __future__ import annotations
+import re
from collections.abc import Callable, Sequence
from dataclasses import dataclass, field
+# Minimum word length for entropy-based secret preservation. Normalized Shannon
+# entropy alone cannot tell a 40-char API key from an 8-char diverse word, so a
+# length floor carries that signal. Matches the entropy-detection floor used by
+# secret scanners (trufflehog, detect-secrets); see
+# compute_entropy_mask_for_content.
+SECRET_ENTROPY_MIN_LENGTH = 20
+
@dataclass
class StructureMask:
@@ -343,3 +351,56 @@ def compute_entropy_mask(
mask=mask,
metadata={"source": "entropy", "threshold": threshold},
)
+
+
+def compute_entropy_mask_for_content(
+ content: str,
+ threshold: float = 0.85,
+ min_token_length: int = SECRET_ENTROPY_MIN_LENGTH,
+) -> StructureMask:
+ """Create a character-indexed mask preserving high-entropy *words*.
+
+ Unlike :func:`compute_entropy_mask`, which scores whatever token sequence
+ it is handed, this scans ``content`` for whitespace-delimited words and
+ marks the character ranges of high-entropy words (API keys, OAuth tokens,
+ UUIDs, hashes, private keys) for preservation. The returned mask is aligned
+ to ``list(content)``, so it can be unioned with a character-level structure
+ mask.
+
+ This exists because character-level tokenization (``list(content)``) yields
+ single-character tokens that never reach ``min_token_length`` — which makes
+ :func:`compute_entropy_mask` a silent no-op on plain text, defeating the
+ purpose of entropy preservation. Mapping word-level scores back to character
+ positions restores it.
+
+ The default ``min_token_length`` is a length floor, not a magic number:
+ *normalized* Shannon entropy rates short but character-diverse English words
+ (e.g. "detailed") nearly as high as a real secret, so length is the signal
+ that separates the two. ``SECRET_ENTROPY_MIN_LENGTH`` matches the entropy
+ floor used by secret scanners (trufflehog, detect-secrets); shorter
+ high-entropy strings would over-preserve ordinary prose and defeat
+ compression.
+
+ Args:
+ content: Raw content to scan.
+ threshold: Entropy threshold (0.0-1.0). Higher = more selective.
+ min_token_length: Only score words this long or longer.
+
+ Returns:
+ StructureMask aligned to ``list(content)`` with high-entropy word
+ spans marked for preservation.
+ """
+ mask = [False] * len(content)
+ for match in re.finditer(r"\S+", content):
+ word = match.group()
+ if len(word) < min_token_length:
+ continue
+ if EntropyScore.compute(word, threshold).should_preserve:
+ for i in range(match.start(), match.end()):
+ mask[i] = True
+
+ return StructureMask(
+ tokens=list(content),
+ mask=mask,
+ metadata={"source": "entropy", "threshold": threshold, "granularity": "word"},
+ )
diff --git a/headroom/compression/universal.py b/headroom/compression/universal.py
index 529e83280..e384826ea 100644
--- a/headroom/compression/universal.py
+++ b/headroom/compression/universal.py
@@ -38,7 +38,7 @@ from headroom.compression.handlers.code_handler import CodeStructureHandler
from headroom.compression.handlers.json_handler import JSONStructureHandler
from headroom.compression.masks import (
StructureMask,
- compute_entropy_mask,
+ compute_entropy_mask_for_content,
mask_to_spans,
)
@@ -209,11 +209,13 @@ class UniversalCompressor:
if len(text) <= target_len:
return text
- # Keep first and last portions
+ # Keep first and last portions. The separator must contain no control
+ # characters (RFC 8259 §7): this fallback can run on a span inside a
+ # JSON string value, and a raw newline there produces invalid JSON.
keep_start = target_len * 2 // 3
keep_end = target_len // 3
- return text[:keep_start] + "\n...[compressed]...\n" + text[-keep_end:]
+ return text[:keep_start] + " ...[compressed]... " + text[-keep_end:]
def compress(
self,
@@ -266,10 +268,13 @@ class UniversalCompressor:
handler_result = handler.get_mask(content, tokens, **kwargs)
structure_mask = handler_result.mask
- # Optionally add entropy-based preservation
+ # Optionally add entropy-based preservation. Score whole words (mapped
+ # back to character positions) rather than the character-level `tokens`
+ # above — single-character tokens never reach the min length, which
+ # would make entropy preservation a silent no-op on plain text.
if self.config.use_entropy_preservation:
- entropy_mask = compute_entropy_mask(
- tokens,
+ entropy_mask = compute_entropy_mask_for_content(
+ content,
threshold=self.config.entropy_threshold,
)
# Union: preserve if either mask says preserve
diff --git a/tests/test_compression/test_masks.py b/tests/test_compression/test_masks.py
index 4573454ac..25f746833 100644
--- a/tests/test_compression/test_masks.py
+++ b/tests/test_compression/test_masks.py
@@ -8,6 +8,7 @@ from headroom.compression.masks import (
StructureMask,
apply_mask_to_text,
compute_entropy_mask,
+ compute_entropy_mask_for_content,
mask_to_spans,
)
@@ -232,6 +233,47 @@ class TestComputeEntropyMask:
assert mask.metadata["threshold"] == 0.9
+class TestComputeEntropyMaskForContent:
+ """Tests for compute_entropy_mask_for_content (SEC-01 regression).
+
+ The character-level path (`compute_entropy_mask(list(content))`) is a silent
+ no-op on plain text because every single-character token is below
+ min_token_length. The content-level helper must restore preservation by
+ scoring whole words and mapping them back to character positions.
+ """
+
+ def test_char_level_tokenization_is_inert(self):
+ """Regression: char tokens never reach min length -> nothing preserved."""
+ secret = "Zx9Kq3Wm7Pv2Lr8Nt4Bc6Df1Gh5Jy" # gitleaks:allow synthetic test fixture
+ char_mask = compute_entropy_mask(list(f"k={secret}"), threshold=0.85)
+ # This is the bug the fix routes around: zero preservation.
+ assert sum(char_mask.mask) == 0
+
+ def test_high_entropy_word_char_range_preserved(self):
+ """The full character span of a high-entropy word is marked True."""
+ secret = "Zx9Kq3Wm7Pv2Lr8Nt4Bc6Df1Gh5Jy" # gitleaks:allow synthetic test fixture
+ content = f"prefix {secret} suffix"
+ mask = compute_entropy_mask_for_content(content, threshold=0.85)
+
+ start = content.index(secret)
+ end = start + len(secret)
+ assert all(mask.mask[start:end]) # secret preserved
+ assert not any(mask.mask[:start]) # ordinary words not preserved
+ assert not any(mask.mask[end:]) # trailing words not preserved
+ assert len(mask.mask) == len(content) # char-aligned
+
+ def test_short_words_not_preserved(self):
+ """Short words are not scored regardless of entropy."""
+ mask = compute_entropy_mask_for_content("a b cd ef", threshold=0.5)
+ assert sum(mask.mask) == 0
+
+ def test_metadata_marks_word_granularity(self):
+ mask = compute_entropy_mask_for_content("plain words only", threshold=0.9)
+ assert mask.metadata["source"] == "entropy"
+ assert mask.metadata["threshold"] == 0.9
+ assert mask.metadata["granularity"] == "word"
+
+
class TestApplyMaskToText:
"""Tests for apply_mask_to_text function."""
diff --git a/tests/test_compression/test_universal.py b/tests/test_compression/test_universal.py
index bd190d8db..7c2602a00 100644
--- a/tests/test_compression/test_universal.py
+++ b/tests/test_compression/test_universal.py
@@ -338,3 +338,62 @@ class ShoppingCart:
# Should achieve some compression
assert result.tokens_after < result.tokens_before
assert result.compression_ratio < 1.0
+
+
+class TestSecurityAuditRegressions:
+ """End-to-end regressions for the June 2026 security audit findings."""
+
+ def test_sec01_entropy_preservation_keeps_high_entropy_secret(self):
+ """SEC-01: an enabled entropy pass must preserve high-entropy strings.
+
+ With the character-level tokenization bug, use_entropy_preservation was
+ a silent no-op, so a secret buried in a compressible region was dropped.
+ Uses the deterministic truncation fallback (use_kompress=False) so the
+ middle of the content is genuinely removed when preservation is off.
+ """
+ secret = "Zx9Kq3Wm7Pv2Lr8Nt4Bc6Df1Gh5Jy" # gitleaks:allow synthetic test fixture
+ filler = "the quick brown fox jumps over the lazy dog and runs on. " * 6
+ content = filler + " " + secret + " " + filler
+
+ off = UniversalCompressor(
+ config=UniversalCompressorConfig(
+ use_magika=False,
+ use_kompress=False,
+ use_entropy_preservation=False,
+ ccr_enabled=False,
+ )
+ ).compress(content, content_type=ContentType.TEXT)
+ on = UniversalCompressor(
+ config=UniversalCompressorConfig(
+ use_magika=False,
+ use_kompress=False,
+ use_entropy_preservation=True,
+ ccr_enabled=False,
+ )
+ ).compress(content, content_type=ContentType.TEXT)
+
+ assert secret not in off.compressed # dropped without preservation
+ assert secret in on.compressed # preserved with the fix
+
+ def test_sec02_json_fallback_stays_valid_json(self):
+ """SEC-02: the truncation fallback must not emit control chars into JSON.
+
+ The old separator embedded raw newlines, producing invalid JSON inside a
+ string value when Kompress was unavailable. Validate both paths.
+ """
+ payload = json.dumps(
+ {
+ "service": "api",
+ "payload": "This is a fictional long string value " * 4 + "END",
+ }
+ )
+ for use_kompress in (False, True):
+ res = UniversalCompressor(
+ config=UniversalCompressorConfig(
+ use_magika=False,
+ use_kompress=use_kompress,
+ ccr_enabled=False,
+ )
+ ).compress(payload, content_type=ContentType.JSON)
+ assert "\n" not in res.compressed, f"raw newline leaked (use_kompress={use_kompress})"
+ json.loads(res.compressed) # must not raise