Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(kompress): append the CCR marker whenever the saving pays for it (#3484)

gglucass7bd4dbaf8f40
diff --git a/headroom/transforms/kompress_compressor.py b/headroom/transforms/kompress_compressor.py
index 770b32476..29d0486ef 100644
--- a/headroom/transforms/kompress_compressor.py
+++ b/headroom/transforms/kompress_compressor.py
@@ -1294,6 +1294,41 @@ class KompressResult:
return (self.tokens_saved / self.original_tokens) * 100
+_payload_encoder: Any = None
+
+
+def payload_tokens(text: str) -> int:
+ """Token count of a complete payload, in one consistent unit.
+
+ The unit is cl100k_base (tiktoken, a hard dependency), used as a fixed
+ estimate. Actual model tokenizers, including those of other OpenAI
+ models, can differ. Without an encoder, the fallback compares character
+ counts; that heuristic is not a bound on provider token counts.
+
+ The CCR gate measures the whole original and the whole candidate-plus-
+ marker with this, never a marker-only cost against a word count: the
+ marker is 36-45 tokens for 12 words, the words Kompress drops can be one
+ token each, and the word left at the head of the candidate can tokenize
+ differently from its space-prefixed form in the source. Only a comparison
+ of the two complete texts in one unit establishes that the shipped
+ payload is smaller.
+ """
+ global _payload_encoder
+ if _payload_encoder is None:
+ try:
+ import tiktoken
+
+ _payload_encoder = tiktoken.get_encoding("cl100k_base")
+ except Exception:
+ _payload_encoder = False
+ if _payload_encoder:
+ try:
+ return len(_payload_encoder.encode(text, disallowed_special=()))
+ except Exception:
+ pass
+ return len(text)
+
+
def ccr_retrieval_marker(
n_words: int, compressed_count: int, ccr_source: str, cache_key: str
) -> str:
@@ -1725,30 +1760,43 @@ class KompressCompressor(Transform):
compressed_words = [words[w] for w in sorted(kept_ids) if w < n_words]
compressed = " ".join(compressed_words)
compressed_count = len(compressed_words)
- ratio = compressed_count / n_words if n_words else 1.0
-
- result = KompressResult(
- compressed=compressed,
- original=content,
- original_tokens=n_words,
- compressed_tokens=compressed_count,
- compression_ratio=ratio,
- model_used=self.config.model_id,
- )
-
- # CCR marker
- if self.config.enable_ccr and ratio < 0.8:
+ cache_key: str | None = None
+ original_tokens = n_words
+ compressed_tokens = compressed_count
+
+ # CCR marker: anything the lossy pass shrank must stay retrievable,
+ # and the complete marked payload must be smaller than the
+ # original. Both are measured whole, in one unit (payload_tokens);
+ # a candidate that is not is passed through. The accounting then
+ # reports that same measurement, so ``tokens_saved`` describes
+ # the shipped payload rather than a word count.
+ if self.config.enable_ccr:
ccr_source = ccr_original if ccr_original is not None else content
ccr_source_tokens = len(ccr_source.split())
cache_key = self._store_in_ccr(ccr_source, compressed, ccr_source_tokens)
if cache_key:
- result.cache_key = cache_key
# Report the source line span so a reader can tell content was
# compressed away rather than absent — "items" counts words, which
# does not map to lines and reads as evidence of absence (#2586).
- result.compressed += ccr_retrieval_marker(
+ marked = compressed + ccr_retrieval_marker(
n_words, compressed_count, ccr_source, cache_key
)
+ original_tokens = payload_tokens(content)
+ compressed_tokens = payload_tokens(marked)
+ if compressed_tokens >= original_tokens:
+ return self._passthrough(content, n_words)
+ compressed = marked
+
+ ratio = compressed_tokens / original_tokens if original_tokens else 1.0
+ result = KompressResult(
+ compressed=compressed,
+ original=content,
+ original_tokens=original_tokens,
+ compressed_tokens=compressed_tokens,
+ compression_ratio=ratio,
+ cache_key=cache_key,
+ model_used=self.config.model_id,
+ )
if inference_ms >= 1000.0:
logger.info(
@@ -2119,33 +2167,40 @@ class KompressCompressor(Transform):
compressed_words = [words[w] for w in sorted(kept_ids) if w < n_words]
compressed = " ".join(compressed_words)
compressed_count = len(compressed_words)
- comp_ratio = compressed_count / n_words if n_words else 1.0
+ cache_key: str | None = None
+ original_tokens = n_words
+ compressed_tokens = compressed_count
- result = KompressResult(
- compressed=compressed,
- original=content,
- original_tokens=n_words,
- compressed_tokens=compressed_count,
- compression_ratio=comp_ratio,
- model_used=self.config.model_id,
- )
-
- if self.config.enable_ccr and comp_ratio < 0.8:
+ # Same gate and accounting as the single path.
+ if self.config.enable_ccr:
ccr_source = ccr_sources[text_idx]
if ccr_source is None:
ccr_source = content
ccr_source_tokens = len(ccr_source.split())
cache_key = self._store_in_ccr(ccr_source, compressed, ccr_source_tokens)
if cache_key:
- result.cache_key = cache_key
# Report the source line span so a reader can tell content was
# compressed away rather than absent — "items" counts words, which
# does not map to lines and reads as evidence of absence (#2586).
- result.compressed += ccr_retrieval_marker(
+ marked = compressed + ccr_retrieval_marker(
n_words, compressed_count, ccr_source, cache_key
)
+ original_tokens = payload_tokens(content)
+ compressed_tokens = payload_tokens(marked)
+ if compressed_tokens >= original_tokens:
+ results[text_idx] = self._passthrough(content, n_words)
+ continue
+ compressed = marked
- results[text_idx] = result
+ results[text_idx] = KompressResult(
+ compressed=compressed,
+ original=content,
+ original_tokens=original_tokens,
+ compressed_tokens=compressed_tokens,
+ compression_ratio=compressed_tokens / original_tokens if original_tokens else 1.0,
+ cache_key=cache_key,
+ model_used=self.config.model_id,
+ )
# Safety: every slot must be populated.
final: list[KompressResult] = []
diff --git a/headroom/transforms/kompress_remote.py b/headroom/transforms/kompress_remote.py
index 984d9db9f..0d1be810d 100644
--- a/headroom/transforms/kompress_remote.py
+++ b/headroom/transforms/kompress_remote.py
@@ -57,7 +57,13 @@ import logging
import httpx
-from .kompress_compressor import KompressConfig, KompressResult, store_kompress_in_ccr
+from .kompress_compressor import (
+ KompressConfig,
+ KompressResult,
+ ccr_retrieval_marker,
+ payload_tokens,
+ store_kompress_in_ccr,
+)
logger = logging.getLogger(__name__)
@@ -94,7 +100,6 @@ _MIN_WORDS = 10
# Accept-any-shrink CCR gate, identical to KompressCompressor.compress: only
# store + mark when the shrink is worth the retrieval marker's own cost.
-_CCR_RATIO_GATE = 0.8
class RemoteKompressCompressor:
@@ -238,7 +243,9 @@ class RemoteKompressCompressor:
# CCR stays PROXY-LOCAL: endpoint is stateless (enable_ccr=False), so we
# store the mapping + append the retrieval marker here — same policy and
# marker format as KompressCompressor.compress.
- if self.config.enable_ccr and result.compression_ratio < _CCR_RATIO_GATE:
+ if self.config.enable_ccr and compressed != content:
+ # Same gate as KompressCompressor: the marked payload must save
+ # tokens, and a result that cannot pay for the marker passes through.
# Store the PRE-protection text when the caller supplied it. ``content``
# may be the tag-protected placeholder intermediate, and storing that
# makes a later full retrieval hand back {{HEADROOM_TAG_N}} instead of
@@ -253,15 +260,23 @@ class RemoteKompressCompressor:
)
cache_key = store_kompress_in_ccr(ccr_source, compressed, ccr_source_tokens)
if cache_key:
- result.cache_key = cache_key
# Report the source line span so a reader can tell content was
# compressed away rather than absent (#2586).
- source_lines = ccr_source.count("\n") + 1
- line_word = "line" if source_lines == 1 else "lines"
- result.compressed += (
- f"\n[{result.original_tokens} words compressed to "
- f"{result.compressed_tokens} (from {source_lines} source {line_word})."
- f" Retrieve more: hash={cache_key}]"
+ marked = compressed + ccr_retrieval_marker(
+ result.original_tokens, result.compressed_tokens, ccr_source, cache_key
+ )
+ # Whole original against whole marked candidate, one unit,
+ # and the accounting reports that measurement.
+ original_tokens = payload_tokens(content)
+ compressed_tokens = payload_tokens(marked)
+ if compressed_tokens >= original_tokens:
+ return self._passthrough(content, n_words)
+ result.cache_key = cache_key
+ result.compressed = marked
+ result.original_tokens = original_tokens
+ result.compressed_tokens = compressed_tokens
+ result.compression_ratio = (
+ compressed_tokens / original_tokens if original_tokens else 1.0
)
return result
diff --git a/tests/test_ccr_tag_placeholder_regression.py b/tests/test_ccr_tag_placeholder_regression.py
index 2a3068bb4..94ad18a0e 100644
--- a/tests/test_ccr_tag_placeholder_regression.py
+++ b/tests/test_ccr_tag_placeholder_regression.py
@@ -142,8 +142,9 @@ def test_compress_batch_validates_ccr_originals_length():
# it keeps the first two words of each chunk so the compression ratio clears the
# ``< 0.8`` CCR-store threshold.
-_RAW = "<system-reminder>CRITICAL: invoke the skill</system-reminder> " + " ".join(["filler"] * 40)
-_PLACEHOLDER = "{{HEADROOM_TAG_0}} " + " ".join(["filler"] * 40)
+# 100 filler words: keeping two saves ~99, well past the marker cost gate.
+_RAW = "<system-reminder>CRITICAL: invoke the skill</system-reminder> " + " ".join(["filler"] * 100)
+_PLACEHOLDER = "{{HEADROOM_TAG_0}} " + " ".join(["filler"] * 100)
class _FakeEncoding:
diff --git a/tests/test_kompress_marker_gate.py b/tests/test_kompress_marker_gate.py
new file mode 100644
index 000000000..8cea9356b
--- /dev/null
+++ b/tests/test_kompress_marker_gate.py
@@ -0,0 +1,154 @@
+"""The CCR retrieval marker follows the saving, and the saving must be real.
+
+A lossy Kompress result that shipped without a marker was discarded by the
+router's lossy-unrecoverable guard (#1307), so every pass that shrank a block
+by less than 20 percent ran the model and saved nothing. The gate now compares
+the whole original with the whole candidate-plus-marker in one token unit
+(cl100k_base, an estimate for non-OpenAI providers; one token per character
+without an encoder) and passes the candidate through unless it is smaller. A
+marked result reports that same measurement. Word counts cannot do this: the
+marker is 36-45 tokens for 12 words, dropped words can be one token each, and
+the word left at the head of the candidate can tokenize differently from its
+space-prefixed form in the source.
+"""
+
+from __future__ import annotations
+
+import hashlib
+
+import pytest
+
+from headroom.transforms import kompress_compressor as kc
+from headroom.transforms.kompress_compressor import (
+ KompressCompressor,
+ KompressConfig,
+ ccr_retrieval_marker,
+ payload_tokens,
+)
+
+
+class _Enc(dict):
+ def word_ids(self, batch_index=0):
+ return self["_word_ids"][batch_index]
+
+
+class _Tok:
+ def __call__(self, chunk_words, **kw):
+ batch_words = (
+ chunk_words if chunk_words and isinstance(chunk_words[0], list) else [chunk_words]
+ )
+ return _Enc(
+ input_ids=[[0] * len(words) for words in batch_words],
+ attention_mask=[[1] * len(words) for words in batch_words],
+ _word_ids=[list(range(len(words))) for words in batch_words],
+ )
+
+
+class _DropFirst:
+ """Keep every word except the first ``drop`` of each row."""
+
+ def __init__(self, drop: int) -> None:
+ self.drop = drop
+
+ def get_keep_mask(self, input_ids, attention_mask):
+ return [[idx >= self.drop for idx, _ in enumerate(row)] for row in input_ids]
+
+ def get_scores(self, input_ids, attention_mask):
+ return [[0.0 if idx < self.drop else 1.0 for idx, _ in enumerate(row)] for row in input_ids]
+
+
+def _install(monkeypatch, drop: int) -> None:
+ monkeypatch.setattr(kc, "_load_kompress", lambda *a, **k: (_DropFirst(drop), _Tok(), "onnx"))
+ monkeypatch.setattr(kc, "_model_device_type", lambda *a, **k: "cpu")
+
+
+def _prose(n: int) -> str:
+ # Plain lowercase words: nothing the must-keep override would pin.
+ return " ".join(f"w{chr(97 + i % 26)}{chr(97 + (i // 26) % 26)}" for i in range(n))
+
+
+def _real_key(source: str) -> str:
+ """The CCR store's own key: the SHA-256 prefix of the stored source."""
+ return hashlib.sha256(source.encode()).hexdigest()[:24]
+
+
+def _compressor(monkeypatch, **config) -> KompressCompressor:
+ compressor = KompressCompressor(KompressConfig(min_input_words=10, **config))
+ monkeypatch.setattr(compressor, "_should_batch_single_content", lambda *a, **k: False)
+ monkeypatch.setattr(compressor, "_should_use_sequential_fallback", lambda: False)
+ monkeypatch.setattr(compressor, "_store_in_ccr", lambda source, *a, **k: _real_key(source))
+ return compressor
+
+
+# 100 single-token words: the tightest source there is. Dropping 41 saves 41
+# tokens, and the marker for it (with its real source-derived hash) costs 43.
+SINGLE_TOKEN_SOURCE = " ".join(["alpha"] * 99 + ["nfs"])
+# Dropping 36 leaves "bureaucratic" at the head of the candidate, where it
+# tokenizes differently from " bureaucratic" in the source: 100 -> 103 with
+# the marker, though a marker-only cost against saved words called it 99.
+HEAD_WORD_SOURCE = " ".join(["alpha"] * 36 + ["bureaucratic"] + ["alpha"] * 63)
+
+
+def _tok(text: str) -> int:
+ enc = pytest.importorskip("tiktoken").get_encoding("cl100k_base")
+ return len(enc.encode(text))
+
+
+def test_payload_tokens_is_cl100k_or_one_per_character(monkeypatch):
+ marker = ccr_retrieval_marker(100, 59, SINGLE_TOKEN_SOURCE, _real_key(SINGLE_TOKEN_SOURCE))
+ assert _real_key(SINGLE_TOKEN_SOURCE) == "7dbb8f8de9f3e1d7c6f3a6e1"
+ assert payload_tokens(marker) == _tok(marker) == 43
+ assert payload_tokens(SINGLE_TOKEN_SOURCE) == _tok(SINGLE_TOKEN_SOURCE) == 100
+ monkeypatch.setattr(kc, "_payload_encoder", False)
+ assert payload_tokens(marker) == len(marker)
+
+
+ ("source", "drop"),
+ [(SINGLE_TOKEN_SOURCE, 41), (HEAD_WORD_SOURCE, 36)],
+ ids=["single-token-words", "retokenized-head-word"],
+)
+def test_candidates_that_do_not_shrink_the_whole_payload_pass_through(monkeypatch, source, drop):
+ kept = " ".join(source.split()[drop:])
+ marked = kept + ccr_retrieval_marker(100, 100 - drop, source, _real_key(source))
+ assert _tok(marked) > _tok(source) # the reviewer's measurement
+ _install(monkeypatch, drop=drop)
+ result = _compressor(monkeypatch).compress(source)
+ assert result.compressed == source
+ assert result.cache_key is None
+ assert result.compression_ratio == 1.0
+ [batched] = _compressor(monkeypatch).compress_batch([source], batch_size=8)
+ assert batched.compressed == source and batched.compression_ratio == 1.0
+
+
+def test_marked_result_reports_the_whole_payload_measurement(monkeypatch):
+ # 300 single-token words, drop 60: 240 kept plus a ~43-token marker is
+ # smaller than 300, and the accounting is that measurement, not words.
+ source = " ".join(["alpha"] * 299 + ["nfs"])
+ _install(monkeypatch, drop=60)
+ result = _compressor(monkeypatch).compress(source)
+ marked = " ".join(source.split()[60:]) + ccr_retrieval_marker(
+ 300, 240, source, _real_key(source)
+ )
+ assert result.compressed == marked
+ assert result.cache_key == _real_key(source)
+ assert (result.original_tokens, result.compressed_tokens) == (_tok(source), _tok(marked))
+ assert result.compression_ratio == _tok(marked) / _tok(source)
+ [batched] = _compressor(monkeypatch).compress_batch([source], batch_size=8)
+ assert batched.compressed == result.compressed
+ assert (batched.original_tokens, batched.compressed_tokens) == (
+ result.original_tokens,
+ result.compressed_tokens,
+ )
+
+
+def test_no_ccr_mode_ships_unmarked_lossy_unchanged(monkeypatch):
+ # Without CCR there is no marker and no whole-payload gate: the deliberate
+ # output is the bare lossy result in word counts, on both paths, as before.
+ _install(monkeypatch, drop=41)
+ compressor = _compressor(monkeypatch, enable_ccr=False)
+ result = compressor.compress(SINGLE_TOKEN_SOURCE)
+ assert (result.original_tokens, result.compressed_tokens) == (100, 59)
+ assert "Retrieve more" not in result.compressed
+ [batched] = compressor.compress_batch([SINGLE_TOKEN_SOURCE], batch_size=8)
+ assert batched.compressed_tokens == 59
diff --git a/tests/test_kompress_request_nonblocking.py b/tests/test_kompress_request_nonblocking.py
index c01a8405f..5c92a146d 100644
--- a/tests/test_kompress_request_nonblocking.py
+++ b/tests/test_kompress_request_nonblocking.py
@@ -235,7 +235,9 @@ def test_capacity_available_still_compresses(monkeypatch):
lambda *args, **kwargs: (_FakeModel(), _FakeTokenizer(), "onnx"),
)
- result = KompressCompressor(KompressConfig(min_input_words=10)).compress(
+ # No CCR: this test is about capacity, and 10 saved words would not pay
+ # for a retrieval marker (CCR_MARKER_COST_WORDS), which is a passthrough.
+ result = KompressCompressor(KompressConfig(min_input_words=10, enable_ccr=False)).compress(
" ".join(["word"] * 20), allow_download=False
)
assert 0 < result.compression_ratio < 1.0
diff --git a/tests/test_remote_kompress_dropin.py b/tests/test_remote_kompress_dropin.py
index d423bfa3c..6b295e961 100644
--- a/tests/test_remote_kompress_dropin.py
+++ b/tests/test_remote_kompress_dropin.py
@@ -117,8 +117,9 @@ def _compressor(monkeypatch, *, enable_ccr: bool, payload: dict):
return c
-ORIGINAL = "real secret block " * 20
-PLACEHOLDER = "{{HEADROOM_TAG_0}} " * 20
+# 60 words: "short" saves 59, past the marker cost gate (CCR_MARKER_COST_WORDS).
+ORIGINAL = "real secret block " * 60
+PLACEHOLDER = "{{HEADROOM_TAG_0}} " * 60
def test_passing_ccr_original_no_longer_raises(monkeypatch) -> None:
@@ -190,3 +191,39 @@ def test_the_common_path_without_an_override_is_unchanged(monkeypatch) -> None:
assert stored["original"] == ORIGINAL
# Still the endpoint's own count when no override was supplied.
assert stored["tokens"] == 999
+
+
+def test_remote_marker_gate_measures_the_whole_payload_like_local(monkeypatch) -> None:
+ """Parity with KompressCompressor: the whole original against the whole
+ marked candidate in one token unit. Both boundary sources pass through
+ (41 single-token words saved against a 43-token marker; a head word that
+ retokenizes: 100 -> 103), and a real saving reports that measurement."""
+ import hashlib
+
+ from headroom.transforms.kompress_compressor import ccr_retrieval_marker, payload_tokens
+
+ def _store(original, compressed, original_tokens): # noqa: ANN001
+ return hashlib.sha256(original.encode()).hexdigest()[:24]
+
+ monkeypatch.setattr("headroom.transforms.kompress_remote.store_kompress_in_ccr", _store)
+ for source, drop in (
+ (" ".join(["alpha"] * 99 + ["nfs"]), 41),
+ (" ".join(["alpha"] * 36 + ["bureaucratic"] + ["alpha"] * 63), 36),
+ ):
+ kept = " ".join(source.split()[drop:])
+ c = _compressor(monkeypatch, enable_ccr=True, payload={"compressed": kept})
+ result = c.compress(source)
+ assert result.compressed == source
+ assert result.cache_key is None
+ assert result.compression_ratio == 1.0
+
+ big = " ".join(["alpha"] * 299 + ["nfs"])
+ kept = " ".join(big.split()[60:])
+ c = _compressor(monkeypatch, enable_ccr=True, payload={"compressed": kept})
+ result = c.compress(big)
+ marked = kept + ccr_retrieval_marker(300, 240, big, _store(big, kept, 300))
+ assert result.compressed == marked
+ assert (result.original_tokens, result.compressed_tokens) == (
+ payload_tokens(big),
+ payload_tokens(marked),
+ )
diff --git a/tests/test_transforms/test_kompress_deadline.py b/tests/test_transforms/test_kompress_deadline.py
index b420fe362..2ae94f458 100644
--- a/tests/test_transforms/test_kompress_deadline.py
+++ b/tests/test_transforms/test_kompress_deadline.py
@@ -10,6 +10,10 @@ one that leaks.
from __future__ import annotations
+import hashlib
+
+import pytest
+
from headroom.transforms import kompress_compressor as kc
@@ -32,14 +36,26 @@ def test_compress_bails_at_deadline_keeping_tail_verbatim(monkeypatch):
assert result.compressed.split() == content.split()
-def test_compress_partial_run_keeps_processed_head_plus_verbatim_tail(monkeypatch):
- # The real partial case: chunk 0 processes (gets compressed), chunk 1 trips
- # the deadline (kept verbatim). Output must be compressed-head + verbatim-tail.
- # Clock: call1=t_deadline(0); calls 2-4 are chunk-0's check+inference reads
- # (under budget); call 5+ is chunk-1's check -> trips.
- # Robust clock: jump past the deadline only AFTER chunk 0 is processed
- # (tracked via the model mock), so adding perf_counter calls inside the chunk
- # body -- e.g. sub-stage timing -- can't shift when the deadline trips.
+ ("n_words", "net_saving"),
+ [(200, True), (20, False)],
+ ids=["net-saving", "no-net-saving"],
+)
+def test_compress_partial_run_keeps_processed_head_plus_verbatim_tail(
+ monkeypatch, n_words, net_saving
+):
+ # real partial case: chunk 0 processes (gets compressed), chunk 1 trips the
+ # deadline (kept verbatim). Output must be compressed-head + verbatim-tail.
+ # Clock: call 1 = t_deadline (0); calls 2-4 chunk-0's check + inference
+ # reads (under budget); call 5+ chunk-1's check -> trips.
+ # Robust clock: jump past the deadline only AFTER chunk 0 processed
+ # (tracked via the model mock), so adding perf_counter calls inside the
+ # chunk body -- e.g. sub-stage timing -- can't shift when the deadline trips.
+ #
+ # Two sizes: at 200 words the marked partial result is smaller than the
+ # original and ships; at 20 words (150 -> 300-odd tokens either way, plus a
+ # ~43-token marker) the CCR gate finds no net saving and passes the whole
+ # payload through, which is the other half of the contract.
state = {"chunks_done": 0}
def fake_clock():
@@ -68,15 +84,27 @@ def test_compress_partial_run_keeps_processed_head_plus_verbatim_tail(monkeypatc
monkeypatch.setenv("HEADROOM_COMPRESSION_DEADLINE_MS", "20000")
comp = kc.KompressCompressor(kc.KompressConfig(min_input_words=10))
- comp.config.chunk_words = 10 # 20 words -> 2 chunks
+ comp.config.chunk_words = n_words // 2 # two chunks
monkeypatch.setattr(comp, "_should_batch_single_content", lambda *a, **k: False)
-
- words = [f"w{i}" for i in range(20)]
- out = comp.compress(" ".join(words)).compressed.split()
-
- # chunk 0 processed: first half kept (w0..w4), second half dropped (w5..w9)
- assert "w0" in out and "w4" in out
- assert "w5" not in out and "w9" not in out
- # chunk 1 tripped the deadline -> its words kept verbatim (w10..w19 all present)
- for i in range(10, 20):
+ monkeypatch.setattr(
+ comp,
+ "_store_in_ccr",
+ lambda source, *a, **k: hashlib.sha256(source.encode()).hexdigest()[:24],
+ )
+
+ words = [f"w{i}" for i in range(n_words)]
+ result = comp.compress(" ".join(words))
+ if not net_saving:
+ assert result.compressed == " ".join(words)
+ assert result.compression_ratio == 1.0
+ return
+
+ out = result.compressed.split()
+ half = n_words // 2
+ assert result.cache_key is not None and "Retrieve more" in result.compressed
+ # chunk 0 processed: its first half kept, its second half dropped
+ assert "w0" in out and f"w{half // 2 - 1}" in out
+ assert f"w{half // 2}" not in out and f"w{half - 1}" not in out
+ # chunk 1 tripped the deadline -> its words kept verbatim (all present)
+ for i in range(half, n_words):
assert f"w{i}" in out