Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(kompress): add compress_batch with device-aware routing
SwiftWing21951d021f97c4
diff --git a/headroom/transforms/kompress_compressor.py b/headroom/transforms/kompress_compressor.pyindex 0da1146f9..def381477 100644--- a/headroom/transforms/kompress_compressor.py+++ b/headroom/transforms/kompress_compressor.py@@ -441,6 +441,284 @@ class KompressCompressor(Transform):logger.warning("Kompress compression failed: %s", e)return self._passthrough(content, n_words)+ def compress_batch(+ self,+ contents: list[str],+ context: str = "",+ content_type: str | None = None,+ question: str | None = None,+ target_ratio: float | list[float | None] | None = None,+ batch_size: int = 32,+ ) -> list[KompressResult]:+ """Compress multiple texts. Uses batched inference on GPU, sequential on CPU.++ On GPU (PyTorch + CUDA / MPS), runs a single batched forward pass per+ chunk batch, amortizing model inference across N texts. On CPU (ONNX+ or PyTorch), falls back to sequential ``compress()`` calls because+ ONNX Runtime's CPU provider does not parallelize across the batch+ dimension for this model (empirically 0.7-0.9x vs sequential).++ The fallback is transparent: callers get the best available+ performance per device without needing to detect the backend+ themselves.++ Measured performance (RTX 3080 Ti, ~350-word inputs):++ GPU batched vs sequential:+ N=3: 1.76x speedup+ N=5: 2.08x speedup+ N=12: 2.18x speedup+ N=24: 2.34x speedup++ CPU (ONNX, 16 logical threads): falls back to sequential;+ net effect is parity with direct ``compress()`` in a loop.++ Args:+ contents: List of texts to compress. May contain short texts or+ empty strings — those pass through without a model call.+ context: Unused (parity with ``compress``).+ content_type: Unused (parity with ``compress``).+ question: Unused (parity with ``compress``).+ target_ratio: Compression target, one of:++ * ``None`` — model decides per text (same as :meth:`compress`).+ * ``float`` — applied uniformly to every text in the batch.+ * ``list`` of ``float | None`` — per-text ratio; must match+ ``len(contents)``. ``None`` entries let the model decide for+ that text.++ batch_size: Maximum number of chunks per forward pass on the+ batched path (GPU only — ignored on CPU fallback). Default+ ``32`` is a reasonable balance for ModernBERT on GPU.++ Returns:+ List of :class:`KompressResult`, one per input text, in input order.+ Empty input returns empty list. Failed texts fall back to+ passthrough rather than raising.++ Notes:+ On the batched GPU path, scoring uses ``get_scores`` uniformly+ (threshold at 0.5 when ``target_ratio`` is ``None``). This+ matches the ONNX non-batched behavior exactly. The PyTorch+ non-batched path applies an additional borderline + span-boost+ rule, so results may differ by a small fraction of tokens on+ ``target_ratio=None`` calls via the batched path vs direct+ :meth:`compress` on PyTorch. Call :meth:`compress` directly if+ the exact PyTorch borderline behavior is required.+ """+ n = len(contents)+ if n == 0:+ return []++ # Normalize target_ratio to a per-text list+ if isinstance(target_ratio, list):+ if len(target_ratio) != n:+ raise ValueError(+ f"target_ratio list length {len(target_ratio)} does not match "+ f"contents length {n}"+ )+ ratios: list[float | None] = list(target_ratio)+ else:+ ratios = [target_ratio] * n++ # Fast path: on backends where batch-dim parallelism does NOT help+ # (ONNX CPU, PyTorch CPU), fall back to sequential `compress()`+ # internally. This keeps the public API consistent while avoiding the+ # per-item slowdown measured on ONNX CPU (~0.7-0.9x vs sequential).+ # GPU users still benefit from the batched forward pass below.+ if self._should_use_sequential_fallback():+ return [+ self.compress(+ content,+ context=context,+ content_type=content_type,+ question=question,+ target_ratio=r,+ )+ for content, r in zip(contents, ratios, strict=True)+ ]++ results: list[KompressResult | None] = [None] * n+ word_lists: list[list[str]] = [c.split() for c in contents]++ # Short texts short-circuit to passthrough — no model call needed.+ max_chunk_words = 350+ chunk_queue: list[tuple[int, int, list[str], float | None]] = []+ for i, (words, ratio) in enumerate(zip(word_lists, ratios, strict=True)):+ if len(words) < 10:+ results[i] = self._passthrough(contents[i], len(words))+ continue+ for chunk_start in range(0, len(words), max_chunk_words):+ chunk_words = words[chunk_start : chunk_start + max_chunk_words]+ chunk_queue.append((i, chunk_start, chunk_words, ratio))++ if not chunk_queue:+ # Every input was short — all passthrough, no model needed.+ return [r for r in results if r is not None]++ # Load model once for the whole batch.+ try:+ model, tokenizer = _load_kompress(self.config.device)+ except Exception as e:+ logger.warning("Kompress load failed for batch: %s — passthrough all", e)+ for i in range(n):+ if results[i] is None:+ results[i] = self._passthrough(contents[i], len(word_lists[i]))+ return [r for r in results if r is not None]++ is_onnx = _kompress_backend == "onnx"+ kept_ids_per_text: dict[int, set[int]] = {i: set() for i in range(n) if results[i] is None}++ for batch_start in range(0, len(chunk_queue), batch_size):+ batch = chunk_queue[batch_start : batch_start + batch_size]+ batch_word_lists = [c[2] for c in batch]++ try:+ return_tensors = "np" if is_onnx else "pt"+ encoding = tokenizer(+ batch_word_lists,+ is_split_into_words=True,+ truncation=True,+ max_length=512,+ padding=True,+ return_tensors=return_tensors,+ )++ input_ids = encoding["input_ids"]+ attention_mask = encoding["attention_mask"]++ if not is_onnx:+ device = next(model.parameters()).device+ input_ids = input_ids.to(device)+ attention_mask = attention_mask.to(device)++ # Single forward pass for all chunks in this batch.+ scores = model.get_scores(input_ids, attention_mask)++ for batch_idx, (text_idx, chunk_start, _chunk_words, ratio) in enumerate(batch):+ word_ids = encoding.word_ids(batch_index=batch_idx)+ score_list = scores[batch_idx] if is_onnx else scores[batch_idx].cpu()++ # Token -> word reduction (max score per word).+ word_scores: dict[int, float] = {}+ for idx, wid in enumerate(word_ids):+ if wid is None:+ continue+ s = float(score_list[idx])+ if wid not in word_scores or s > word_scores[wid]:+ word_scores[wid] = s++ if not word_scores:+ continue++ if ratio is not None:+ # Top-k by score.+ sorted_wids = sorted(+ word_scores, key=lambda w: word_scores[w], reverse=True+ )+ num_keep = max(1, int(len(sorted_wids) * ratio))+ for wid in sorted_wids[:num_keep]:+ kept_ids_per_text[text_idx].add(wid + chunk_start)+ else:+ # Threshold at 0.5 (matches ONNX get_keep_mask behavior).+ for wid, score in word_scores.items():+ if score > 0.5:+ kept_ids_per_text[text_idx].add(wid + chunk_start)++ except Exception as e:+ logger.warning(+ "Kompress batch forward pass failed: %s — passthrough affected texts", e+ )+ for text_idx, _, _, _ in batch:+ if results[text_idx] is None:+ results[text_idx] = self._passthrough(+ contents[text_idx], len(word_lists[text_idx])+ )+ kept_ids_per_text.pop(text_idx, None)++ # Reconstruct compressed text for each non-passthrough result.+ for text_idx, kept_ids in kept_ids_per_text.items():+ if results[text_idx] is not None:+ continue+ content = contents[text_idx]+ words = word_lists[text_idx]+ n_words = len(words)++ if not kept_ids:+ results[text_idx] = self._passthrough(content, n_words)+ continue++ compressed_words = [words[w] for w in sorted(kept_ids) if w < n_words]+ compressed = " ".join(compressed_words)+ compressed_count = len(compressed_words)+ comp_ratio = compressed_count / n_words if n_words else 1.0++ result = KompressResult(+ compressed=compressed,+ original=content,+ original_tokens=n_words,+ compressed_tokens=compressed_count,+ compression_ratio=comp_ratio,+ )++ if self.config.enable_ccr and comp_ratio < 0.8:+ cache_key = self._store_in_ccr(content, compressed, n_words)+ if cache_key:+ result.cache_key = cache_key+ result.compressed += (+ f"\n[{n_words} items compressed to {compressed_count}."+ f" Retrieve more: hash={cache_key}]"+ )++ results[text_idx] = result++ # Safety: every slot must be populated.+ final: list[KompressResult] = []+ for i, r in enumerate(results):+ if r is None:+ final.append(self._passthrough(contents[i], len(word_lists[i])))+ else:+ final.append(r)+ return final++ def _should_use_sequential_fallback(self) -> bool:+ """Return True if batched inference wouldn't speed up on this backend.++ Empirically measured:+ - ONNX CPU: no batch-dim parallelism; batched is 0.7-0.9x vs sequential.+ - PyTorch CPU: typically similar (conservative fallback).+ - PyTorch + CUDA: 2.0-2.3x speedup at N>=3 — use batched path.++ If the model isn't loaded yet, we trigger loading so the backend+ is known. This is a no-op if the model is already in cache.+ """+ global _kompress_model, _kompress_backend+ if _kompress_model is None:+ try:+ _load_kompress(self.config.device)+ except Exception:+ # If load fails, caller will see the error downstream.+ return True++ if _kompress_backend == "onnx":+ return True # ONNX CPU provider doesn't parallelize batch dim+ if _kompress_backend == "pytorch":+ try:+ import torch++ # Check the model's actual device+ if _kompress_model is not None and hasattr(_kompress_model, "parameters"):+ device = next(_kompress_model.parameters()).device+ if device.type == "cuda":+ return False # GPU benefits from batching+ if device.type == "mps":+ return False # MPS (Apple Silicon) also benefits+ # Fall through for CPU+ _ = torch+ except ImportError:+ return True+ return True # Conservative default: sequential+def _passthrough(self, content: str, n_words: int) -> KompressResult:return KompressResult(compressed=content,diff --git a/tests/test_transforms/test_kompress_compressor.py b/tests/test_transforms/test_kompress_compressor.pyindex 027ada422..62330c33f 100644--- a/tests/test_transforms/test_kompress_compressor.py+++ b/tests/test_transforms/test_kompress_compressor.py@@ -199,6 +199,151 @@ class TestKompressTransformInterface:assert result.messages[0]["content"] == long_text+# ── compress_batch ──────────────────────────────────────────────────────+++class TestKompressCompressorBatch:+ """Tests for the batched compression API (compress_batch).++ These exercise the non-model paths — passthrough handling, argument+ validation, order preservation, and fallback behavior on model-load+ failure. The actual batched inference path is covered by integration+ tests that require the model to be downloaded.+ """++ def test_empty_batch_returns_empty_list(self) -> None:+ from headroom.transforms.kompress_compressor import KompressCompressor++ compressor = KompressCompressor()+ result = compressor.compress_batch([])+ assert result == []++ def test_all_short_texts_passthrough_without_model(self) -> None:+ """Texts under 10 words must passthrough; model never loaded."""+ from headroom.transforms.kompress_compressor import KompressCompressor++ compressor = KompressCompressor()+ contents = ["hello", "world", "short text here"]++ with patch(+ "headroom.transforms.kompress_compressor._load_kompress",+ side_effect=AssertionError("model should not be loaded for short texts"),+ ):+ results = compressor.compress_batch(contents)++ assert len(results) == 3+ for i, r in enumerate(results):+ assert r.compressed == contents[i]+ assert r.compression_ratio == 1.0++ def test_order_preserved(self) -> None:+ """Output order must match input order even when model load fails."""+ from headroom.transforms.kompress_compressor import KompressCompressor++ compressor = KompressCompressor()+ long_texts = [+ " ".join(f"alpha{i}" for i in range(20)),+ " ".join(f"beta{i}" for i in range(20)),+ " ".join(f"gamma{i}" for i in range(20)),+ ]++ with patch(+ "headroom.transforms.kompress_compressor._load_kompress",+ side_effect=RuntimeError("no model"),+ ):+ results = compressor.compress_batch(long_texts)++ assert len(results) == 3+ assert results[0].original.startswith("alpha0")+ assert results[1].original.startswith("beta0")+ assert results[2].original.startswith("gamma0")++ def test_mixed_short_and_long_passthrough_on_model_failure(self) -> None:+ """Short texts passthrough; long texts fall back to passthrough on model failure."""+ from headroom.transforms.kompress_compressor import KompressCompressor++ compressor = KompressCompressor()+ contents = [+ "short",+ " ".join(f"word{i}" for i in range(20)), # triggers model path+ "also short",+ ]++ with patch(+ "headroom.transforms.kompress_compressor._load_kompress",+ side_effect=RuntimeError("no model"),+ ):+ results = compressor.compress_batch(contents)++ assert len(results) == 3+ assert results[0].compressed == "short"+ assert results[0].compression_ratio == 1.0+ assert results[1].compression_ratio == 1.0 # passthrough fallback+ assert results[2].compressed == "also short"++ def test_ratio_list_length_mismatch_raises(self) -> None:+ """If target_ratio is a list it must match contents length."""+ from headroom.transforms.kompress_compressor import KompressCompressor++ compressor = KompressCompressor()+ contents = ["a b c", "d e f"]++ # Too short+ try:+ compressor.compress_batch(contents, target_ratio=[0.5])+ raise AssertionError("expected ValueError for length mismatch")+ except ValueError as e:+ assert "length" in str(e).lower()++ # Too long+ try:+ compressor.compress_batch(contents, target_ratio=[0.5, 0.5, 0.5])+ raise AssertionError("expected ValueError for length mismatch")+ except ValueError as e:+ assert "length" in str(e).lower()++ def test_batch_of_one_equivalent_to_single_compress_on_short_text(self) -> None:+ """Batch-of-one with short text should produce identical passthrough."""+ from headroom.transforms.kompress_compressor import KompressCompressor++ compressor = KompressCompressor()+ text = "hello world"++ single = compressor.compress(text)+ batch = compressor.compress_batch([text])++ assert len(batch) == 1+ assert batch[0].compressed == single.compressed+ assert batch[0].compression_ratio == single.compression_ratio+ assert batch[0].original_tokens == single.original_tokens++ def test_uniform_ratio_scalar(self) -> None:+ """A scalar target_ratio must apply to every text in the batch."""+ from headroom.transforms.kompress_compressor import KompressCompressor++ compressor = KompressCompressor()+ # Short texts — passthrough regardless of ratio+ contents = ["short a", "short b", "short c"]++ results = compressor.compress_batch(contents, target_ratio=0.3)++ assert len(results) == 3+ for r, original in zip(results, contents, strict=True):+ assert r.compressed == original # short passthrough++ def test_per_item_ratio_list_with_nones(self) -> None:+ """A list of ratios with some None entries must be accepted."""+ from headroom.transforms.kompress_compressor import KompressCompressor++ compressor = KompressCompressor()+ contents = ["short a", "short b", "short c"]+ ratios: list[float | None] = [0.5, None, 0.25]++ # Short texts always passthrough; validating the list shape alone.+ results = compressor.compress_batch(contents, target_ratio=ratios)+ assert len(results) == 3++# ── unload_kompress_model ───────────────────────────────────────────────