Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat: switch Kompress default to kompress-v2-base with weight-only int8 ONNX (#799)
Tejas Chopra74392b238e4f
diff --git a/.gitignore b/.gitignoreindex 7adbfffe6..1edd20517 100644--- a/.gitignore+++ b/.gitignore@@ -3,6 +3,10 @@.fastembed_cache/**/.fastembed_cache/+# Local Kompress ONNX export artifacts (scripts/export_kompress_v2_onnx.py).+# Hundreds of MB each — published to HuggingFace, never committed.+/onnx/+# Private scripts (contain credentials). Allowlist checked-in helpers below.scripts/!scripts/@@ -25,6 +29,7 @@ scripts/*!scripts/refresh_model_limits.sh!scripts/audit_wheel_glibc_symbols.py!scripts/replay_codex_ws_load.py+!scripts/export_kompress_v2_onnx.py# Rust / Cargo build artifacts/target/diff --git a/README.md b/README.mdindex bb3333eda..06ac86443 100644--- a/README.md+++ b/README.md@@ -15,7 +15,7 @@<a href="https://app.codecov.io/gh/chopratejas/headroom"><img src="https://codecov.io/gh/chopratejas/headroom/graph/badge.svg" alt="codecov"></a><a href="https://pypi.org/project/headroom-ai/"><img src="https://img.shields.io/pypi/v/headroom-ai.svg" alt="PyPI"></a><a href="https://www.npmjs.com/package/headroom-ai"><img src="https://img.shields.io/npm/v/headroom-ai.svg" alt="npm"></a>- <a href="https://huggingface.co/chopratejas/kompress-base"><img src="https://img.shields.io/badge/model-Kompress--base-yellow.svg" alt="Model: Kompress-base"></a>+ <a href="https://huggingface.co/chopratejas/kompress-v2-base"><img src="https://img.shields.io/badge/model-Kompress--v2--base-yellow.svg" alt="Model: Kompress-v2-base"></a><a href="LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue.svg" alt="License: Apache 2.0"></a><a href="https://headroom-docs.vercel.app/docs"><img src="https://img.shields.io/badge/docs-online-blue.svg" alt="Docs"></a></p>@@ -81,7 +81,7 @@ Headroom compresses everything your AI agent reads — tool outputs, logs, RAG c- **CacheAligner** — stabilizes prefixes so provider KV caches actually hit- **CCR** — stores originals locally; LLM calls `headroom_retrieve` if it needs them-→ [Architecture](https://headroom-docs.vercel.app/docs/architecture) · [CCR reversible compression](https://headroom-docs.vercel.app/docs/ccr) · [Kompress-base model card](https://huggingface.co/chopratejas/kompress-base)+→ [Architecture](https://headroom-docs.vercel.app/docs/architecture) · [CCR reversible compression](https://headroom-docs.vercel.app/docs/ccr) · [Kompress-v2-base model card](https://huggingface.co/chopratejas/kompress-v2-base)## Get started (60 seconds)@@ -283,7 +283,7 @@ Devcontainers in `.devcontainer/` (default + `memory-stack` with Qdrant & Neo4j)## Community- **[Discord](https://discord.gg/yRmaUNpsPJ)** — questions, feedback, war stories.-- **[Kompress-base on HuggingFace](https://huggingface.co/chopratejas/kompress-base)** — the model behind our text compression.+- **[Kompress-v2-base on HuggingFace](https://huggingface.co/chopratejas/kompress-v2-base)** — the model behind our text compression.## Licensediff --git a/headroom/compress.py b/headroom/compress.pyindex 657c0926a..f34865ec8 100644--- a/headroom/compress.py+++ b/headroom/compress.py@@ -128,7 +128,7 @@ class CompressConfig:# Model variantkompress_model: str | None = None- """Kompress model ID. None = default (chopratejas/kompress-base).+ """Kompress model ID. None = default (chopratejas/kompress-v2-base).Set to a HuggingFace model ID for domain-specific compression.Set to 'disabled' to skip ML compression entirely(only SmartCrusher + CacheAligner will run)."""diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.pyindex 6225f409a..6960d7692 100644--- a/headroom/transforms/content_router.py+++ b/headroom/transforms/content_router.py@@ -1481,7 +1481,7 @@ class ContentRouter(Transform):compressed: str | None = Nonecompressed_tokens: int | None = None- # Primary: Kompress — downloads from chopratejas/kompress-base on first use+ # Primary: Kompress — downloads from chopratejas/kompress-v2-base on first useif self.config.enable_kompress:compressor = self._get_kompress()if compressor:@@ -1695,7 +1695,7 @@ class ContentRouter(Transform):"""Get KompressCompressor (lazy load). Downloads from HuggingFace on first use.Respects runtime kompress_model kwarg:- - None: use default (chopratejas/kompress-base) — cached on self+ - None: use default (chopratejas/kompress-v2-base) — cached on self- "disabled": return None (skip ML compression entirely)- any model ID string: create compressor with that model(model weights are cached at module level in kompress_compressor.py,diff --git a/headroom/transforms/kompress_compressor.py b/headroom/transforms/kompress_compressor.pyindex c199cce0d..ce0e78cfd 100644--- a/headroom/transforms/kompress_compressor.py+++ b/headroom/transforms/kompress_compressor.py@@ -1,6 +1,6 @@"""Kompress: ModernBERT token compressor for structured tool outputs.-Auto-downloads the model from HuggingFace (chopratejas/kompress-base)+Auto-downloads the model from HuggingFace (chopratejas/kompress-v2-base)on first use.Requires the [ml] extra: pip install headroom-ai[ml]@@ -36,8 +36,28 @@ from .base import Transformlogger = logging.getLogger(__name__)# Default HuggingFace model ID-HF_MODEL_ID = "chopratejas/kompress-base"+HF_MODEL_ID = "chopratejas/kompress-v2-base"KOMPRESS_BACKEND_ENV = "HEADROOM_KOMPRESS_BACKEND"+KOMPRESS_ONNX_FILENAME_ENV = "HEADROOM_KOMPRESS_ONNX_FILENAME"++# ONNX artifacts are resolved against the model repo in this order, falling+# through on download miss OR session-load failure:+#+# - kompress-int8-wo.onnx: weight-only int8 (MatMulNBits), 261MB. Evaluated on+# the labeled dataset_v2 test split (n=500): f1=0.9130 vs fp32's 0.9128,+# must_keep_recall 0.9765 vs 0.9770, keep_rate 0.8097 vs 0.8100, 99.6%+# keep-decision agreement — fp32-equivalent at 2.2x less memory. Uses the+# com.microsoft MatMulNBits contrib op; older onnxruntime builds without the+# 8-bit kernel fail at session load and fall through to fp32.+# - kompress-fp32.onnx: lossless reference, 601MB.+# - kompress-int8.onnx: v1-era dynamic int8 (kept for custom domain repos).+#+# An operator can pin an exact file via HEADROOM_KOMPRESS_ONNX_FILENAME.+_DEFAULT_ONNX_FILENAMES = (+ "onnx/kompress-int8-wo.onnx",+ "onnx/kompress-fp32.onnx",+ "onnx/kompress-int8.onnx",+)KOMPRESS_ONNX_INTRA_THREADS_ENV = "HEADROOM_KOMPRESS_ONNX_INTRA_THREADS"KOMPRESS_ONNX_INTER_THREADS_ENV = "HEADROOM_KOMPRESS_ONNX_INTER_THREADS"KOMPRESS_COREML_CACHE_DIR_ENV = "HEADROOM_KOMPRESS_COREML_CACHE_DIR"@@ -320,12 +340,56 @@ class _OnnxModel:return (np.array(scores) > 0.5).tolist()+def _onnx_filename_candidates() -> tuple[str, ...]:+ """ONNX repo paths to try, honoring an optional exact-file override."""+ override = os.environ.get(KOMPRESS_ONNX_FILENAME_ENV, "").strip()+ if override:+ # Put the override first but keep the defaults as a safety net.+ return (override, *(f for f in _DEFAULT_ONNX_FILENAMES if f != override))+ return _DEFAULT_ONNX_FILENAMES+++def _create_onnx_session(model_id: str, ort: Any, providers: list[Any]) -> Any:+ """Resolve and load the model's ONNX artifact, trying candidates in order.++ A candidate is skipped on download miss (file not in the repo) or on+ session-load failure (e.g. the weight-only int8 artifact uses the+ MatMulNBits contrib op, which old onnxruntime builds can't run — those+ installs fall through to the fp32 artifact instead of losing Kompress).+ """+ last_err: Exception | None = None+ for filename in _onnx_filename_candidates():+ try:+ onnx_path = hf_hub_download_local_first(model_id, filename)+ except Exception as exc:+ last_err = exc+ logger.debug("ONNX artifact %r not in %s: %s", filename, model_id, exc)+ continue+ try:+ return ort.InferenceSession(+ onnx_path,+ _onnx_session_options(ort),+ providers=providers,+ )+ except Exception as exc:+ last_err = exc+ logger.warning(+ "ONNX artifact %r from %s failed to load (%s); trying next candidate",+ filename,+ model_id,+ exc,+ )+ raise FileNotFoundError(+ f"No loadable ONNX artifact in {model_id}; tried {_onnx_filename_candidates()}"+ ) from last_err++def _load_kompress_onnx(model_id: str,*,use_coreml: bool = False,) -> tuple[Any, Any, str]:- """Download ONNX INT8 model from HuggingFace and load with onnxruntime."""+ """Download the ONNX model from HuggingFace and load with onnxruntime."""import onnxruntime as ortfrom transformers import AutoTokenizer@@ -335,8 +399,6 @@ def _load_kompress_onnx(logger.info("Downloading Kompress ONNX model from %s ...", model_id)- onnx_path = hf_hub_download_local_first(model_id, "onnx/kompress-int8.onnx")-backend = "onnx_coreml" if use_coreml else "onnx"providers: list[Any]if use_coreml:@@ -364,16 +426,12 @@ def _load_kompress_onnx(else:providers = ["CPUExecutionProvider"]- session = ort.InferenceSession(- onnx_path,- _onnx_session_options(ort),- providers=providers,- )+ session = _create_onnx_session(model_id, ort, providers)model = _OnnxModel(session)tokenizer = AutoTokenizer.from_pretrained("answerdotai/ModernBERT-base")_kompress_cache[model_id] = (model, tokenizer, backend)- logger.info("Kompress ONNX INT8 loaded: %s backend=%s", model_id, backend)+ logger.info("Kompress ONNX loaded: %s backend=%s", model_id, backend)return model, tokenizer, backend@@ -533,7 +591,7 @@ class KompressConfig:The model_id, chunk_words, and score_threshold are coupled: a modeltrained on 50-word chunks needs chunk_words=50 at inference. The- defaults match kompress-base. For domain-specific models, set all three.+ defaults match kompress-v2-base. For domain-specific models, set all three.Example — financial documents::diff --git a/scripts/export_kompress_v2_onnx.py b/scripts/export_kompress_v2_onnx.pynew file mode 100644index 000000000..383f473db--- /dev/null+++ b/scripts/export_kompress_v2_onnx.py@@ -0,0 +1,267 @@+#!/usr/bin/env python+"""Export a Kompress PyTorch checkpoint to ONNX INT8 for Headroom's light path.++Why this exists+---------------+Headroom's ``[proxy]`` extra ships ``onnxruntime`` but **not** torch — the+proxy runs Kompress text compression on ONNX Runtime alone. The loader+(``headroom/transforms/kompress_compressor.py``) downloads+``onnx/kompress-int8.onnx`` from the model repo and runs it through+``_OnnxModel``, which expects a single graph output named ``final_scores``+(per-token importance in ``[0, 1]``, kept when ``> 0.5``).++``chopratejas/kompress-v2-base`` ships only PyTorch weights+(``model.safetensors`` / ``merged.pt``) — no ONNX. So pointing Headroom at v2+without an ONNX export would silently force the heavier ``[ml]`` (torch) path+on every proxy install. This script reproduces v1's exact ONNX contract from+the v2 PyTorch checkpoint, so a default swap stays zero-cost for light installs.++The model is a *custom* dual-head ModernBERT (token classifier + span CNN), not+a standard HF architecture, so ``optimum-cli export onnx`` does not apply — we+trace the real module from ``kompress_compressor._get_model_class()``.++Requires+--------+ pip install headroom-ai[ml] onnxruntime # torch + transformers + onnxruntime++Usage+-----+ # Convert + verify locally (writes onnx/kompress-int8.onnx):+ python scripts/export_kompress_v2_onnx.py --model-id chopratejas/kompress-v2-base++ # Convert, verify, and upload back to the HF repo (needs `huggingface-cli login`):+ python scripts/export_kompress_v2_onnx.py --model-id chopratejas/kompress-v2-base --upload+"""++from __future__ import annotations++import argparse+import logging+import sys+from pathlib import Path++logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")+logger = logging.getLogger("export_kompress_v2_onnx")++# ModernBERT encoder + tokenizer base (must match training and the loader).+BASE_MODEL = "answerdotai/ModernBERT-base"+DEFAULT_MODEL_ID = "chopratejas/kompress-v2-base"+++def _build_core(model_id: str):+ """Instantiate HeadroomCompressorModel and load the merged v2 weights.++ The v2 repo's ``model.safetensors`` is the *unmerged* PEFT structure+ (``encoder.base_model.model...`` with separate ``base_layer`` + LoRA+ adapters), which does not map onto ``HeadroomCompressorModel``. The+ canonical artifact is ``merged.pt`` — a structured checkpoint with already+ LoRA-merged sub-state-dicts:++ {"encoder_state_dict", "token_head_state_dict",+ "span_conv_state_dict", "config", "checkpoint_kind"}++ Each loads cleanly (0 missing / 0 unexpected) into the encoder + heads.+ """+ import torch+ from huggingface_hub import hf_hub_download++ from headroom.transforms.kompress_compressor import _get_model_class++ ckpt_path = hf_hub_download(model_id, "merged.pt")+ ckpt = torch.load(ckpt_path, map_location="cpu")+ for key in ("encoder_state_dict", "token_head_state_dict", "span_conv_state_dict"):+ if key not in ckpt:+ raise RuntimeError(+ f"merged.pt missing '{key}'. Found: {sorted(ckpt)}. "+ "This script targets the v2 'merged' checkpoint format."+ )++ core = _get_model_class()(model_name=BASE_MODEL)++ def _strict_load(module, sd, label: str) -> None:+ missing, unexpected = module.load_state_dict(sd, strict=False)+ if missing or unexpected:+ raise RuntimeError(+ f"{label}: state_dict mismatch (missing={list(missing)[:5]}, "+ f"unexpected={list(unexpected)[:5]}). Architecture drifted from the checkpoint."+ )+ logger.info(" %s loaded (%d tensors, exact match)", label, len(sd))++ logger.info("Loading merged.pt (checkpoint_kind=%s)", ckpt.get("checkpoint_kind"))+ _strict_load(core.encoder, ckpt["encoder_state_dict"], "encoder")+ _strict_load(core.token_head, ckpt["token_head_state_dict"], "token_head")+ _strict_load(core.span_conv, ckpt["span_conv_state_dict"], "span_conv")++ core.eval()+ return core+++def _export_wrapper(core):+ """Wrap the dual head so forward() returns `final_scores` (== get_scores)."""+ import torch+ import torch.nn as nn++ class ExportWrapper(nn.Module):+ def __init__(self, inner):+ super().__init__()+ self.inner = inner++ def forward(self, input_ids, attention_mask): # noqa: ANN001+ hidden = self.inner.encoder(input_ids, attention_mask=attention_mask).last_hidden_state+ token_probs = torch.softmax(self.inner.token_head(hidden), dim=-1)[:, :, 1]+ span_scores = self.inner.span_conv(hidden.transpose(1, 2)).squeeze(1)+ return token_probs * (0.5 + 0.5 * span_scores)++ return ExportWrapper(core).eval()+++def export(model_id: str, out_path: Path, opset: int, precision: str) -> None:+ import numpy as np+ import torch++ core = _build_core(model_id)+ wrapper = _export_wrapper(core)++ out_path.parent.mkdir(parents=True, exist_ok=True)++ # fp32 path: trace straight to the final artifact (lossless — verified 100%+ # keep-decision agreement with PyTorch). int8 path: trace to a temp fp32+ # graph, then dynamically quantize into the final artifact.+ trace_target = out_path if precision == "fp32" else out_path.with_name("kompress-fp32-tmp.onnx")++ dummy_ids = torch.randint(0, 1000, (1, 64), dtype=torch.long)+ dummy_mask = torch.ones((1, 64), dtype=torch.long)++ logger.info("Tracing → ONNX (opset %d, precision=%s) ...", opset, precision)+ with torch.no_grad():+ torch.onnx.export(+ wrapper,+ (dummy_ids, dummy_mask),+ str(trace_target),+ input_names=["input_ids", "attention_mask"],+ output_names=["final_scores"],+ dynamic_axes={+ "input_ids": {0: "batch", 1: "seq"},+ "attention_mask": {0: "batch", 1: "seq"},+ "final_scores": {0: "batch", 1: "seq"},+ },+ opset_version=opset,+ do_constant_folding=True,+ dynamo=False,+ )++ if precision == "int8":+ from onnxruntime.quantization import QuantType, quantize_dynamic++ logger.info("INT8 dynamic quantization (MatMul only) → %s", out_path)+ # Restrict to MatMul: the encoder's linear layers carry ~all the weight+ # mass and ORT's CPU provider implements MatMulInteger. Quantizing the+ # tiny span_conv Conv1d layers would emit ConvInteger, which ORT CPU+ # cannot run. per_channel recovers transformer accuracy at the 0.5 boundary.+ quantize_dynamic(+ str(trace_target),+ str(out_path),+ weight_type=QuantType.QInt8,+ op_types_to_quantize=["MatMul"],+ per_channel=True,+ )+ trace_target.unlink(missing_ok=True)++ _verify(model_id, core, out_path, np, torch)+++def _verify(model_id: str, core, out_path: Path, np, torch) -> None:+ """Compare ONNX scores against PyTorch get_scores on a real tokenized sample."""+ import onnxruntime as ort+ from transformers import AutoTokenizer++ tok = AutoTokenizer.from_pretrained(BASE_MODEL)+ sample = (+ "The proxy compresses tool outputs before they reach the model. "+ "Errors and stack traces should survive; boilerplate should not. "+ ) * 6+ words = sample.split()+ enc = tok(+ words,+ is_split_into_words=True,+ truncation=True,+ max_length=512,+ padding=True,+ return_tensors="pt",+ )++ with torch.no_grad():+ torch_scores = core.get_scores(enc["input_ids"], enc["attention_mask"])[0].cpu().numpy()++ sess = ort.InferenceSession(str(out_path), providers=["CPUExecutionProvider"])+ onnx_scores = sess.run(+ ["final_scores"],+ {+ "input_ids": enc["input_ids"].numpy().astype(np.int64),+ "attention_mask": enc["attention_mask"].numpy().astype(np.int64),+ },+ )[0][0]++ max_abs = float(np.max(np.abs(torch_scores - onnx_scores)))+ keep_torch = torch_scores > 0.5+ keep_onnx = onnx_scores > 0.5+ agree = float((keep_torch == keep_onnx).mean())+ logger.info(+ "Verify: max|Δscore|=%.4f keep-decision agreement=%.1f%% (fp32 ~100%%, int8 ~98-100%%)",+ max_abs,+ agree * 100,+ )+ if agree < 0.98:+ logger.warning(+ "Keep-decision agreement below 98%% — for fp32 this means a tracing "+ "problem; for int8 consider per_channel/fp32. Inspect before publishing."+ )+++def upload(model_id: str, out_path: Path) -> None:+ from huggingface_hub import upload_file++ # Publish under onnx/<artifact filename> so int8 and fp32 can coexist.+ repo_path = f"onnx/{out_path.name}"+ logger.info("Uploading %s → %s:%s", out_path, model_id, repo_path)+ upload_file(+ path_or_fileobj=str(out_path),+ path_in_repo=repo_path,+ repo_id=model_id,+ commit_message="Add ONNX export for Headroom lightweight (no-torch) path",+ )+ logger.info("Uploaded. Headroom's ONNX loader will now find it on next cold start.")+++def main() -> int:+ ap = argparse.ArgumentParser(description=__doc__)+ ap.add_argument("--model-id", default=DEFAULT_MODEL_ID)+ ap.add_argument(+ "--precision",+ choices=["fp32", "int8"],+ default="fp32",+ help="fp32 = lossless, larger artifact. int8 = ~2x smaller, tiny accuracy cost.",+ )+ ap.add_argument(+ "--out",+ type=Path,+ default=None,+ help="Local output path. Defaults to onnx/kompress-<precision>.onnx.",+ )+ ap.add_argument("--opset", type=int, default=17)+ ap.add_argument(+ "--upload",+ action="store_true",+ help="Upload to the HF repo under onnx/<filename> (needs HF write auth).",+ )+ args = ap.parse_args()++ out_path = args.out or Path(f"onnx/kompress-{args.precision}.onnx")+ export(args.model_id, out_path, args.opset, args.precision)+ if args.upload:+ upload(args.model_id, out_path)+ return 0+++if __name__ == "__main__":+ sys.exit(main())