Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(kompress): match all ONNX backends with startswith, not exact "onnx" (#2448)

Parideboyc811007f813b
diff --git a/headroom/transforms/kompress_compressor.py b/headroom/transforms/kompress_compressor.py
index 08f5d4cba..cc19ccf76 100644
--- a/headroom/transforms/kompress_compressor.py
+++ b/headroom/transforms/kompress_compressor.py
@@ -1150,7 +1150,7 @@ class KompressCompressor(Transform):
def _timed_canary(self, model: Any, tokenizer: Any, backend: str) -> float:
"""Run one small inference and return its wall-clock seconds."""
words = _canary_words()
- is_onnx = backend == "onnx"
+ is_onnx = backend.startswith("onnx")
encoding = tokenizer(
words,
is_split_into_words=True,
@@ -1251,7 +1251,7 @@ class KompressCompressor(Transform):
model, tokenizer, backend = _load_kompress(
self.config.model_id, self.config.device, allow_download=allow_download
)
- is_onnx = backend == "onnx"
+ is_onnx = backend.startswith("onnx")
device_type = _model_device_type(model, backend)
if self._should_batch_single_content(model, backend):
@@ -1604,7 +1604,7 @@ class KompressCompressor(Transform):
results[i] = self._passthrough(contents[i], len(word_lists[i]))
return [r for r in results if r is not None]
- is_onnx = backend == "onnx"
+ is_onnx = backend.startswith("onnx")
device_type = _model_device_type(model, backend)
kept_ids_per_text: dict[int, set[int]] = {i: set() for i in range(n) if results[i] is None}
inference_ms = 0.0
@@ -1830,8 +1830,8 @@ class KompressCompressor(Transform):
model, _tokenizer, backend = _kompress_cache[model_id]
- if backend == "onnx":
- return True # ONNX CPU provider doesn't parallelize batch dim
+ if backend.startswith("onnx"):
+ return True # ONNX EPs don't parallelize the batch dim
if backend == "pytorch":
try:
import torch
diff --git a/tests/test_transforms/test_kompress_compressor.py b/tests/test_transforms/test_kompress_compressor.py
index 826a3014b..f782b8f2a 100644
--- a/tests/test_transforms/test_kompress_compressor.py
+++ b/tests/test_transforms/test_kompress_compressor.py
@@ -485,3 +485,57 @@ class TestUnloadKompressModel:
# Should return False when no model is loaded
assert unload_kompress_model() is False
+
+
+# ── onnx_coreml backend gating (issue #2442) ────────────────────────────
+
+
+class TestOnnxBackendPrefixGating:
+ """Non-CPU ONNX backends (onnx_coreml, onnx_cpu) must take the ONNX path.
+
+ The bug: sites gated on the exact string ``backend == "onnx"`` misclassified
+ ``onnx_coreml`` as PyTorch and called ``next(model.parameters())`` on the
+ ``_OnnxModel`` wrapper, which has no ``.parameters()`` — crashing every call
+ and silently disabling Kompress. The fix uses ``backend.startswith("onnx")``.
+ """
+
+ class _FakeOnnxModel:
+ """Mimics the ONNX wrapper: has get_keep_mask but no .parameters()."""
+
+ def get_keep_mask(self, input_ids, attention_mask): # noqa: ANN001, ANN201
+ return [[True]]
+
+ @staticmethod
+ def _fake_tokenizer(words, **kwargs): # noqa: ANN001, ANN205
+ # ONNX path must request numpy tensors, never torch.
+ assert kwargs.get("return_tensors") == "np"
+ return {"input_ids": [[1, 2]], "attention_mask": [[1, 1]]}
+
+ def test_timed_canary_onnx_coreml_skips_pytorch_device_dispatch(self) -> None:
+ from headroom.transforms.kompress_compressor import KompressCompressor
+
+ compressor = KompressCompressor()
+ model = self._FakeOnnxModel() # no .parameters()
+
+ # Must not raise AttributeError: '_OnnxModel' object has no attribute
+ # 'parameters'; returns a float wall-clock duration.
+ elapsed = compressor._timed_canary(model, self._fake_tokenizer, "onnx_coreml")
+ assert isinstance(elapsed, float)
+
+ def test_timed_canary_pytorch_still_dispatches_to_device(self) -> None:
+ # Negative control: the PyTorch branch DOES touch .parameters(), so the
+ # paramless fake model raises there — proving the test above is only
+ # green because onnx_coreml correctly skips that branch.
+ import pytest
+
+ from headroom.transforms.kompress_compressor import KompressCompressor
+
+ compressor = KompressCompressor()
+ model = self._FakeOnnxModel()
+
+ def pt_tokenizer(words, **kwargs): # noqa: ANN001, ANN202
+ assert kwargs.get("return_tensors") == "pt"
+ return {"input_ids": [[1, 2]], "attention_mask": [[1, 1]]}
+
+ with pytest.raises(AttributeError):
+ compressor._timed_canary(model, pt_tokenizer, "pytorch")