Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(kompress): load ONNX Runtime through the dynamic loader guard before building a session (#4644)

Kenneth Wong7e827285a46b
diff --git a/crates/headroom-core/src/transforms/kompress.rs b/crates/headroom-core/src/transforms/kompress.rs
index a9826c752..cecc74387 100644
--- a/crates/headroom-core/src/transforms/kompress.rs
+++ b/crates/headroom-core/src/transforms/kompress.rs
@@ -645,6 +645,29 @@ fn build_session(path: &Path) -> Result<Session, Box<dyn std::error::Error + Sen
Ok(session)
}
+/// Whether this process can build Kompress sessions: the same CPU-then-loader
+/// preconditions every `Kompress` constructor checks, in the same order. The
+/// CPU must run the precompiled ONNX Runtime (AVX2 on x86), and only then is
+/// the library resolved (`ORT_DYLIB_PATH`, or the pip `onnxruntime` under
+/// `VIRTUAL_ENV`, `CONDA_PREFIX` or `./.venv`) and committed, once per
+/// process. `Err` is `"<reason>: <detail>"`. For harnesses that should skip,
+/// not fail, on a machine that can't run Kompress.
+pub fn onnx_runtime_ready() -> Result<(), String> {
+ onnx_runtime_ready_with(
+ crate::onnx_cpu::onnx_runtime_supported_by_cpu(),
+ crate::transforms::magika_detector::dynamic_ort_loader_ready,
+ )
+}
+
+/// [`onnx_runtime_ready`] with the CPU verdict and loader passed in, so tests
+/// can check the loader never runs on an unsupported CPU.
+fn onnx_runtime_ready_with(
+ cpu_ok: bool,
+ loader: impl FnOnce() -> Result<(), String>,
+) -> Result<(), String> {
+ ort_session_gate(cpu_ok, loader).map_err(|(reason, e)| format!("{reason}: {e}"))
+}
+
/// The tokenizer and the ONNX artifact `onnx` (e.g.
/// `"onnx/kompress-int8-wo.onnx"`) that the cache-only loader reads for
/// `config`: the pinned snapshots in Python's hub cache, as
@@ -890,6 +913,27 @@ mod tests {
assert!(called.get());
}
+ #[test]
+ fn onnx_runtime_ready_checks_the_cpu_before_the_loader() {
+ // The public readiness check (used by harnesses to skip) must keep
+ // the constructors' order: on an unsupported CPU it reports the CPU
+ // reason without initializing the native runtime.
+ let called = std::cell::Cell::new(false);
+ let out = onnx_runtime_ready_with(false, || {
+ called.set(true);
+ Ok(())
+ });
+ let message = out.unwrap_err();
+ assert!(message.starts_with("cpu_unsupported: "), "{message}");
+ assert!(!called.get(), "loader ran although the CPU check failed");
+
+ assert_eq!(
+ onnx_runtime_ready_with(true, || Err("no dylib".into())),
+ Err("ort_loader_unavailable: no dylib".to_string())
+ );
+ assert_eq!(onnx_runtime_ready_with(true, || Ok(())), Ok(()));
+ }
+
/// `from_pretrained` must refuse before `Api::new()` when the operator
/// air-gapped the box.
///
diff --git a/crates/headroom-core/tests/kompress_ort_unavailable.rs b/crates/headroom-core/tests/kompress_ort_unavailable.rs
new file mode 100644
index 000000000..ce66cd2e4
--- /dev/null
+++ b/crates/headroom-core/tests/kompress_ort_unavailable.rs
@@ -0,0 +1,78 @@
+//! Kompress must return an error, not hang, when ONNX Runtime can't load.
+//!
+//! The crate loads ONNX Runtime dynamically (`ort-load-dynamic`). When the
+//! dylib can't be loaded, `ort` 2.0.0-rc.12 deadlocks in its API-setup error
+//! path (recursive `OnceLock` init) instead of returning an error, so every
+//! session must go through `dynamic_ort_loader_ready` first, as magika and
+//! fastembed do. Kompress used to call `Session::builder()` directly: with
+//! the model in the HF cache and no loadable ONNX Runtime, the
+//! `kompress_parity` test hung forever.
+//!
+//! This is its own test binary because the loader result is process-wide.
+
+#![cfg(feature = "ml")]
+
+use std::io::Write;
+use std::sync::mpsc;
+use std::time::Duration;
+
+use headroom_core::transforms::kompress::{Kompress, KompressConfig, KompressError};
+
+/// Smallest tokenizer `tokenizers` accepts, so loading gets past the
+/// tokenizer and reaches the ONNX Runtime session.
+const TOKENIZER_JSON: &str = r#"{
+ "version": "1.0",
+ "truncation": null,
+ "padding": null,
+ "added_tokens": [],
+ "normalizer": null,
+ "pre_tokenizer": null,
+ "post_processor": null,
+ "decoder": null,
+ "model": {"type": "WordLevel", "vocab": {"[UNK]": 0}, "unk_token": "[UNK]"}
+}"#;
+
+#[test]
+fn missing_onnx_runtime_is_an_error_not_a_hang() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let tokenizer = dir.path().join("tokenizer.json");
+ std::fs::write(&tokenizer, TOKENIZER_JSON).expect("write tokenizer");
+ let onnx = dir.path().join("model.onnx");
+ std::fs::write(&onnx, b"never read: ONNX Runtime fails to load first").expect("write onnx");
+ std::env::set_var(
+ "ORT_DYLIB_PATH",
+ dir.path().join("libonnxruntime-does-not-exist.dylib"),
+ );
+
+ let (tx, rx) = mpsc::channel();
+ std::thread::spawn(move || {
+ let _ = tx.send(Kompress::from_files(
+ &tokenizer,
+ &onnx,
+ KompressConfig::default(),
+ ));
+ });
+
+ match rx.recv_timeout(Duration::from_secs(60)) {
+ Ok(Err(err @ KompressError::Onnx { .. })) => {
+ // On an x86 CPU without AVX2 the CPU gate answers first, before
+ // any library is loaded; either reason proves it returned.
+ let message = err.to_string();
+ assert!(
+ message.contains("ORT_DYLIB_PATH") || message.contains("cpu_unsupported"),
+ "error should name ORT_DYLIB_PATH (or the CPU gate): {message}"
+ );
+ }
+ Ok(Err(other)) => panic!("expected KompressError::Onnx, got: {other}"),
+ Ok(Ok(_)) => panic!("Kompress loaded without an ONNX Runtime library"),
+ Err(_) => {
+ // The stuck thread also wedges normal process exit (ort's
+ // teardown waits on the same lock), so abort instead of
+ // returning a failure the harness could never report. Write to
+ // stderr directly: libtest's capture would swallow `eprintln!`.
+ let _ = std::io::stderr()
+ .write_all(b"Kompress::from_files deadlocked loading a missing ONNX Runtime\n");
+ std::process::abort();
+ }
+ }
+}
diff --git a/crates/headroom-core/tests/kompress_parity.rs b/crates/headroom-core/tests/kompress_parity.rs
index f98bcb51c..207328625 100644
--- a/crates/headroom-core/tests/kompress_parity.rs
+++ b/crates/headroom-core/tests/kompress_parity.rs
@@ -10,6 +10,14 @@
//! mirroring the parity harness's "stub → Skipped" tolerance. Run it
//! locally after `python scripts/record_kompress_trace.py` to get the
//! real assertion.
+//!
+//! Runtime-gated too: with the model cached but no way to run it (a CPU
+//! without AVX2, or no ONNX Runtime library: `ORT_DYLIB_PATH` unset and no
+//! pip `onnxruntime` under `VIRTUAL_ENV`, `CONDA_PREFIX` or `./.venv`), it
+//! also SKIPS, the way `magika_idle_threads` skips without an ONNX Runtime.
+//! The check is `kompress::onnx_runtime_ready`, the constructors' own
+//! CPU-then-loader preconditions, so it never loads the runtime on a CPU
+//! that can't run it.
use std::fs;
use std::path::{Path, PathBuf};
@@ -23,17 +31,27 @@ fn cached_artifacts() -> Option<(PathBuf, PathBuf)> {
kompress::cached_artifacts(&KompressConfig::default(), "onnx/kompress-int8-wo.onnx")
}
+/// [`cached_artifacts`], or `None` (with the reason on stderr) when the
+/// model isn't cached or this machine can't run ONNX Runtime.
+fn loadable_artifacts() -> Option<(PathBuf, PathBuf)> {
+ let Some(paths) = cached_artifacts() else {
+ eprintln!(
+ "SKIP: kompress model/tokenizer not in HF cache; \
+ run `python scripts/record_kompress_trace.py` first"
+ );
+ return None;
+ };
+ if let Err(reason) = kompress::onnx_runtime_ready() {
+ eprintln!("SKIP: can't run ONNX Runtime for kompress here: {reason}");
+ return None;
+ }
+ Some(paths)
+}
+
#[test]
fn kompress_matches_python_fixtures_byte_for_byte() {
- let (tok, onnx) = match cached_artifacts() {
- Some(paths) => paths,
- None => {
- eprintln!(
- "SKIP: kompress model/tokenizer not in HF cache; \
- run `python scripts/record_kompress_trace.py` first"
- );
- return;
- }
+ let Some((tok, onnx)) = loadable_artifacts() else {
+ return;
};
let fixtures_dir =
@@ -94,9 +112,8 @@ fn kompress_matches_python_fixtures_byte_for_byte() {
fn short_input_passes_through() {
// Pure-logic check — no model needed. Fewer than MIN_WORDS words must
// pass through unchanged regardless of model availability... but the
- // engine needs a model to construct. Guard on cache like the main test.
- let Some((tok, onnx)) = cached_artifacts() else {
- eprintln!("SKIP: model not cached");
+ // engine needs a model to construct. Guard like the main test.
+ let Some((tok, onnx)) = loadable_artifacts() else {
return;
};
let kompress = Kompress::from_files(&tok, &onnx, KompressConfig::default()).unwrap();