Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(kompress): load ONNX Runtime through the dynamic loader guard before building a session (#4644)
Kenneth Wong7e827285a46b
diff --git a/crates/headroom-core/src/transforms/kompress.rs b/crates/headroom-core/src/transforms/kompress.rsindex a9826c752..cecc74387 100644--- a/crates/headroom-core/src/transforms/kompress.rs+++ b/crates/headroom-core/src/transforms/kompress.rs@@ -645,6 +645,29 @@ fn build_session(path: &Path) -> Result<Session, Box<dyn std::error::Error + SenOk(session)}+/// Whether this process can build Kompress sessions: the same CPU-then-loader+/// preconditions every `Kompress` constructor checks, in the same order. The+/// CPU must run the precompiled ONNX Runtime (AVX2 on x86), and only then is+/// the library resolved (`ORT_DYLIB_PATH`, or the pip `onnxruntime` under+/// `VIRTUAL_ENV`, `CONDA_PREFIX` or `./.venv`) and committed, once per+/// process. `Err` is `"<reason>: <detail>"`. For harnesses that should skip,+/// not fail, on a machine that can't run Kompress.+pub fn onnx_runtime_ready() -> Result<(), String> {+ onnx_runtime_ready_with(+ crate::onnx_cpu::onnx_runtime_supported_by_cpu(),+ crate::transforms::magika_detector::dynamic_ort_loader_ready,+ )+}++/// [`onnx_runtime_ready`] with the CPU verdict and loader passed in, so tests+/// can check the loader never runs on an unsupported CPU.+fn onnx_runtime_ready_with(+ cpu_ok: bool,+ loader: impl FnOnce() -> Result<(), String>,+) -> Result<(), String> {+ ort_session_gate(cpu_ok, loader).map_err(|(reason, e)| format!("{reason}: {e}"))+}+/// The tokenizer and the ONNX artifact `onnx` (e.g./// `"onnx/kompress-int8-wo.onnx"`) that the cache-only loader reads for/// `config`: the pinned snapshots in Python's hub cache, as@@ -890,6 +913,27 @@ mod tests {assert!(called.get());}+ #[test]+ fn onnx_runtime_ready_checks_the_cpu_before_the_loader() {+ // The public readiness check (used by harnesses to skip) must keep+ // the constructors' order: on an unsupported CPU it reports the CPU+ // reason without initializing the native runtime.+ let called = std::cell::Cell::new(false);+ let out = onnx_runtime_ready_with(false, || {+ called.set(true);+ Ok(())+ });+ let message = out.unwrap_err();+ assert!(message.starts_with("cpu_unsupported: "), "{message}");+ assert!(!called.get(), "loader ran although the CPU check failed");++ assert_eq!(+ onnx_runtime_ready_with(true, || Err("no dylib".into())),+ Err("ort_loader_unavailable: no dylib".to_string())+ );+ assert_eq!(onnx_runtime_ready_with(true, || Ok(())), Ok(()));+ }+/// `from_pretrained` must refuse before `Api::new()` when the operator/// air-gapped the box.///diff --git a/crates/headroom-core/tests/kompress_ort_unavailable.rs b/crates/headroom-core/tests/kompress_ort_unavailable.rsnew file mode 100644index 000000000..ce66cd2e4--- /dev/null+++ b/crates/headroom-core/tests/kompress_ort_unavailable.rs@@ -0,0 +1,78 @@+//! Kompress must return an error, not hang, when ONNX Runtime can't load.+//!+//! The crate loads ONNX Runtime dynamically (`ort-load-dynamic`). When the+//! dylib can't be loaded, `ort` 2.0.0-rc.12 deadlocks in its API-setup error+//! path (recursive `OnceLock` init) instead of returning an error, so every+//! session must go through `dynamic_ort_loader_ready` first, as magika and+//! fastembed do. Kompress used to call `Session::builder()` directly: with+//! the model in the HF cache and no loadable ONNX Runtime, the+//! `kompress_parity` test hung forever.+//!+//! This is its own test binary because the loader result is process-wide.++#![cfg(feature = "ml")]++use std::io::Write;+use std::sync::mpsc;+use std::time::Duration;++use headroom_core::transforms::kompress::{Kompress, KompressConfig, KompressError};++/// Smallest tokenizer `tokenizers` accepts, so loading gets past the+/// tokenizer and reaches the ONNX Runtime session.+const TOKENIZER_JSON: &str = r#"{+ "version": "1.0",+ "truncation": null,+ "padding": null,+ "added_tokens": [],+ "normalizer": null,+ "pre_tokenizer": null,+ "post_processor": null,+ "decoder": null,+ "model": {"type": "WordLevel", "vocab": {"[UNK]": 0}, "unk_token": "[UNK]"}+}"#;++#[test]+fn missing_onnx_runtime_is_an_error_not_a_hang() {+ let dir = tempfile::tempdir().expect("tempdir");+ let tokenizer = dir.path().join("tokenizer.json");+ std::fs::write(&tokenizer, TOKENIZER_JSON).expect("write tokenizer");+ let onnx = dir.path().join("model.onnx");+ std::fs::write(&onnx, b"never read: ONNX Runtime fails to load first").expect("write onnx");+ std::env::set_var(+ "ORT_DYLIB_PATH",+ dir.path().join("libonnxruntime-does-not-exist.dylib"),+ );++ let (tx, rx) = mpsc::channel();+ std::thread::spawn(move || {+ let _ = tx.send(Kompress::from_files(+ &tokenizer,+ &onnx,+ KompressConfig::default(),+ ));+ });++ match rx.recv_timeout(Duration::from_secs(60)) {+ Ok(Err(err @ KompressError::Onnx { .. })) => {+ // On an x86 CPU without AVX2 the CPU gate answers first, before+ // any library is loaded; either reason proves it returned.+ let message = err.to_string();+ assert!(+ message.contains("ORT_DYLIB_PATH") || message.contains("cpu_unsupported"),+ "error should name ORT_DYLIB_PATH (or the CPU gate): {message}"+ );+ }+ Ok(Err(other)) => panic!("expected KompressError::Onnx, got: {other}"),+ Ok(Ok(_)) => panic!("Kompress loaded without an ONNX Runtime library"),+ Err(_) => {+ // The stuck thread also wedges normal process exit (ort's+ // teardown waits on the same lock), so abort instead of+ // returning a failure the harness could never report. Write to+ // stderr directly: libtest's capture would swallow `eprintln!`.+ let _ = std::io::stderr()+ .write_all(b"Kompress::from_files deadlocked loading a missing ONNX Runtime\n");+ std::process::abort();+ }+ }+}diff --git a/crates/headroom-core/tests/kompress_parity.rs b/crates/headroom-core/tests/kompress_parity.rsindex f98bcb51c..207328625 100644--- a/crates/headroom-core/tests/kompress_parity.rs+++ b/crates/headroom-core/tests/kompress_parity.rs@@ -10,6 +10,14 @@//! mirroring the parity harness's "stub → Skipped" tolerance. Run it//! locally after `python scripts/record_kompress_trace.py` to get the//! real assertion.+//!+//! Runtime-gated too: with the model cached but no way to run it (a CPU+//! without AVX2, or no ONNX Runtime library: `ORT_DYLIB_PATH` unset and no+//! pip `onnxruntime` under `VIRTUAL_ENV`, `CONDA_PREFIX` or `./.venv`), it+//! also SKIPS, the way `magika_idle_threads` skips without an ONNX Runtime.+//! The check is `kompress::onnx_runtime_ready`, the constructors' own+//! CPU-then-loader preconditions, so it never loads the runtime on a CPU+//! that can't run it.use std::fs;use std::path::{Path, PathBuf};@@ -23,17 +31,27 @@ fn cached_artifacts() -> Option<(PathBuf, PathBuf)> {kompress::cached_artifacts(&KompressConfig::default(), "onnx/kompress-int8-wo.onnx")}+/// [`cached_artifacts`], or `None` (with the reason on stderr) when the+/// model isn't cached or this machine can't run ONNX Runtime.+fn loadable_artifacts() -> Option<(PathBuf, PathBuf)> {+ let Some(paths) = cached_artifacts() else {+ eprintln!(+ "SKIP: kompress model/tokenizer not in HF cache; \+ run `python scripts/record_kompress_trace.py` first"+ );+ return None;+ };+ if let Err(reason) = kompress::onnx_runtime_ready() {+ eprintln!("SKIP: can't run ONNX Runtime for kompress here: {reason}");+ return None;+ }+ Some(paths)+}+#[test]fn kompress_matches_python_fixtures_byte_for_byte() {- let (tok, onnx) = match cached_artifacts() {- Some(paths) => paths,- None => {- eprintln!(- "SKIP: kompress model/tokenizer not in HF cache; \- run `python scripts/record_kompress_trace.py` first"- );- return;- }+ let Some((tok, onnx)) = loadable_artifacts() else {+ return;};let fixtures_dir =@@ -94,9 +112,8 @@ fn kompress_matches_python_fixtures_byte_for_byte() {fn short_input_passes_through() {// Pure-logic check — no model needed. Fewer than MIN_WORDS words must// pass through unchanged regardless of model availability... but the- // engine needs a model to construct. Guard on cache like the main test.- let Some((tok, onnx)) = cached_artifacts() else {- eprintln!("SKIP: model not cached");+ // engine needs a model to construct. Guard like the main test.+ let Some((tok, onnx)) = loadable_artifacts() else {return;};let kompress = Kompress::from_files(&tok, &onnx, KompressConfig::default()).unwrap();