Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

chore(rust): port Magika detection (Stage 3d PR3 — Tier 1)

chopratejasd34658b22ef2
diff --git a/Cargo.lock b/Cargo.lock
index 7600b900a..40a038430 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1326,6 +1326,7 @@ dependencies = [
"fastembed",
"flate2",
"hf-hub 0.4.3",
+ "magika",
"md-5",
"proptest",
"regex",
@@ -1987,6 +1988,18 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30"
+[[package]]
+name = "magika"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3aee5ecdbd182547ca3dfcd74c5bcd7f8c57384ad03cb79ef6e3bdf8d56abcdf"
+dependencies = [
+ "ndarray",
+ "ort",
+ "thiserror 1.0.69",
+ "tokio",
+]
+
[[package]]
name = "matchers"
version = "0.2.0"
diff --git a/crates/headroom-core/Cargo.toml b/crates/headroom-core/Cargo.toml
index 1039439c6..39b5976f4 100644
--- a/crates/headroom-core/Cargo.toml
+++ b/crates/headroom-core/Cargo.toml
@@ -54,6 +54,13 @@ flate2 = "1"
# Same library + same model = byte-equal embeddings between Python and Rust
# (both call into ONNX Runtime over the identical ONNX file).
fastembed = "5"
+# `magika` is Google's ONNX-backed content classifier (Tier 1 of the
+# Stage-3d ContentRouter detection arch). Bundled standard-model is
+# loaded once per process via `OnceLock` and shared across calls. The
+# crate depends on `ort`, which is already in our dep tree via
+# `fastembed`, so adding it doesn't pull a new ML runtime — both
+# crates share the ONNX Runtime singleton.
+magika = "1"
[dev-dependencies]
proptest = "1"
diff --git a/crates/headroom-core/src/transforms/magika_detector.rs b/crates/headroom-core/src/transforms/magika_detector.rs
new file mode 100644
index 000000000..614911d6d
--- /dev/null
+++ b/crates/headroom-core/src/transforms/magika_detector.rs
@@ -0,0 +1,344 @@
+//! Magika-based content detection (Stage 3d Tier 1).
+//!
+//! Wraps Google's [`magika`] crate — an ONNX-backed content classifier —
+//! and maps its 200+ labels onto Headroom's existing
+//! [`crate::transforms::content_detector::ContentType`] enum so the
+//! ContentRouter dispatch (PR5) can stay enum-stable.
+//!
+//! # Design
+//!
+//! - **Singleton session.** Magika model loading is the expensive part
+//! (one-time ONNX init, ~50 ms cold). We do it exactly once per
+//! process via [`OnceLock`]. The `Session` requires `&mut self`
+//! for inference, so the singleton wraps it in a `Mutex` — fine for
+//! our throughput; if benchmarks show contention later we'll pool.
+//!
+//! - **Loud failures.** If the model fails to load or inference fails,
+//! `magika_detect` returns `Err`. The ContentRouter (PR5) decides
+//! whether to fall back to Tier 2 (`unidiff-rs`) or surface to the
+//! caller. We deliberately do **not** silently return `PlainText` on
+//! error — that's the kind of silent fallback the audit doc forbids.
+//!
+//! - **Mapping table is explicit.** Every magika label we care about
+//! has an explicit case in [`map_magika_label`]; everything else
+//! falls into [`ContentType::PlainText`]. This is a code-route
+//! decision, not a regex — adding a new mapping is one line, and
+//! readers can audit the dispatch in one screen.
+//!
+//! - **No router rewiring here.** PR3 lands the detector + tests
+//! only. PR5 flips the ContentRouter to call us instead of the
+//! regex-based [`crate::transforms::content_detector`].
+
+use std::sync::{Mutex, OnceLock};
+
+use magika::Session;
+use thiserror::Error;
+
+use crate::transforms::content_detector::ContentType;
+
+/// Errors from the magika detector. Wraps the underlying `magika::Error`
+/// so callers can match on whether init or inference broke without
+/// pulling magika types into their imports.
+#[derive(Debug, Error)]
+pub enum MagikaDetectorError {
+ /// One-time session initialization failed (model load, ONNX init).
+ /// Once we hit this, every subsequent call also fails — there is no
+ /// retry path here. The router should surface and stop.
+ #[error("magika session init failed: {0}")]
+ Init(String),
+
+ /// Inference call failed for this input. Usually transient; future
+ /// calls may succeed. The error message is the magika-side text;
+ /// we don't try to wrap it.
+ #[error("magika inference failed: {0}")]
+ Inference(String),
+
+ /// Singleton lock was poisoned (a previous holder panicked while
+ /// holding it). The detector is unusable until the process
+ /// restarts. We don't auto-recover — a panicked detector means
+ /// something is corrupt and continuing would mask it.
+ #[error("magika session lock poisoned")]
+ Poisoned,
+}
+
+/// One-process singleton holding the magika session. Lazily
+/// initialized on first call to [`magika_detect`].
+///
+/// `Mutex<Result<Session, ...>>` rather than `Result<Mutex<Session>>`
+/// so init failure is recorded once and replayed cheaply on every
+/// subsequent call (no re-attempting the load — if the model file is
+/// missing or ort can't init, retrying just wastes cycles).
+static MAGIKA_SESSION: OnceLock<Mutex<Result<Session, String>>> = OnceLock::new();
+
+fn session() -> &'static Mutex<Result<Session, String>> {
+ MAGIKA_SESSION.get_or_init(|| Mutex::new(Session::new().map_err(|e| e.to_string())))
+}
+
+/// Classify `content` and return the mapped Headroom [`ContentType`].
+///
+/// Empty input shortcuts to [`ContentType::PlainText`] without touching
+/// the model — saves the round trip on every empty tool result.
+pub fn magika_detect(content: &str) -> Result<ContentType, MagikaDetectorError> {
+ if content.is_empty() {
+ return Ok(ContentType::PlainText);
+ }
+
+ let mutex = session();
+ let mut guard = mutex.lock().map_err(|_| MagikaDetectorError::Poisoned)?;
+ let session = guard
+ .as_mut()
+ .map_err(|e| MagikaDetectorError::Init(e.clone()))?;
+
+ let bytes = content.as_bytes();
+ let file_type = session
+ .identify_content_sync(bytes)
+ .map_err(|e| MagikaDetectorError::Inference(e.to_string()))?;
+
+ Ok(map_magika_label(file_type.info().label))
+}
+
+/// Map a magika label string to Headroom's [`ContentType`] enum.
+///
+/// **Why explicit cases instead of `group == "code"`:** magika's
+/// `group` field is a coarse bucket ("code", "text", "binary",
+/// "executable", ...). Some entries we want — like `markdown`,
+/// `txt`, `latex` — are in the `text` group along with formats we
+/// route differently. So we case on the label directly: clear, one
+/// match arm per decision, no group-vs-label semantic confusion.
+///
+/// **Unmapped labels return [`ContentType::PlainText`]**, the safest
+/// default — passthrough at the router level rather than misroute to
+/// a wrong compressor. PR5 will refine this for `SearchResults` /
+/// `BuildOutput` (which magika has no equivalent for).
+pub fn map_magika_label(label: &str) -> ContentType {
+ match label {
+ // ── JSON ───────────────────────────────────────────────────
+ // PR5 will refine this with the existing `is_json_array_of_dicts`
+ // check — magika says "this is JSON" but doesn't tell us if it's
+ // an array of records vs. a single object. For PR3 the mapping
+ // exists; the refinement is a router concern.
+ "json" | "jsonl" => ContentType::JsonArray,
+
+ // ── Diffs ──────────────────────────────────────────────────
+ "diff" => ContentType::GitDiff,
+
+ // ── HTML ───────────────────────────────────────────────────
+ "html" | "xml" => ContentType::Html,
+
+ // ── Source code ────────────────────────────────────────────
+ // The big "code" group from magika. We list the labels we
+ // actually expect to see in tool outputs / pasted code in
+ // proxy traffic. Anything else in the code group falls
+ // through to PlainText — better passthrough than misroute.
+ "rust" | "python" | "javascript" | "typescript" | "go" | "java" | "c" | "cpp" | "cs"
+ | "php" | "ruby" | "swift" | "kotlin" | "scala" | "haskell" | "lua" | "dart" | "perl"
+ | "shell" | "powershell" | "batch" | "sql" | "css" | "vue" | "groovy" | "clojure"
+ | "asm" | "cmake" | "dockerfile" | "makefile" | "yaml" | "toml" | "ini" | "hcl"
+ | "jinja" => ContentType::SourceCode,
+
+ // ── Plain text-ish ─────────────────────────────────────────
+ // markdown, rst, latex, log-style, txt, empty/unknown all
+ // route as plain text. The router won't try to compress these
+ // with a code-aware compressor.
+ "markdown" | "rst" | "latex" | "txt" | "empty" | "unknown" | "undefined" => {
+ ContentType::PlainText
+ }
+
+ // ── Default: passthrough ───────────────────────────────────
+ _ => ContentType::PlainText,
+ }
+}
+
+// ─── Tests ─────────────────────────────────────────────────────────────
+//
+// These tests are integration-y — they hit the real magika model, which
+// loads ONNX on first call (~50 ms cold). Total wall-clock for the full
+// suite is dominated by that one-time load, so we keep cases compact.
+//
+// Detection is probabilistic; we assert against `ContentType` enum
+// values rather than confidence scores or labels directly. If magika's
+// model version changes (`MODEL_NAME` in their crate), individual
+// label assignments may shift but our `match` arms are wide enough to
+// stay stable.
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn assert_detect(content: &str, expected: ContentType, hint: &str) {
+ match magika_detect(content) {
+ Ok(got) => assert_eq!(got, expected, "{hint}: expected {expected:?}, got {got:?}"),
+ Err(e) => panic!("{hint}: detection failed: {e}"),
+ }
+ }
+
+ #[test]
+ fn empty_input_is_plain_text_without_model_call() {
+ // The shortcut path — should not touch the model.
+ let result = magika_detect("").unwrap();
+ assert_eq!(result, ContentType::PlainText);
+ }
+
+ #[test]
+ fn detects_json() {
+ assert_detect(
+ r#"{"name": "Alice", "age": 30, "tags": ["a", "b"]}"#,
+ ContentType::JsonArray,
+ "single-object JSON",
+ );
+ }
+
+ #[test]
+ fn detects_json_array() {
+ let payload = r#"[{"id": 1, "v": "a"}, {"id": 2, "v": "b"}, {"id": 3, "v": "c"}]"#;
+ assert_detect(payload, ContentType::JsonArray, "array-of-records JSON");
+ }
+
+ #[test]
+ fn detects_python_source() {
+ let src = r#"
+def fibonacci(n):
+ if n <= 1:
+ return n
+ return fibonacci(n-1) + fibonacci(n-2)
+
+class Tree:
+ def __init__(self, value):
+ self.value = value
+ self.children = []
+"#;
+ assert_detect(src, ContentType::SourceCode, "python class+def");
+ }
+
+ #[test]
+ fn detects_rust_source() {
+ let src = r#"
+use std::collections::HashMap;
+
+pub struct Counter {
+ counts: HashMap<String, u32>,
+}
+
+impl Counter {
+ pub fn new() -> Self {
+ Self { counts: HashMap::new() }
+ }
+}
+"#;
+ assert_detect(src, ContentType::SourceCode, "rust struct+impl");
+ }
+
+ #[test]
+ fn detects_javascript_source() {
+ let src = r#"
+const fetchUser = async (id) => {
+ const response = await fetch(`/api/users/${id}`);
+ if (!response.ok) throw new Error('Not found');
+ return response.json();
+};
+"#;
+ assert_detect(src, ContentType::SourceCode, "JS arrow + async");
+ }
+
+ #[test]
+ fn detects_unified_diff() {
+ let diff = r#"diff --git a/foo.py b/foo.py
+index abc123..def456 100644
+--- a/foo.py
++++ b/foo.py
+@@ -1,3 +1,4 @@
+ def hello():
++ print("new line")
+ return "world"
+"#;
+ assert_detect(diff, ContentType::GitDiff, "git unified diff");
+ }
+
+ #[test]
+ fn detects_markdown_as_plain_text() {
+ // Markdown isn't routed to a code compressor — it goes to
+ // plain text. This is by design; markdown compression has its
+ // own path that isn't hooked up yet.
+ let md = "# Hello\n\nThis is **bold** and *italic*.\n\n- Item 1\n- Item 2\n";
+ assert_detect(md, ContentType::PlainText, "markdown");
+ }
+
+ #[test]
+ fn detects_plain_text() {
+ let prose = "The quick brown fox jumps over the lazy dog. \
+ This is just regular English prose with no \
+ special structure.";
+ assert_detect(prose, ContentType::PlainText, "english prose");
+ }
+
+ #[test]
+ fn detects_html() {
+ let html =
+ "<!DOCTYPE html><html><head><title>x</title></head><body><h1>Hi</h1></body></html>";
+ assert_detect(html, ContentType::Html, "minimal HTML page");
+ }
+
+ #[test]
+ fn detects_yaml_as_source_code() {
+ let yaml = "name: my-app\nversion: 1.0\ndependencies:\n - foo\n - bar\n";
+ assert_detect(yaml, ContentType::SourceCode, "YAML config");
+ }
+
+ #[test]
+ fn detects_shell_script_as_source_code() {
+ let sh = "#!/bin/bash\nset -euo pipefail\nfor f in *.txt; do\n echo \"$f\"\ndone\n";
+ assert_detect(sh, ContentType::SourceCode, "bash script with shebang");
+ }
+
+ #[test]
+ fn detects_sql_as_source_code() {
+ let sql = "SELECT u.id, u.name, COUNT(o.id) AS order_count \
+ FROM users u LEFT JOIN orders o ON u.id = o.user_id \
+ WHERE u.active = TRUE GROUP BY u.id, u.name;";
+ assert_detect(sql, ContentType::SourceCode, "SQL query");
+ }
+
+ #[test]
+ fn singleton_session_is_reused_across_calls() {
+ // Two back-to-back calls should both succeed without re-initing
+ // the model (no panic, no error). We can't directly observe
+ // the singleton hit-rate without instrumenting the test, but
+ // wall-clock asymmetry between the first and second call is
+ // strong evidence (cold ~50 ms, warm <1 ms). For the unit
+ // suite, just prove neither call errors.
+ magika_detect("hello world").unwrap();
+ magika_detect("def f(): pass").unwrap();
+ magika_detect(r#"{"a":1}"#).unwrap();
+ }
+
+ #[test]
+ fn unmapped_labels_route_to_plain_text() {
+ // Direct test of the mapping table — covers labels we
+ // explicitly didn't enumerate. Future magika versions may
+ // add new labels and we want unknown-but-real labels to
+ // safely passthrough rather than misroute.
+ assert_eq!(map_magika_label("ace"), ContentType::PlainText);
+ assert_eq!(map_magika_label("flac"), ContentType::PlainText);
+ assert_eq!(map_magika_label("3gp"), ContentType::PlainText);
+ assert_eq!(
+ map_magika_label("garbage_unseen_label"),
+ ContentType::PlainText
+ );
+ }
+
+ #[test]
+ fn known_label_table_round_trips() {
+ // Cheap sanity that the mapping arms compile and behave.
+ // No magika session needed — pure table lookup.
+ assert_eq!(map_magika_label("json"), ContentType::JsonArray);
+ assert_eq!(map_magika_label("jsonl"), ContentType::JsonArray);
+ assert_eq!(map_magika_label("diff"), ContentType::GitDiff);
+ assert_eq!(map_magika_label("html"), ContentType::Html);
+ assert_eq!(map_magika_label("rust"), ContentType::SourceCode);
+ assert_eq!(map_magika_label("python"), ContentType::SourceCode);
+ assert_eq!(map_magika_label("yaml"), ContentType::SourceCode);
+ assert_eq!(map_magika_label("markdown"), ContentType::PlainText);
+ assert_eq!(map_magika_label("txt"), ContentType::PlainText);
+ assert_eq!(map_magika_label("empty"), ContentType::PlainText);
+ }
+}
diff --git a/crates/headroom-core/src/transforms/mod.rs b/crates/headroom-core/src/transforms/mod.rs
index 1596fab20..9f8522ecc 100644
--- a/crates/headroom-core/src/transforms/mod.rs
+++ b/crates/headroom-core/src/transforms/mod.rs
@@ -19,6 +19,7 @@ pub mod adaptive_sizer;
pub mod anchor_selector;
pub mod content_detector;
pub mod diff_compressor;
+pub mod magika_detector;
pub mod smart_crusher;
pub use content_detector::{
@@ -27,3 +28,4 @@ pub use content_detector::{
pub use diff_compressor::{
DiffCompressionResult, DiffCompressor, DiffCompressorConfig, DiffCompressorStats,
};
+pub use magika_detector::{magika_detect, map_magika_label, MagikaDetectorError};