Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
chore(rust): port Magika detection (Stage 3d PR3 — Tier 1)
chopratejasd34658b22ef2
diff --git a/Cargo.lock b/Cargo.lockindex 7600b900a..40a038430 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1326,6 +1326,7 @@ dependencies = ["fastembed","flate2","hf-hub 0.4.3",+ "magika","md-5","proptest","regex",@@ -1987,6 +1988,18 @@ version = "0.2.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30"+[[package]]+name = "magika"+version = "1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3aee5ecdbd182547ca3dfcd74c5bcd7f8c57384ad03cb79ef6e3bdf8d56abcdf"+dependencies = [+ "ndarray",+ "ort",+ "thiserror 1.0.69",+ "tokio",+]+[[package]]name = "matchers"version = "0.2.0"diff --git a/crates/headroom-core/Cargo.toml b/crates/headroom-core/Cargo.tomlindex 1039439c6..39b5976f4 100644--- a/crates/headroom-core/Cargo.toml+++ b/crates/headroom-core/Cargo.toml@@ -54,6 +54,13 @@ flate2 = "1"# Same library + same model = byte-equal embeddings between Python and Rust# (both call into ONNX Runtime over the identical ONNX file).fastembed = "5"+# `magika` is Google's ONNX-backed content classifier (Tier 1 of the+# Stage-3d ContentRouter detection arch). Bundled standard-model is+# loaded once per process via `OnceLock` and shared across calls. The+# crate depends on `ort`, which is already in our dep tree via+# `fastembed`, so adding it doesn't pull a new ML runtime — both+# crates share the ONNX Runtime singleton.+magika = "1"[dev-dependencies]proptest = "1"diff --git a/crates/headroom-core/src/transforms/magika_detector.rs b/crates/headroom-core/src/transforms/magika_detector.rsnew file mode 100644index 000000000..614911d6d--- /dev/null+++ b/crates/headroom-core/src/transforms/magika_detector.rs@@ -0,0 +1,344 @@+//! Magika-based content detection (Stage 3d Tier 1).+//!+//! Wraps Google's [`magika`] crate — an ONNX-backed content classifier —+//! and maps its 200+ labels onto Headroom's existing+//! [`crate::transforms::content_detector::ContentType`] enum so the+//! ContentRouter dispatch (PR5) can stay enum-stable.+//!+//! # Design+//!+//! - **Singleton session.** Magika model loading is the expensive part+//! (one-time ONNX init, ~50 ms cold). We do it exactly once per+//! process via [`OnceLock`]. The `Session` requires `&mut self`+//! for inference, so the singleton wraps it in a `Mutex` — fine for+//! our throughput; if benchmarks show contention later we'll pool.+//!+//! - **Loud failures.** If the model fails to load or inference fails,+//! `magika_detect` returns `Err`. The ContentRouter (PR5) decides+//! whether to fall back to Tier 2 (`unidiff-rs`) or surface to the+//! caller. We deliberately do **not** silently return `PlainText` on+//! error — that's the kind of silent fallback the audit doc forbids.+//!+//! - **Mapping table is explicit.** Every magika label we care about+//! has an explicit case in [`map_magika_label`]; everything else+//! falls into [`ContentType::PlainText`]. This is a code-route+//! decision, not a regex — adding a new mapping is one line, and+//! readers can audit the dispatch in one screen.+//!+//! - **No router rewiring here.** PR3 lands the detector + tests+//! only. PR5 flips the ContentRouter to call us instead of the+//! regex-based [`crate::transforms::content_detector`].++use std::sync::{Mutex, OnceLock};++use magika::Session;+use thiserror::Error;++use crate::transforms::content_detector::ContentType;++/// Errors from the magika detector. Wraps the underlying `magika::Error`+/// so callers can match on whether init or inference broke without+/// pulling magika types into their imports.+#[derive(Debug, Error)]+pub enum MagikaDetectorError {+ /// One-time session initialization failed (model load, ONNX init).+ /// Once we hit this, every subsequent call also fails — there is no+ /// retry path here. The router should surface and stop.+ #[error("magika session init failed: {0}")]+ Init(String),++ /// Inference call failed for this input. Usually transient; future+ /// calls may succeed. The error message is the magika-side text;+ /// we don't try to wrap it.+ #[error("magika inference failed: {0}")]+ Inference(String),++ /// Singleton lock was poisoned (a previous holder panicked while+ /// holding it). The detector is unusable until the process+ /// restarts. We don't auto-recover — a panicked detector means+ /// something is corrupt and continuing would mask it.+ #[error("magika session lock poisoned")]+ Poisoned,+}++/// One-process singleton holding the magika session. Lazily+/// initialized on first call to [`magika_detect`].+///+/// `Mutex<Result<Session, ...>>` rather than `Result<Mutex<Session>>`+/// so init failure is recorded once and replayed cheaply on every+/// subsequent call (no re-attempting the load — if the model file is+/// missing or ort can't init, retrying just wastes cycles).+static MAGIKA_SESSION: OnceLock<Mutex<Result<Session, String>>> = OnceLock::new();++fn session() -> &'static Mutex<Result<Session, String>> {+ MAGIKA_SESSION.get_or_init(|| Mutex::new(Session::new().map_err(|e| e.to_string())))+}++/// Classify `content` and return the mapped Headroom [`ContentType`].+///+/// Empty input shortcuts to [`ContentType::PlainText`] without touching+/// the model — saves the round trip on every empty tool result.+pub fn magika_detect(content: &str) -> Result<ContentType, MagikaDetectorError> {+ if content.is_empty() {+ return Ok(ContentType::PlainText);+ }++ let mutex = session();+ let mut guard = mutex.lock().map_err(|_| MagikaDetectorError::Poisoned)?;+ let session = guard+ .as_mut()+ .map_err(|e| MagikaDetectorError::Init(e.clone()))?;++ let bytes = content.as_bytes();+ let file_type = session+ .identify_content_sync(bytes)+ .map_err(|e| MagikaDetectorError::Inference(e.to_string()))?;++ Ok(map_magika_label(file_type.info().label))+}++/// Map a magika label string to Headroom's [`ContentType`] enum.+///+/// **Why explicit cases instead of `group == "code"`:** magika's+/// `group` field is a coarse bucket ("code", "text", "binary",+/// "executable", ...). Some entries we want — like `markdown`,+/// `txt`, `latex` — are in the `text` group along with formats we+/// route differently. So we case on the label directly: clear, one+/// match arm per decision, no group-vs-label semantic confusion.+///+/// **Unmapped labels return [`ContentType::PlainText`]**, the safest+/// default — passthrough at the router level rather than misroute to+/// a wrong compressor. PR5 will refine this for `SearchResults` /+/// `BuildOutput` (which magika has no equivalent for).+pub fn map_magika_label(label: &str) -> ContentType {+ match label {+ // ── JSON ───────────────────────────────────────────────────+ // PR5 will refine this with the existing `is_json_array_of_dicts`+ // check — magika says "this is JSON" but doesn't tell us if it's+ // an array of records vs. a single object. For PR3 the mapping+ // exists; the refinement is a router concern.+ "json" | "jsonl" => ContentType::JsonArray,++ // ── Diffs ──────────────────────────────────────────────────+ "diff" => ContentType::GitDiff,++ // ── HTML ───────────────────────────────────────────────────+ "html" | "xml" => ContentType::Html,++ // ── Source code ────────────────────────────────────────────+ // The big "code" group from magika. We list the labels we+ // actually expect to see in tool outputs / pasted code in+ // proxy traffic. Anything else in the code group falls+ // through to PlainText — better passthrough than misroute.+ "rust" | "python" | "javascript" | "typescript" | "go" | "java" | "c" | "cpp" | "cs"+ | "php" | "ruby" | "swift" | "kotlin" | "scala" | "haskell" | "lua" | "dart" | "perl"+ | "shell" | "powershell" | "batch" | "sql" | "css" | "vue" | "groovy" | "clojure"+ | "asm" | "cmake" | "dockerfile" | "makefile" | "yaml" | "toml" | "ini" | "hcl"+ | "jinja" => ContentType::SourceCode,++ // ── Plain text-ish ─────────────────────────────────────────+ // markdown, rst, latex, log-style, txt, empty/unknown all+ // route as plain text. The router won't try to compress these+ // with a code-aware compressor.+ "markdown" | "rst" | "latex" | "txt" | "empty" | "unknown" | "undefined" => {+ ContentType::PlainText+ }++ // ── Default: passthrough ───────────────────────────────────+ _ => ContentType::PlainText,+ }+}++// ─── Tests ─────────────────────────────────────────────────────────────+//+// These tests are integration-y — they hit the real magika model, which+// loads ONNX on first call (~50 ms cold). Total wall-clock for the full+// suite is dominated by that one-time load, so we keep cases compact.+//+// Detection is probabilistic; we assert against `ContentType` enum+// values rather than confidence scores or labels directly. If magika's+// model version changes (`MODEL_NAME` in their crate), individual+// label assignments may shift but our `match` arms are wide enough to+// stay stable.++#[cfg(test)]+mod tests {+ use super::*;++ fn assert_detect(content: &str, expected: ContentType, hint: &str) {+ match magika_detect(content) {+ Ok(got) => assert_eq!(got, expected, "{hint}: expected {expected:?}, got {got:?}"),+ Err(e) => panic!("{hint}: detection failed: {e}"),+ }+ }++ #[test]+ fn empty_input_is_plain_text_without_model_call() {+ // The shortcut path — should not touch the model.+ let result = magika_detect("").unwrap();+ assert_eq!(result, ContentType::PlainText);+ }++ #[test]+ fn detects_json() {+ assert_detect(+ r#"{"name": "Alice", "age": 30, "tags": ["a", "b"]}"#,+ ContentType::JsonArray,+ "single-object JSON",+ );+ }++ #[test]+ fn detects_json_array() {+ let payload = r#"[{"id": 1, "v": "a"}, {"id": 2, "v": "b"}, {"id": 3, "v": "c"}]"#;+ assert_detect(payload, ContentType::JsonArray, "array-of-records JSON");+ }++ #[test]+ fn detects_python_source() {+ let src = r#"+def fibonacci(n):+ if n <= 1:+ return n+ return fibonacci(n-1) + fibonacci(n-2)++class Tree:+ def __init__(self, value):+ self.value = value+ self.children = []+"#;+ assert_detect(src, ContentType::SourceCode, "python class+def");+ }++ #[test]+ fn detects_rust_source() {+ let src = r#"+use std::collections::HashMap;++pub struct Counter {+ counts: HashMap<String, u32>,+}++impl Counter {+ pub fn new() -> Self {+ Self { counts: HashMap::new() }+ }+}+"#;+ assert_detect(src, ContentType::SourceCode, "rust struct+impl");+ }++ #[test]+ fn detects_javascript_source() {+ let src = r#"+const fetchUser = async (id) => {+ const response = await fetch(`/api/users/${id}`);+ if (!response.ok) throw new Error('Not found');+ return response.json();+};+"#;+ assert_detect(src, ContentType::SourceCode, "JS arrow + async");+ }++ #[test]+ fn detects_unified_diff() {+ let diff = r#"diff --git a/foo.py b/foo.py+index abc123..def456 100644+--- a/foo.py++++ b/foo.py+@@ -1,3 +1,4 @@+ def hello():++ print("new line")+ return "world"+"#;+ assert_detect(diff, ContentType::GitDiff, "git unified diff");+ }++ #[test]+ fn detects_markdown_as_plain_text() {+ // Markdown isn't routed to a code compressor — it goes to+ // plain text. This is by design; markdown compression has its+ // own path that isn't hooked up yet.+ let md = "# Hello\n\nThis is **bold** and *italic*.\n\n- Item 1\n- Item 2\n";+ assert_detect(md, ContentType::PlainText, "markdown");+ }++ #[test]+ fn detects_plain_text() {+ let prose = "The quick brown fox jumps over the lazy dog. \+ This is just regular English prose with no \+ special structure.";+ assert_detect(prose, ContentType::PlainText, "english prose");+ }++ #[test]+ fn detects_html() {+ let html =+ "<!DOCTYPE html><html><head><title>x</title></head><body><h1>Hi</h1></body></html>";+ assert_detect(html, ContentType::Html, "minimal HTML page");+ }++ #[test]+ fn detects_yaml_as_source_code() {+ let yaml = "name: my-app\nversion: 1.0\ndependencies:\n - foo\n - bar\n";+ assert_detect(yaml, ContentType::SourceCode, "YAML config");+ }++ #[test]+ fn detects_shell_script_as_source_code() {+ let sh = "#!/bin/bash\nset -euo pipefail\nfor f in *.txt; do\n echo \"$f\"\ndone\n";+ assert_detect(sh, ContentType::SourceCode, "bash script with shebang");+ }++ #[test]+ fn detects_sql_as_source_code() {+ let sql = "SELECT u.id, u.name, COUNT(o.id) AS order_count \+ FROM users u LEFT JOIN orders o ON u.id = o.user_id \+ WHERE u.active = TRUE GROUP BY u.id, u.name;";+ assert_detect(sql, ContentType::SourceCode, "SQL query");+ }++ #[test]+ fn singleton_session_is_reused_across_calls() {+ // Two back-to-back calls should both succeed without re-initing+ // the model (no panic, no error). We can't directly observe+ // the singleton hit-rate without instrumenting the test, but+ // wall-clock asymmetry between the first and second call is+ // strong evidence (cold ~50 ms, warm <1 ms). For the unit+ // suite, just prove neither call errors.+ magika_detect("hello world").unwrap();+ magika_detect("def f(): pass").unwrap();+ magika_detect(r#"{"a":1}"#).unwrap();+ }++ #[test]+ fn unmapped_labels_route_to_plain_text() {+ // Direct test of the mapping table — covers labels we+ // explicitly didn't enumerate. Future magika versions may+ // add new labels and we want unknown-but-real labels to+ // safely passthrough rather than misroute.+ assert_eq!(map_magika_label("ace"), ContentType::PlainText);+ assert_eq!(map_magika_label("flac"), ContentType::PlainText);+ assert_eq!(map_magika_label("3gp"), ContentType::PlainText);+ assert_eq!(+ map_magika_label("garbage_unseen_label"),+ ContentType::PlainText+ );+ }++ #[test]+ fn known_label_table_round_trips() {+ // Cheap sanity that the mapping arms compile and behave.+ // No magika session needed — pure table lookup.+ assert_eq!(map_magika_label("json"), ContentType::JsonArray);+ assert_eq!(map_magika_label("jsonl"), ContentType::JsonArray);+ assert_eq!(map_magika_label("diff"), ContentType::GitDiff);+ assert_eq!(map_magika_label("html"), ContentType::Html);+ assert_eq!(map_magika_label("rust"), ContentType::SourceCode);+ assert_eq!(map_magika_label("python"), ContentType::SourceCode);+ assert_eq!(map_magika_label("yaml"), ContentType::SourceCode);+ assert_eq!(map_magika_label("markdown"), ContentType::PlainText);+ assert_eq!(map_magika_label("txt"), ContentType::PlainText);+ assert_eq!(map_magika_label("empty"), ContentType::PlainText);+ }+}diff --git a/crates/headroom-core/src/transforms/mod.rs b/crates/headroom-core/src/transforms/mod.rsindex 1596fab20..9f8522ecc 100644--- a/crates/headroom-core/src/transforms/mod.rs+++ b/crates/headroom-core/src/transforms/mod.rs@@ -19,6 +19,7 @@ pub mod adaptive_sizer;pub mod anchor_selector;pub mod content_detector;pub mod diff_compressor;+pub mod magika_detector;pub mod smart_crusher;pub use content_detector::{@@ -27,3 +28,4 @@ pub use content_detector::{pub use diff_compressor::{DiffCompressionResult, DiffCompressor, DiffCompressorConfig, DiffCompressorStats,};+pub use magika_detector::{magika_detect, map_magika_label, MagikaDetectorError};