Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(rust): audit cleanup — DiffCompressor CCR leak, CCR TOCTOU race, clippy debt, dep dedup
chopratejas378d8a0f055f
diff --git a/Cargo.lock b/Cargo.lockindex c80b2b6fd..ba0fc5a2a 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1008,7 +1008,7 @@ dependencies = ["safetensors","serde","serde_json",- "tokenizers 0.22.2",+ "tokenizers",][[package]]@@ -1337,7 +1337,7 @@ dependencies = ["sha2","thiserror 1.0.69","tiktoken-rs",- "tokenizers 0.21.4",+ "tokenizers","toml","tracing","unidiff",@@ -3602,40 +3602,6 @@ version = "0.1.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"-[[package]]-name = "tokenizers"-version = "0.21.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476"-dependencies = [- "ahash",- "aho-corasick",- "compact_str",- "dary_heap",- "derive_builder",- "esaxx-rs",- "getrandom 0.3.4",- "indicatif 0.17.11",- "itertools 0.14.0",- "log",- "macro_rules_attribute",- "monostate",- "onig",- "paste",- "rand 0.9.4",- "rayon",- "rayon-cond",- "regex",- "regex-syntax",- "serde",- "serde_json",- "spm_precompiled",- "thiserror 2.0.18",- "unicode-normalization-alignments",- "unicode-segmentation",- "unicode_categories",-]-[[package]]name = "tokenizers"version = "0.22.2"@@ -3649,6 +3615,7 @@ dependencies = ["derive_builder","esaxx-rs","getrandom 0.3.4",+ "indicatif 0.18.4","itertools 0.14.0","log","macro_rules_attribute",diff --git a/RUST_DEV.md b/RUST_DEV.mdindex c1d7db734..5c6ab1809 100644--- a/RUST_DEV.md+++ b/RUST_DEV.md@@ -91,6 +91,36 @@ curl -si http://127.0.0.1:8787/v1/models| `--rewrite-host` / `--no-rewrite-host` | | rewrite | rewrite Host to upstream (default) || `--graceful-shutdown-timeout` | | `30s` | wait for in-flight on SIGTERM/SIGINT |+### Picking the next port: invocation telemetry++Before porting another Python compressor to Rust, check what's actually+running. The Python proxy already exposes per-transform telemetry on+`/stats` (`headroom.proxy.prometheus_metrics`):++```bash+# Top compressors by invocation count (last process lifetime)+curl -s http://127.0.0.1:8788/stats | jq '.compressions_by_strategy'+# {+# "intelligent_context": 12453,+# "smart_crusher": 487,+# "search": 312,+# "diff": 28,+# "code": 0, # ← never fires; safe to defer porting+# ...+# }++# Per-transform timing (avg/max/count by transform name)+curl -s http://127.0.0.1:8788/stats | jq '.pipeline_timing'++# Token savings attributable to each strategy+curl -s http://127.0.0.1:8788/stats | jq '.tokens_saved_by_strategy'+```++This is the data the audit-cleanup PR (2026-04-30) recommended for+prioritizing the next Python → Rust port. Strategies with zero or+near-zero invocations are deferral candidates; strategies on the hot+path are porting candidates regardless of LOC count.+### Reserved paths`/healthz` and `/healthz/upstream` are intercepted by the Rust proxy anddiff --git a/crates/headroom-core/Cargo.toml b/crates/headroom-core/Cargo.tomlindex 37521a0ea..ec1ff15d7 100644--- a/crates/headroom-core/Cargo.toml+++ b/crates/headroom-core/Cargo.toml@@ -17,7 +17,7 @@ tiktoken-rs = "0.11"# `tokenizers` is the HuggingFace pure-Rust tokenizer crate. Default features# pull in `onig` for the BPE pre-tokenizer regex; that vendors oniguruma so it# builds without a system dep on macOS/Linux.-tokenizers = "0.21"+tokenizers = "0.22"# `hf-hub` is the HuggingFace Hub client. We use the blocking `ureq` transport# with `rustls` (no system OpenSSL dep — keeps the binary static-linkable for# AWS deploys). `from_pretrained` is called once at startup, so blocking isdiff --git a/crates/headroom-core/src/ccr.rs b/crates/headroom-core/src/ccr.rsindex 12ef64e63..cbc54e85b 100644--- a/crates/headroom-core/src/ccr.rs+++ b/crates/headroom-core/src/ccr.rs@@ -167,19 +167,37 @@ impl CcrStore for InMemoryCcrStore {// Read path: shard read-lock, check TTL, clone payload out.// No global lock involvement at all — distinct hashes hash to// distinct shards and never contend.- let expired_at = {- let entry = self.map.get(hash)?;- if entry.inserted.elapsed() > self.ttl {- Some(()) // signal expired; drop guard before we remove- } else {+ //+ // Lazy expiry uses DashMap's `remove_if` so the check-and-remove+ // is atomic on the shard. An earlier 2-step (drop read lock,+ // then `remove`) had a TOCTOU race: between dropping the read+ // lock and calling `remove`, a concurrent `put()` of the same+ // hash with a fresh timestamp could land — and our `remove`+ // would then wipe that fresh entry. Under multi-worker proxy+ // load this manifested as "I just stored it; why is it gone?"+ // `remove_if` closes the window because the shard write lock+ // is held across both the predicate evaluation and the removal.+ if let Some(entry) = self.map.get(hash) {+ if entry.inserted.elapsed() <= self.ttl {return Some(entry.payload.clone());}- };- if expired_at.is_some() {- self.map.remove(hash);+ } else {return None;}- None+ // Out-of-band path: the entry exists and looks expired. Re-check+ // under the shard write lock; if it's still expired, evict.+ // Otherwise (a concurrent `put` refreshed it) leave it alone+ // and re-fetch its payload.+ let was_removed = self+ .map+ .remove_if(hash, |_, entry| entry.inserted.elapsed() > self.ttl)+ .is_some();+ if was_removed {+ None+ } else {+ // Concurrent refresh — return the fresh payload.+ self.map.get(hash).map(|e| e.payload.clone())+ }}fn len(&self) -> usize {@@ -281,4 +299,66 @@ mod tests {}assert_eq!(store.len(), n_threads * per_thread);}++ #[test]+ fn expired_get_does_not_wipe_concurrent_refresh() {+ // Regression for the TOCTOU race fixed in the audit-cleanup PR.+ // Two threads contend on the SAME key:+ // - Thread A: stores fresh value, then `get` it many times.+ // - Thread B: keeps re-storing the same key with FRESH+ // timestamps in a tight loop (simulating a second worker+ // touching the same payload).+ // With the old 2-step check-then-remove, A's `get` could see+ // an "expired" entry, drop the read lock, and remove B's+ // freshly-inserted entry between drop and remove. With+ // `remove_if`, the predicate runs under the shard write lock,+ // so the race window is closed.+ use std::sync::Arc;+ use std::thread;++ let store = Arc::new(InMemoryCcrStore::with_capacity_and_ttl(+ 64,+ Duration::from_millis(20),+ ));+ let key = "shared_key";+ let payload = "fresh";++ // Seed.+ store.put(key, payload);++ let writer = {+ let s = store.clone();+ thread::spawn(move || {+ // 200 fresh re-stores, racing the reader.+ for _ in 0..200 {+ s.put(key, payload);+ }+ })+ };++ let reader = {+ let s = store.clone();+ thread::spawn(move || {+ let mut hits = 0;+ for _ in 0..200 {+ if s.get(key).as_deref() == Some(payload) {+ hits += 1;+ }+ }+ hits+ })+ };++ writer.join().unwrap();+ let hits = reader.join().unwrap();+ // The entry must be live at the end (writer's last put won).+ assert_eq!(store.get(key).as_deref(), Some(payload));+ // Reader should have observed the live entry the vast majority+ // of the time. Allow some misses on first iterations / TTL+ // transitions but require strong majority.+ assert!(+ hits > 100,+ "reader should mostly observe live entry, hits={hits}"+ );+ }}diff --git a/crates/headroom-core/src/transforms/diff_compressor.rs b/crates/headroom-core/src/transforms/diff_compressor.rsindex e1a1c39fc..959addff5 100644--- a/crates/headroom-core/src/transforms/diff_compressor.rs+++ b/crates/headroom-core/src/transforms/diff_compressor.rs@@ -42,6 +42,8 @@ use std::time::Instant;use md5::{Digest, Md5};use regex::Regex;+use crate::ccr::CcrStore;+// ─── Score-weight constants ────────────────────────────────────────────────//// These knobs tune the relevance scorer (used only when `max_hunks_per_file`@@ -244,12 +246,37 @@ impl DiffCompressor {}/// Same as [`compress`] but also returns rich observability stats.+ /// Equivalent to `compress_with_store(content, context, None).0`.////// [`compress`]: Self::compresspub fn compress_with_stats(&self,content: &str,context: &str,+ ) -> (DiffCompressionResult, DiffCompressorStats) {+ self.compress_with_store(content, context, None)+ }++ /// Compress with optional CCR persistence.+ ///+ /// Mirrors [`crate::transforms::log_compressor::LogCompressor::compress_with_store`]+ /// and the search-compressor sibling: when `store` is `Some`, the+ /// original `content` is written to it under the same `cache_key`+ /// the wire-output marker carries. When `store` is `None`, the+ /// `cache_key` is still emitted but the caller is responsible for+ /// persisting (e.g. the Python shim's `_persist_to_python_ccr` path).+ ///+ /// **Why this exists:** prior to this method, `DiffCompressor` minted+ /// a `cache_key` and embedded it in the output marker but never+ /// stored — leaving Python ContentRouter with a dangling marker that+ /// would 404 on retrieval. The `DiffOffload` orchestrator wrapper+ /// papered over this for the new pipeline path; this method+ /// upstreams the fix so any caller can wire in storage cleanly.+ pub fn compress_with_store(+ &self,+ content: &str,+ context: &str,+ store: Option<&dyn CcrStore>,) -> (DiffCompressionResult, DiffCompressorStats) {let start = Instant::now();let mut stats = DiffCompressorStats::default();@@ -452,6 +479,14 @@ impl DiffCompressor {"[{} lines compressed to {}. Retrieve full diff: hash={}]",original_line_count, compressed_line_count, key));+ // Persist the original under the same key. When `store` is+ // `Some`, the marker we just emitted resolves through it on+ // the LLM's retrieval tool call. When `None`, the caller+ // (typically the Python shim) is responsible — see the+ // method-level docs.+ if let Some(s) = store {+ s.put(&key, content);+ }cache_key = Some(key);stats.cache_key_emitted = true;} else if !self.config.enable_ccr {@@ -1375,6 +1410,56 @@ mod tests {assert!(stats.ccr_skipped_reason.is_some());}+ #[test]+ fn compress_with_store_persists_original_under_cache_key() {+ // Regression: pre-fix, `DiffCompressor` minted a `cache_key` and+ // embedded `[... hash=abc123]` in the output marker but never+ // wrote the original anywhere — leaving Python ContentRouter+ // with a dangling marker that 404'd on retrieval. Now any caller+ // can pass a store and the marker resolves.+ use crate::ccr::InMemoryCcrStore;+ let store = InMemoryCcrStore::new();+ let input = build_synthetic_diff(8);+ let (r, stats) = DiffCompressor::default().compress_with_store(&input, "", Some(&store));+ let key = r.cache_key.expect("default 0.8 should emit CCR");+ assert!(stats.cache_key_emitted);+ // Marker text must reference the same key.+ assert!(r.compressed.contains(&format!("hash={key}")));+ // Original must round-trip through the store.+ assert_eq!(store.get(&key).as_deref(), Some(input.as_str()));+ }++ #[test]+ fn compress_with_store_none_matches_compress_with_stats_behavior() {+ // Passing `None` must be byte-identical to the legacy API:+ // emits cache_key, leaves persistence to the caller. This pins+ // the parity-preserving-default contract.+ let input = build_synthetic_diff(8);+ let (legacy_result, _) = DiffCompressor::default().compress_with_stats(&input, "");+ let (new_result, _) = DiffCompressor::default().compress_with_store(&input, "", None);+ assert_eq!(new_result.compressed, legacy_result.compressed);+ assert_eq!(new_result.cache_key, legacy_result.cache_key);+ }++ #[test]+ fn compress_with_store_no_op_when_ccr_skipped() {+ // When compression doesn't clear the savings threshold, no+ // cache_key is minted AND no store write happens.+ use crate::ccr::InMemoryCcrStore;+ let cfg = DiffCompressorConfig {+ min_compression_ratio_for_ccr: 0.1, // very strict+ ..Default::default()+ };+ let store = InMemoryCcrStore::new();+ let (r, _) = DiffCompressor::new(cfg).compress_with_store(+ &build_synthetic_diff(8),+ "",+ Some(&store),+ );+ assert!(r.cache_key.is_none());+ assert_eq!(store.len(), 0);+ }+#[test]fn score_constants_match_inline_values() {// Pin the constants so a future tuning PR has to update both.diff --git a/crates/headroom-core/src/transforms/pipeline/offloads/diff_offload.rs b/crates/headroom-core/src/transforms/pipeline/offloads/diff_offload.rsindex c9effe69e..d5a5df487 100644--- a/crates/headroom-core/src/transforms/pipeline/offloads/diff_offload.rs+++ b/crates/headroom-core/src/transforms/pipeline/offloads/diff_offload.rs@@ -20,16 +20,15 @@//! No regex — pure byte-prefix checks. Cost: O(n) over the input//! (single pass, no allocations).//!-//! # Bug-fix-on-port+//! # CCR persistence//!-//! [`DiffCompressor`] emits a `cache_key` in its output marker without-//! actually persisting the original to any store (it never sees one).-//! That's a leak the existing parity-bound port couldn't fix without-//! breaking byte equality. As an [`OffloadTransform`] consumer we close-//! the leak by storing the original payload under the same key after-//! the compressor returns. The trait contract is that-//! `cache_key MUST resolve in store`; this wrapper is what makes that-//! true for the diff path.+//! `DiffCompressor::compress_with_store` writes the original payload to+//! the orchestrator-supplied store under the same `cache_key` it embeds+//! in the wire marker. The trait contract is satisfied by the+//! compressor itself; no double-store hack needed at the wrapper level.+//! (Earlier revisions of this offload did the post-hoc store from here+//! because the compressor lacked a store parameter — that's been+//! upstreamed in the audit-cleanup PR.)//!//! [`DiffCompressor`]: crate::transforms::diff_compressor::DiffCompressor//! [`OffloadTransform`]: crate::transforms::pipeline::traits::OffloadTransform@@ -134,7 +133,14 @@ impl OffloadTransform for DiffOffload {ctx: &CompressionContext,store: &dyn CcrStore,) -> Result<OffloadOutput, TransformError> {- let result = self.compressor.compress(content, &ctx.query);+ // `compress_with_store` (added in the audit-cleanup PR) writes+ // the original to `store` under the same `cache_key` it embeds+ // in the marker. Earlier versions of this offload had to+ // double-store post-hoc because the compressor lacked a store+ // parameter — that hack is gone.+ let (result, _) = self+ .compressor+ .compress_with_store(content, &ctx.query, Some(store));let Some(key) = result.cache_key else {return Err(TransformError::skipped(@@ -143,11 +149,6 @@ impl OffloadTransform for DiffOffload {));};- // Bug-fix-on-port: DiffCompressor mints a key but never stores.- // Persist the original here so the trait contract (key resolves- // in store) holds.- store.put(&key, content);-Ok(OffloadOutput::from_lengths(content.len(),result.compressed,diff --git a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs b/crates/headroom-core/src/transforms/smart_crusher/crusher.rsindex 39648132f..8054f9caa 100644--- a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs+++ b/crates/headroom-core/src/transforms/smart_crusher/crusher.rs@@ -933,14 +933,11 @@ fn hash_canonical(canonical: &str) -> String {.collect()}-/// Convenience: canonical-serialize `items` and hash the result. Kept-/// for sites (e.g. tests) that don't also need the canonical bytes for-/// storage. Production lossy path inlines `canonical_array_json` +-/// `hash_canonical` so the bytes are reused for the store payload.-#[cfg(test)]-fn hash_array_for_ccr(items: &[Value]) -> String {- hash_canonical(&canonical_array_json(items))-}+// `hash_array_for_ccr` (a test-only `canonical_array_json + hash_canonical`+// convenience) lived here previously but had no callers — clippy flagged+// it as dead code. Reintroduce as a test fixture if a future test wants+// the one-liner; production callsites inline both steps so the canonical+// bytes can be reused for the store payload.// ─── PR5 walker-integration helpers (string handling) ──────────────────────//@@ -1277,8 +1274,10 @@ mod tests {// Use low-uniqueness items so the analyzer is willing to// crush (unique id+name per row would trip the// "unique_entities_no_signal" skip gate instead).- let mut cfg = SmartCrusherConfig::default();- cfg.lossless_min_savings_ratio = 0.99;+ let cfg = SmartCrusherConfig {+ lossless_min_savings_ratio: 0.99,+ ..Default::default()+ };let c = SmartCrusher::new(cfg);let items: Vec<Value> = (0..50).map(|_| json!({"status": "ok"})).collect();let result = c.crush_array(&items, "", 1.0);@@ -1305,8 +1304,10 @@ mod tests {#[test]fn ccr_hash_is_deterministic() {// Same input → same hash, so the runtime cache key is stable.- let mut cfg = SmartCrusherConfig::default();- cfg.lossless_min_savings_ratio = 0.99; // force lossy path+ let cfg = SmartCrusherConfig {+ lossless_min_savings_ratio: 0.99, // force lossy path+ ..Default::default()+ };let c = SmartCrusher::new(cfg);let items: Vec<Value> = (0..30).map(|i| json!({"id": i, "tag": "ok"})).collect();let r1 = c.crush_array(&items, "", 1.0);@@ -1317,8 +1318,10 @@ mod tests {#[test]fn ccr_hash_changes_with_input() {- let mut cfg = SmartCrusherConfig::default();- cfg.lossless_min_savings_ratio = 0.99;+ let cfg = SmartCrusherConfig {+ lossless_min_savings_ratio: 0.99,+ ..Default::default()+ };let c = SmartCrusher::new(cfg);let a: Vec<Value> = (0..30).map(|i| json!({"id": i})).collect();let b: Vec<Value> = (100..130).map(|i| json!({"id": i})).collect();diff --git a/headroom/transforms/diff_compressor.py b/headroom/transforms/diff_compressor.pyindex 7b0418f74..67742f3e0 100644--- a/headroom/transforms/diff_compressor.py+++ b/headroom/transforms/diff_compressor.py@@ -104,6 +104,16 @@ class DiffCompressor:def compress(self, content: str, context: str = "") -> DiffCompressionResult:r = self._rust.compress(content, context)+ cache_key: str | None = r.cache_key+ if cache_key is not None:+ # Mirror log_compressor.py + search_compressor.py: when the+ # Rust path emits a CCR retrieval marker, persist the+ # original payload to Python's CompressionStore so the+ # marker actually resolves on the LLM's retrieval tool+ # call. Without this, every diff CCR marker emitted in+ # production is dangling — the regression fixed in the+ # audit-cleanup PR.+ self._persist_to_python_ccr(content, r.compressed, cache_key)return DiffCompressionResult(compressed=r.compressed,original_line_count=r.original_line_count,@@ -113,9 +123,30 @@ class DiffCompressor:deletions=r.deletions,hunks_kept=r.hunks_kept,hunks_removed=r.hunks_removed,- cache_key=r.cache_key,+ cache_key=cache_key,)+ def _persist_to_python_ccr(self, original: str, compressed: str, cache_key: str) -> None:+ """Promote a Rust-emitted cache_key into the production Python+ CompressionStore. Failures are logged at warning level — a+ store hiccup must not break the response, just degrade+ retrieval. Mirrors the same helper on log_compressor.py and+ search_compressor.py."""+ try:+ from ..cache.compression_store import get_compression_store+ except ImportError as e:+ logger.warning("CCR store import failed; cache_key %s won't persist: %s", cache_key, e)+ return+ try:+ store: Any = get_compression_store()+ store.store(original, compressed)+ except Exception as e:+ logger.warning(+ "CCR store write failed; cache_key %s remains in-marker only: %s",+ cache_key,+ e,+ )+def compress_with_stats(self, content: str, context: str = "") -> tuple[DiffCompressionResult, Any]: