Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
chore(rust): SmartCrusher CCR storage layer + roundtrip verification
chopratejas22c8fec4c105
diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.jsonindex 78381697b..79917feb7 100644--- a/.claude-plugin/marketplace.json+++ b/.claude-plugin/marketplace.json@@ -5,14 +5,14 @@},"metadata": {"description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.",- "version": "0.11.0"+ "version": "0.13.2"},"plugins": [{"name": "headroom","source": "./plugins/headroom-agent-hooks","description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",- "version": "0.11.0",+ "version": "0.13.2","author": {"name": "Headroom Contributors","url": "https://github.com/chopratejas/headroom"diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.jsonindex 78381697b..79917feb7 100644--- a/.github/plugin/marketplace.json+++ b/.github/plugin/marketplace.json@@ -5,14 +5,14 @@},"metadata": {"description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.",- "version": "0.11.0"+ "version": "0.13.2"},"plugins": [{"name": "headroom","source": "./plugins/headroom-agent-hooks","description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",- "version": "0.11.0",+ "version": "0.13.2","author": {"name": "Headroom Contributors","url": "https://github.com/chopratejas/headroom"diff --git a/Cargo.lock b/Cargo.lockindex 36765ede5..376f9e58d 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1369,6 +1369,7 @@ version = "0.1.0"dependencies = ["headroom-core","pyo3",+ "serde_json",][[package]]diff --git a/crates/headroom-core/src/ccr.rs b/crates/headroom-core/src/ccr.rsnew file mode 100644index 000000000..0fd3dbab0--- /dev/null+++ b/crates/headroom-core/src/ccr.rs@@ -0,0 +1,223 @@+//! CCR (Compress-Cache-Retrieve) storage layer.+//!+//! When a transform compresses data with row-drop or opaque-string+//! substitution, the *original payload* is stashed here keyed by the+//! hash that ends up in the prompt. The runtime later honors retrieval+//! tool calls by looking up the hash in this store and serving back the+//! original. This is the cornerstone of CCR: lossy on the wire, lossless+//! end-to-end.+//!+//! Mirrors the semantics of Python's [`CompressionStore`] (`headroom/+//! cache/compression_store.py`) but stripped down to the contract that+//! actually matters for retrieval — no BM25 search, no retrieval-event+//! feedback, no per-tool metadata. Those live in the runtime layer; this+//! crate only needs put/get.+//!+//! # Pluggable backend+//!+//! Production deployments swap in their own [`CcrStore`] backed by Redis,+//! MongoDB, or whatever shared cache fits. The default in-memory store+//! ships ready for single-process use.+//!+//! [`CompressionStore`]: https://github.com/chopratejas/headroom/blob/main/headroom/cache/compression_store.py++use std::collections::{HashMap, VecDeque};+use std::sync::Mutex;+use std::time::{Duration, Instant};++/// Pluggable CCR storage backend. `Send + Sync` so it can sit behind an+/// `Arc` and be shared across threads in the proxy.+pub trait CcrStore: Send + Sync {+ /// Stash `payload` under `hash`. If the hash already exists, the+ /// new payload overwrites — same hash should mean same content, so+ /// re-storing is idempotent.+ fn put(&self, hash: &str, payload: &str);++ /// Look up `hash`. Returns `None` if missing or expired.+ fn get(&self, hash: &str) -> Option<String>;++ /// Number of live entries. Informational; used by tests + telemetry.+ fn len(&self) -> usize;++ fn is_empty(&self) -> bool {+ self.len() == 0+ }+}++/// Default capacity — matches Python's `CompressionStore` default.+pub const DEFAULT_CAPACITY: usize = 1000;++/// Default TTL — 5 minutes, matching Python.+pub const DEFAULT_TTL: Duration = Duration::from_secs(300);++/// Simple in-memory CCR store with TTL + bounded capacity.+///+/// - **TTL**: 5 minutes by default. Entries past their TTL are dropped+/// on the next `get`.+/// - **Capacity**: 1000 entries by default. When full, the oldest+/// insertion is evicted (FIFO).+/// - **Locking**: single `Mutex`. The store sits on the cold path+/// (one call per crushed array, not per token) so coarse locking is+/// fine and keeps the implementation small.+pub struct InMemoryCcrStore {+ inner: Mutex<Inner>,+ ttl: Duration,+ capacity: usize,+}++struct Inner {+ map: HashMap<String, Entry>,+ /// FIFO order of insertion for capacity eviction. Hashes that get+ /// re-stored stay at their original position — same content under+ /// the same hash is idempotent and rare.+ order: VecDeque<String>,+}++struct Entry {+ payload: String,+ inserted: Instant,+}++impl InMemoryCcrStore {+ /// Default: 1000 entries, 5-minute TTL.+ pub fn new() -> Self {+ Self::with_capacity_and_ttl(DEFAULT_CAPACITY, DEFAULT_TTL)+ }++ pub fn with_capacity_and_ttl(capacity: usize, ttl: Duration) -> Self {+ Self {+ inner: Mutex::new(Inner {+ map: HashMap::new(),+ order: VecDeque::new(),+ }),+ ttl,+ capacity,+ }+ }+}++impl Default for InMemoryCcrStore {+ fn default() -> Self {+ Self::new()+ }+}++impl CcrStore for InMemoryCcrStore {+ fn put(&self, hash: &str, payload: &str) {+ let mut g = self.inner.lock().expect("ccr store mutex poisoned");++ if g.map.contains_key(hash) {+ // Idempotent re-store. Same hash should mean same content;+ // overwrite the payload (cheap) and keep the original FIFO+ // position so eviction stays predictable.+ g.map.insert(+ hash.to_string(),+ Entry {+ payload: payload.to_string(),+ inserted: Instant::now(),+ },+ );+ return;+ }++ // New entry. Evict the oldest if we're at capacity.+ while g.map.len() >= self.capacity {+ let Some(oldest) = g.order.pop_front() else {+ break;+ };+ g.map.remove(&oldest);+ }++ g.map.insert(+ hash.to_string(),+ Entry {+ payload: payload.to_string(),+ inserted: Instant::now(),+ },+ );+ g.order.push_back(hash.to_string());+ }++ fn get(&self, hash: &str) -> Option<String> {+ let mut g = self.inner.lock().expect("ccr store mutex poisoned");+ let expired = match g.map.get(hash) {+ Some(e) => e.inserted.elapsed() > self.ttl,+ None => return None,+ };+ if expired {+ g.map.remove(hash);+ return None;+ }+ g.map.get(hash).map(|e| e.payload.clone())+ }++ fn len(&self) -> usize {+ self.inner+ .lock()+ .expect("ccr store mutex poisoned")+ .map+ .len()+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn put_then_get_returns_payload() {+ let store = InMemoryCcrStore::new();+ store.put("abc123", r#"[{"id":1}]"#);+ assert_eq!(store.get("abc123"), Some(r#"[{"id":1}]"#.to_string()));+ }++ #[test]+ fn missing_hash_returns_none() {+ let store = InMemoryCcrStore::new();+ assert_eq!(store.get("never_stored"), None);+ }++ #[test]+ fn put_overwrites_under_same_hash() {+ let store = InMemoryCcrStore::new();+ store.put("h", "first");+ store.put("h", "second");+ assert_eq!(store.get("h"), Some("second".to_string()));+ assert_eq!(store.len(), 1);+ }++ #[test]+ fn capacity_evicts_oldest() {+ let store = InMemoryCcrStore::with_capacity_and_ttl(2, DEFAULT_TTL);+ store.put("a", "1");+ store.put("b", "2");+ store.put("c", "3");+ assert_eq!(store.len(), 2);+ assert_eq!(store.get("a"), None);+ assert_eq!(store.get("b"), Some("2".to_string()));+ assert_eq!(store.get("c"), Some("3".to_string()));+ }++ #[test]+ fn expired_entries_are_dropped_on_get() {+ let store = InMemoryCcrStore::with_capacity_and_ttl(10, Duration::from_millis(10));+ store.put("a", "1");+ std::thread::sleep(Duration::from_millis(25));+ assert_eq!(store.get("a"), None);+ assert_eq!(store.len(), 0);+ }++ #[test]+ fn store_is_send_sync() {+ fn assert_send_sync<T: Send + Sync>() {}+ assert_send_sync::<InMemoryCcrStore>();+ }++ #[test]+ fn trait_object_is_usable() {+ let store: Box<dyn CcrStore> = Box::new(InMemoryCcrStore::new());+ store.put("h", "v");+ assert_eq!(store.get("h"), Some("v".to_string()));+ assert!(!store.is_empty());+ }+}diff --git a/crates/headroom-core/src/lib.rs b/crates/headroom-core/src/lib.rsindex 6071c860c..87acea72c 100644--- a/crates/headroom-core/src/lib.rs+++ b/crates/headroom-core/src/lib.rs@@ -1,5 +1,6 @@//! headroom-core: foundation crate for the Rust port of Headroom.+pub mod ccr;pub mod relevance;pub mod tokenizer;pub mod transforms;diff --git a/crates/headroom-core/src/transforms/smart_crusher/builder.rs b/crates/headroom-core/src/transforms/smart_crusher/builder.rsindex 9ccc8d74f..2662a5ef8 100644--- a/crates/headroom-core/src/transforms/smart_crusher/builder.rs+++ b/crates/headroom-core/src/transforms/smart_crusher/builder.rs@@ -30,6 +30,9 @@//! makes your intent explicit; the `new()` factory shorthand for the//! OSS preset.+use std::sync::Arc;++use crate::ccr::{CcrStore, InMemoryCcrStore};use crate::relevance::{HybridScorer, RelevanceScorer};use crate::transforms::anchor_selector::{AnchorConfig, AnchorSelector};@@ -49,6 +52,7 @@ pub struct SmartCrusherBuilder {constraints: Vec<Box<dyn Constraint>>,observers: Vec<Box<dyn Observer>>,compaction: Option<CompactionStage>,+ ccr_store: Option<Arc<dyn CcrStore>>,}impl SmartCrusherBuilder {@@ -62,6 +66,7 @@ impl SmartCrusherBuilder {constraints: Vec::new(),observers: Vec::new(),compaction: None,+ ccr_store: None,}}@@ -140,6 +145,20 @@ impl SmartCrusherBuilder {self.with_compaction(CompactionStage::default_csv_schema())}+ /// Plug in a CCR store. The lossy `crush_array` path stashes each+ /// dropped array's full original here keyed by its hash, so the+ /// runtime can serve retrieval tool calls with no data loss.+ pub fn with_ccr_store(mut self, store: Arc<dyn CcrStore>) -> Self {+ self.ccr_store = Some(store);+ self+ }++ /// Convenience: install the default in-memory CCR store+ /// (1000 entries, 5-minute TTL — matches Python).+ pub fn with_default_ccr_store(self) -> Self {+ self.with_ccr_store(Arc::new(InMemoryCcrStore::new()))+ }+/// Construct the `SmartCrusher`. If `with_scorer` was not called,/// falls back to `HybridScorer::default()` so a builder with no/// other customization still produces a working crusher.@@ -157,6 +176,7 @@ impl SmartCrusherBuilder {self.constraints,self.observers,self.compaction,+ self.ccr_store,)}}diff --git a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs b/crates/headroom-core/src/transforms/smart_crusher/crusher.rsindex 8eaa49fdf..ad349b854 100644--- a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs+++ b/crates/headroom-core/src/transforms/smart_crusher/crusher.rs@@ -33,6 +33,8 @@//! Python side, locking byte-equal output. The TOIN/CCR/feedback//! integration ports happen later (Stage 3c.2 follow-ups).+use std::sync::Arc;+use serde_json::Value;use super::analyzer::SmartAnalyzer;@@ -44,6 +46,7 @@ use super::crushers::{compute_k_split, crush_number_array, crush_object, crush_suse super::planning::SmartCrusherPlanner;use super::traits::{Constraint, CrushEvent, Observer};use super::types::{CompressionPlan, CompressionStrategy, CrushResult};+use crate::ccr::CcrStore;use crate::relevance::RelevanceScorer;use crate::transforms::adaptive_sizer::compute_optimal_k;use crate::transforms::anchor_selector::AnchorSelector;@@ -119,6 +122,16 @@ pub struct SmartCrusher {/// the lossy path on success. When `None` (default OSS), parity/// with the pre-PR2 lossy-only pipeline is preserved exactly.pub compaction: Option<CompactionStage>,+ /// Optional CCR store. When set, the lossy path stashes the **full+ /// original** array into the store keyed by `ccr_hash` before+ /// returning — the runtime can then serve dropped rows back via+ /// retrieval tool calls. When `None`, hashes are still emitted but+ /// nothing is stored (legacy / parity mode).+ ///+ /// `Arc` so callers can keep their own handle to the same store+ /// (e.g. the proxy server holds it for retrieval lookups while+ /// SmartCrusher writes through it).+ pub ccr_store: Option<Arc<dyn CcrStore>>,}impl SmartCrusher {@@ -141,6 +154,7 @@ impl SmartCrusher {SmartCrusherBuilder::new(config).with_default_oss_setup().with_default_compaction()+ .with_default_ccr_store().build()}@@ -158,6 +172,7 @@ impl SmartCrusher {pub fn without_compaction(config: SmartCrusherConfig) -> Self {SmartCrusherBuilder::new(config).with_default_oss_setup()+ .with_default_ccr_store().build()}@@ -195,6 +210,7 @@ impl SmartCrusher {constraints: Vec<Box<dyn Constraint>>,observers: Vec<Box<dyn Observer>>,compaction: Option<CompactionStage>,+ ccr_store: Option<Arc<dyn CcrStore>>,) -> Self {SmartCrusher {config,@@ -204,9 +220,17 @@ impl SmartCrusher {constraints,observers,compaction,+ ccr_store,}}+ /// Handle to the CCR store, if configured. Used by the runtime+ /// (proxy server, PyO3 bridge) to look up originals when retrieval+ /// tool calls fire.+ pub fn ccr_store(&self) -> Option<&Arc<dyn CcrStore>> {+ self.ccr_store.as_ref()+ }+fn planner(&self) -> SmartCrusherPlanner<'_> {SmartCrusherPlanner::new(&self.config,@@ -544,10 +568,19 @@ impl SmartCrusher {let result = self.execute_plan(&plan, items);// Emit CCR-Dropped marker iff rows were actually dropped.+ // **This is the cornerstone of CCR's no-data-loss guarantee:**+ // we hash the full original, stash it in the configured store,+ // and emit a marker pointing at that hash. The runtime later+ // serves the original back via retrieval tool calls.let dropped_count = items.len().saturating_sub(result.len());let (ccr_hash, dropped_summary) = if dropped_count > 0 {let h = hash_array_for_ccr(items);let marker = format!("<<ccr:{h} {dropped_count}_rows_offloaded>>");+ if let Some(store) = &self.ccr_store {+ let canonical =+ serde_json::to_string(&Value::Array(items.to_vec())).unwrap_or_default();+ store.put(&h, &canonical);+ }(Some(h), marker)} else {(None, String::new())diff --git a/crates/headroom-core/tests/ccr_roundtrip.rs b/crates/headroom-core/tests/ccr_roundtrip.rsnew file mode 100644index 000000000..09b140451--- /dev/null+++ b/crates/headroom-core/tests/ccr_roundtrip.rs@@ -0,0 +1,224 @@+//! End-to-end CCR roundtrip: compress → store → retrieve → reconstruct.+//!+//! These tests pin **the cornerstone guarantee** of CCR: every row the+//! lossy path drops out of the prompt is recoverable from the CCR store+//! by hash. Lossy on the wire, lossless end-to-end.+//!+//! If any test in this file regresses, we are silently losing data —+//! the prompt advertises a `<<ccr:HASH ...>>` pointer that the runtime+//! cannot honor. That is the bug class these tests exist to catch.++use std::sync::Arc;++use serde_json::{json, Value};++use headroom_core::ccr::{CcrStore, InMemoryCcrStore};+use headroom_core::transforms::smart_crusher::{+ SmartCrusher, SmartCrusherBuilder, SmartCrusherConfig,+};++/// Force the lossy path: set the lossless savings threshold above 1.0+/// so no tabular rendering can ever clear it.+fn force_lossy_config() -> SmartCrusherConfig {+ SmartCrusherConfig {+ lossless_min_savings_ratio: 0.99,+ ..SmartCrusherConfig::default()+ }+}++/// Reasonably crushable fixture: low-uniqueness rows so the analyzer+/// is willing to compress, large enough to overshoot adaptive_k.+fn lossy_friendly_items(n: usize) -> Vec<Value> {+ (0..n).map(|i| json!({"id": i, "status": "ok"})).collect()+}++#[test]+fn default_crusher_stores_dropped_rows() {+ // The default `SmartCrusher::new()` ships with both lossless-first+ // compaction AND a CCR store (matches Python's default — CCR+ // enabled). So a real lossy crush should leave the original parked+ // in the store, retrievable by hash.+ let crusher = SmartCrusher::new(force_lossy_config());+ let items = lossy_friendly_items(50);++ let result = crusher.crush_array(&items, "", 1.0);++ let hash = result+ .ccr_hash+ .as_ref()+ .expect("lossy path must emit a hash");+ let store = crusher.ccr_store().expect("default crusher has a store");++ let retrieved = store.get(hash).expect("hash must resolve in the store");+ let parsed: Value = serde_json::from_str(&retrieved).expect("payload is valid JSON");+ assert_eq!(parsed, Value::Array(items.clone()), "roundtrip mismatch");+}++#[test]+fn without_compaction_also_stores_dropped_rows() {+ // The legacy / parity constructor still carries a default store —+ // CCR is the no-data-loss contract, not an opt-in extra.+ let crusher = SmartCrusher::without_compaction(SmartCrusherConfig::default());+ let items = lossy_friendly_items(30);++ let result = crusher.crush_array(&items, "", 1.0);++ if let Some(hash) = result.ccr_hash.as_ref() {+ let store = crusher.ccr_store().expect("default crusher has a store");+ let retrieved = store.get(hash).expect("hash must resolve");+ let parsed: Value = serde_json::from_str(&retrieved).unwrap();+ assert_eq!(parsed, Value::Array(items.clone()));+ }+}++#[test]+fn shared_external_store_sees_writes() {+ // Production wiring: the runtime owns the store; SmartCrusher+ // writes through it. The proxy keeps an `Arc` for retrieval; this+ // test models that arrangement.+ let store: Arc<dyn CcrStore> = Arc::new(InMemoryCcrStore::new());+ let crusher = SmartCrusherBuilder::new(force_lossy_config())+ .with_default_oss_setup()+ .with_default_compaction()+ .with_ccr_store(store.clone())+ .build();++ let items = lossy_friendly_items(40);+ let result = crusher.crush_array(&items, "", 1.0);+ let hash = result.ccr_hash.expect("lossy path emits a hash");++ let retrieved = store.get(&hash).expect("external store has the payload");+ let parsed: Value = serde_json::from_str(&retrieved).unwrap();+ assert_eq!(parsed, Value::Array(items));+}++#[test]+fn passthrough_does_not_write_to_store() {+ // Below adaptive_k: nothing dropped, nothing to recover, no store+ // write. Otherwise we'd accumulate noise on the hot passthrough+ // path.+ let crusher = SmartCrusher::new(SmartCrusherConfig::default());+ let store = crusher.ccr_store().unwrap().clone();+ let starting_len = store.len();++ let small = lossy_friendly_items(3);+ let result = crusher.crush_array(&small, "", 1.0);++ assert!(result.ccr_hash.is_none());+ assert_eq!(store.len(), starting_len, "no write expected");+}++#[test]+fn lossless_win_does_not_write_to_store() {+ // Lossless wins → nothing dropped, no CCR retrieval needed → no+ // store write. The store should only see writes when the prompt+ // actually loses data.+ let crusher = SmartCrusher::new(SmartCrusherConfig::default());+ let store = crusher.ccr_store().unwrap().clone();+ let starting_len = store.len();++ // Highly tabular fixture — lossless compaction should clear the+ // 30% savings threshold easily.+ let items: Vec<Value> = (0..50)+ .map(|i| json!({"id": i, "kind": "click", "ts": 1700000000 + i, "user": "alice"}))+ .collect();++ let result = crusher.crush_array(&items, "", 1.0);++ if result.compacted.is_some() {+ assert!(result.ccr_hash.is_none(), "lossless win → no hash");+ assert_eq!(store.len(), starting_len, "lossless win → no store write");+ }+}++#[test]+fn store_roundtrip_is_deterministic_across_calls() {+ // Same input crushed twice → same hash → store has exactly one+ // entry (not two), and it resolves to the same payload.+ let crusher = SmartCrusher::new(force_lossy_config());+ let store = crusher.ccr_store().unwrap().clone();+ let items = lossy_friendly_items(40);++ let r1 = crusher.crush_array(&items, "", 1.0);+ let len_after_first = store.len();++ let r2 = crusher.crush_array(&items, "", 1.0);+ assert_eq!(r1.ccr_hash, r2.ccr_hash);++ let hash = r1.ccr_hash.unwrap();+ assert_eq!(+ store.len(),+ len_after_first,+ "second call with same input must not grow the store"+ );++ let retrieved = store.get(&hash).unwrap();+ let parsed: Value = serde_json::from_str(&retrieved).unwrap();+ assert_eq!(parsed, Value::Array(items));+}++#[test]+fn distinct_inputs_produce_distinct_store_entries() {+ let crusher = SmartCrusher::new(force_lossy_config());+ let store = crusher.ccr_store().unwrap().clone();+ let starting_len = store.len();++ let a: Vec<Value> = (0..40).map(|i| json!({"id": i, "tag": "a"})).collect();+ let b: Vec<Value> = (0..40).map(|i| json!({"id": i, "tag": "b"})).collect();++ let ra = crusher.crush_array(&a, "", 1.0);+ let rb = crusher.crush_array(&b, "", 1.0);++ let ha = ra.ccr_hash.unwrap();+ let hb = rb.ccr_hash.unwrap();+ assert_ne!(ha, hb);++ // Both originals retrievable.+ let pa: Value = serde_json::from_str(&store.get(&ha).unwrap()).unwrap();+ let pb: Value = serde_json::from_str(&store.get(&hb).unwrap()).unwrap();+ assert_eq!(pa, Value::Array(a));+ assert_eq!(pb, Value::Array(b));+ assert_eq!(store.len(), starting_len + 2);+}++#[test]+fn dropped_summary_marker_points_at_stored_hash() {+ // The marker the LLM sees in the prompt encodes the same hash that+ // resolves the stored payload. Pin the format so the retrieval-tool+ // contract stays honest.+ let crusher = SmartCrusher::new(force_lossy_config());+ let store = crusher.ccr_store().unwrap().clone();+ let items = lossy_friendly_items(50);++ let result = crusher.crush_array(&items, "", 1.0);+ let hash = result.ccr_hash.as_ref().unwrap();++ assert!(+ result.dropped_summary.contains(&format!("<<ccr:{hash} ")),+ "marker {:?} must embed hash {}",+ result.dropped_summary,+ hash+ );+ // The hash actually resolves.+ assert!(store.get(hash).is_some());+}++#[test]+fn full_crush_pipeline_roundtrips_through_store() {+ // End-to-end through the public `crush()` API (the entry point+ // that ContentRouter calls). The result string contains the marker;+ // we verify the marker hash resolves in the store.+ let crusher = SmartCrusher::new(force_lossy_config());+ let store = crusher.ccr_store().unwrap().clone();++ let items: Vec<Value> = (0..50).map(|i| json!({"id": i, "status": "ok"})).collect();+ let raw_input = serde_json::to_string(&Value::Array(items.clone())).unwrap();++ let _ = crusher.crush(&raw_input, "", 1.0);++ // The pipeline routed through `process_value` → `crush_array`,+ // which calls our wired `put`. So the original is in the store+ // under the canonical hash.+ let store_len = store.len();+ assert!(store_len > 0, "expected at least one CCR store entry");+}diff --git a/crates/headroom-py/Cargo.toml b/crates/headroom-py/Cargo.tomlindex 9b014aaab..f509213ad 100644--- a/crates/headroom-py/Cargo.toml+++ b/crates/headroom-py/Cargo.toml@@ -24,3 +24,4 @@ extension-module = ["pyo3/extension-module"][dependencies]headroom-core = { path = "../headroom-core" }pyo3 = { workspace = true }+serde_json = { workspace = true }diff --git a/crates/headroom-py/src/lib.rs b/crates/headroom-py/src/lib.rsindex 09639a9f6..70197de8e 100644--- a/crates/headroom-py/src/lib.rs+++ b/crates/headroom-py/src/lib.rs@@ -23,6 +23,7 @@ use headroom_core::transforms::{DiffCompressionResult, DiffCompressor, DiffCompressorConfig, DiffCompressorStats,};use pyo3::prelude::*;+use pyo3::types::PyDict;/// Identity stub used by the Python smoke test to verify linkage.#[pyfunction]@@ -30,6 +31,42 @@ fn hello() -> &'static str {headroom_core::hello()}+fn type_name(v: &serde_json::Value) -> &'static str {+ match v {+ serde_json::Value::Null => "null",+ serde_json::Value::Bool(_) => "bool",+ serde_json::Value::Number(_) => "number",+ serde_json::Value::String(_) => "string",+ serde_json::Value::Array(_) => "array",+ serde_json::Value::Object(_) => "object",+ }+}++/// Build the dict returned by `SmartCrusher.crush_array_json`. Kept+/// outside `#[pymethods]` so we can `unwrap()` `set_item` (it cannot+/// fail when keys are static str literals and values are owned String /+/// Option<String> / Option<&'static str>) without tripping the+/// `clippy::useless_conversion` false positive that fires inside the+/// pyo3 0.22 method-attribute macro.+fn build_crush_array_dict<'py>(+ py: Python<'py>,+ kept_json: String,+ ccr_hash: Option<String>,+ dropped_summary: String,+ strategy_info: String,+ compacted: Option<String>,+ compaction_kind: Option<&'static str>,+) -> Bound<'py, PyDict> {+ let dict = PyDict::new_bound(py);+ dict.set_item("items", kept_json).unwrap();+ dict.set_item("ccr_hash", ccr_hash).unwrap();+ dict.set_item("dropped_summary", dropped_summary).unwrap();+ dict.set_item("strategy_info", strategy_info).unwrap();+ dict.set_item("compacted", compacted).unwrap();+ dict.set_item("compaction_kind", compaction_kind).unwrap();+ dict+}+// ─── DiffCompressorConfig ──────────────────────────────────────────────────/// Mirror of `headroom.transforms.diff_compressor.DiffCompressorConfig`.@@ -621,6 +658,74 @@ impl PySmartCrusher {fn smart_crush_content(&self, content: &str, query: &str, bias: f64) -> (String, bool, String) {self.inner.smart_crush_content(content, query, bias)}++ /// Crush a JSON array directly and return the structured result.+ ///+ /// Input is a JSON string holding an array (`[item, item, ...]`).+ /// Returns a dict with:+ /// - `items`: JSON array string of the kept rows after compression+ /// - `ccr_hash`: 12-char hash if rows were dropped, else `None`+ /// - `dropped_summary`: `<<ccr:HASH N_rows_offloaded>>` marker+ /// text, empty if nothing dropped+ /// - `strategy_info`: debug string describing what ran (e.g.+ /// `"smart_sample"`, `"lossless:table"`, `"none:adaptive_at_limit"`)+ /// - `compacted`: rendered bytes when the lossless path won, else `None`+ /// - `compaction_kind`: `"table" | "buckets" | "ccr" | None`+ ///+ /// This surfaces `CrushArrayResult` to Python so tests and the proxy+ /// runtime can reach the CCR hash directly (rather than parsing it+ /// out of the prompt marker).+ #[pyo3(signature = (items_json, query = "", bias = 1.0))]+ fn crush_array_json<'py>(+ &self,+ py: Python<'py>,+ items_json: &str,+ query: &str,+ bias: f64,+ ) -> Bound<'py, PyDict> {+ // Errors here surface as Python `RuntimeError` via pyo3's panic+ // catcher — callers are expected to pass valid array-shaped JSON.+ let parsed: serde_json::Value = serde_json::from_str(items_json)+ .unwrap_or_else(|e| panic!("items_json must be JSON: {e}"));+ let items = match parsed {+ serde_json::Value::Array(a) => a,+ other => panic!("items_json must be a JSON array, got {}", type_name(&other)),+ };+ let result = self.inner.crush_array(&items, query, bias);+ let kept_json = serde_json::to_string(&serde_json::Value::Array(result.items))+ .expect("serialize kept items");+ build_crush_array_dict(+ py,+ kept_json,+ result.ccr_hash,+ result.dropped_summary,+ result.strategy_info,+ result.compacted,+ result.compaction_kind,+ )+ }++ /// Look up an original payload by CCR hash.+ ///+ /// When the lossy path drops rows, it stashes the **full original**+ /// array into the in-memory CCR store keyed by the 12-char hash+ /// embedded in the prompt's `<<ccr:HASH ...>>` marker. The runtime+ /// (proxy server / MCP retrieval tool) calls this to serve the+ /// dropped rows back to the LLM on demand.+ ///+ /// Returns the canonical-JSON serialization of the original+ /// `[item, item, ...]` array, or `None` if the hash is unknown,+ /// expired, or the crusher was constructed without a CCR store.+ fn ccr_get(&self, hash: &str) -> Option<String> {+ self.inner.ccr_store().and_then(|s| s.get(hash))+ }++ /// Number of entries currently held by the CCR store. `0` if no+ /// store is configured. Informational; use it from tests and+ /// telemetry, not from the retrieval hot path.+ fn ccr_len(&self) -> usize {+ self.inner.ccr_store().map(|s| s.len()).unwrap_or(0)+ }}// ─── Module init ───────────────────────────────────────────────────────────diff --git a/headroom/transforms/smart_crusher.py b/headroom/transforms/smart_crusher.pyindex 18e2ee19f..3dc7819c3 100644--- a/headroom/transforms/smart_crusher.py+++ b/headroom/transforms/smart_crusher.py@@ -195,6 +195,44 @@ class SmartCrusher(Transform):strategy=r.strategy,)+ def crush_array_json(+ self,+ items_json: str,+ query: str = "",+ bias: float = 1.0,+ ) -> dict[str, Any]:+ """Crush a JSON array directly and surface the structured result.++ Returns a dict with `items` (kept rows as JSON), `ccr_hash` (12-char+ hash if rows were dropped), `dropped_summary` (the marker text),+ `strategy_info`, `compacted` (rendered bytes when lossless won),+ and `compaction_kind`.++ Used by tests and by the proxy's CCR retrieval flow when it needs+ the hash directly rather than parsing it out of a prompt marker.+ """+ result: dict[str, Any] = self._rust.crush_array_json(items_json, query, bias)+ return result++ def ccr_get(self, hash_key: str) -> str | None:+ """Look up an original payload by CCR hash from the Rust store.++ Returns the canonical-JSON serialization of the original+ `[item, item, ...]` array that the lossy path stashed before+ emitting `<<ccr:HASH ...>>`. Returns ``None`` if the hash is+ unknown, expired, or no store is configured.++ Used by the proxy's CCR retrieval tool to serve the dropped+ rows back to the LLM on demand.+ """+ result: str | None = self._rust.ccr_get(hash_key)+ return result++ def ccr_len(self) -> int:+ """Number of entries currently held by the Rust CCR store."""+ n: int = self._rust.ccr_len()+ return n+def _smart_crush_content(self,content: str,diff --git a/plugins/headroom-agent-hooks/.claude-plugin/plugin.json b/plugins/headroom-agent-hooks/.claude-plugin/plugin.jsonindex 5413f88ea..569eb4505 100644--- a/plugins/headroom-agent-hooks/.claude-plugin/plugin.json+++ b/plugins/headroom-agent-hooks/.claude-plugin/plugin.json@@ -1,6 +1,6 @@{"name": "headroom",- "version": "0.11.0",+ "version": "0.13.2","description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.","author": {"name": "Headroom Contributors",diff --git a/plugins/headroom-agent-hooks/.github/plugin/plugin.json b/plugins/headroom-agent-hooks/.github/plugin/plugin.jsonindex 8b69de7cb..c9a3de90f 100644--- a/plugins/headroom-agent-hooks/.github/plugin/plugin.json+++ b/plugins/headroom-agent-hooks/.github/plugin/plugin.json@@ -1,6 +1,6 @@{"name": "headroom",- "version": "0.11.0",+ "version": "0.13.2","description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.","author": {"name": "Headroom Contributors",diff --git a/tests/test_transforms/test_smart_crusher_ccr_roundtrip.py b/tests/test_transforms/test_smart_crusher_ccr_roundtrip.pynew file mode 100644index 000000000..69018fe04--- /dev/null+++ b/tests/test_transforms/test_smart_crusher_ccr_roundtrip.py@@ -0,0 +1,264 @@+"""End-to-end CCR roundtrip via the Python bridge.++The Rust core integration test (`crates/headroom-core/tests/ccr_roundtrip.rs`)+already pins the contract from the Rust side. These tests verify the same+guarantee is reachable from Python — i.e. the Rust-side CCR store is+exposed correctly through PyO3, the runtime can read originals back via+`ccr_get`, and the wiring from the Python `SmartCrusher` shim hits the+same store the Rust crate writes through.++If these regress, the Python proxy's CCR retrieval tool is silently+serving nothing — the `<<ccr:HASH ...>>` marker would point at a void.+"""++from __future__ import annotations++import json++import pytest+++def _build_extension() -> None:+ try:+ from headroom._core import SmartCrusher # noqa: F401+ except ImportError:+ pytest.skip(+ "headroom._core not built — run `bash scripts/build_rust_extension.sh`",+ allow_module_level=True,+ )+++_build_extension()+++def _force_lossy_config():+ """Force the lossy path: lossless threshold above 1.0 means no+ rendering can ever clear it, so `crush_array` falls through."""+ from headroom._core import SmartCrusherConfig++ return SmartCrusherConfig(lossless_min_savings_ratio=0.99)+++# ─── Native PyO3 surface ───────────────────────────────────────────────────+++def test_native_default_crusher_has_a_store() -> None:+ """Default constructor wires up the in-memory CCR store. Empty+ until we crush something."""+ from headroom._core import SmartCrusher++ crusher = SmartCrusher()+ assert crusher.ccr_len() == 0+ assert crusher.ccr_get("anything") is None+++def test_native_lossy_crush_stores_original() -> None:+ """The cornerstone roundtrip: lossy crush → store entry → retrieve+ → original payload comes back intact."""+ from headroom._core import SmartCrusher++ crusher = SmartCrusher(_force_lossy_config())+ items = [{"id": i, "status": "ok"} for i in range(50)]+ content = json.dumps(items)++ result = crusher.crush(content, "", 1.0)++ # Some store activity is expected — the lossy path fires through+ # `crush_array` which stashes the original.+ assert crusher.ccr_len() > 0, (+ f"expected store entries after lossy crush, got 0; "+ f"strategy={result.strategy!r} compressed_len={len(result.compressed)}"+ )+++def test_native_ccr_get_recovers_original_array() -> None:+ """Pull the hash out of the marker that ends up in the strategy+ string and verify the store actually returns the original list."""+ from headroom._core import SmartCrusher++ crusher = SmartCrusher(_force_lossy_config())+ items = [{"id": i, "status": "ok"} for i in range(50)]+ content = json.dumps(items)++ crusher.crush(content, "", 1.0)++ # The store should have at least one entry; iterate likely hashes+ # is impossible (no list API) so we walk the canonical hash space+ # by recomputing what the Rust side does. Easier: just verify that+ # *something* round-trips by re-crushing identical input — same+ # hash, same payload, no growth in store size.+ pre_len = crusher.ccr_len()+ crusher.crush(content, "", 1.0)+ assert crusher.ccr_len() == pre_len, (+ "identical re-crush should be idempotent under the same hash"+ )+++def test_native_passthrough_does_not_grow_store() -> None:+ """Below adaptive_k → no drop → no store write."""+ from headroom._core import SmartCrusher++ crusher = SmartCrusher()+ pre = crusher.ccr_len()++ small = json.dumps([{"id": i} for i in range(3)])+ crusher.crush(small, "", 1.0)++ assert crusher.ccr_len() == pre+++# ─── Python shim surface ───────────────────────────────────────────────────+#+# The Python `SmartCrusher` class wraps the Rust crusher and exposes the+# same `ccr_get` / `ccr_len` passthrough. The proxy server uses that+# shim, not the raw `_core` class, so this needs to work too.+++def test_shim_exposes_ccr_get_and_ccr_len() -> None:+ from headroom.config import SmartCrusherConfig as PyConfig+ from headroom.transforms.smart_crusher import SmartCrusher++ crusher = SmartCrusher(PyConfig(), with_compaction=False)+ assert crusher.ccr_len() == 0+ assert crusher.ccr_get("missing") is None+++def test_shim_lossy_crush_populates_store() -> None:+ """Same roundtrip as the native test but driven through the+ `headroom.transforms.smart_crusher.SmartCrusher` shim — the path+ the proxy actually uses."""+ from headroom.config import SmartCrusherConfig as PyConfig+ from headroom.transforms.smart_crusher import SmartCrusher++ # The shim doesn't currently surface `lossless_min_savings_ratio`+ # in `PyConfig`. Use `with_compaction=False` to skip lossless+ # entirely and force the lossy path.+ crusher = SmartCrusher(PyConfig(), with_compaction=False)+ items = [{"id": i, "status": "ok"} for i in range(50)]+ content = json.dumps(items)++ crusher.crush(content, "", 1.0)++ # The lossy path should have stashed at least one original.+ assert crusher.ccr_len() > 0+++# ─── Explicit before/after roundtrip ───────────────────────────────────────+#+# These tests do the full user story end-to-end: take a payload,+# crush it, fetch the original back from the CCR store by hash, and+# assert the reconstructed list **equals the input element-for-element**.+# If anything in compress → store → retrieve → reconstruct breaks,+# these tests yell loudly with both the before and after visible in+# the failure message.+++def test_explicit_before_after_roundtrip_native() -> None:+ """Full story: payload → crush → grab hash from result → ccr_get+ → parse → byte-compare with original input."""+ from headroom._core import SmartCrusher, SmartCrusherConfig++ # Force the lossy path so the CCR store actually gets a write.+ cfg = SmartCrusherConfig(lossless_min_savings_ratio=0.99)+ crusher = SmartCrusher(cfg)++ # The "before" payload — what the tool produced and the proxy is+ # about to send to the LLM.+ original = [{"id": i, "status": "ok", "tag": "alpha"} for i in range(60)]+ original_json = json.dumps(original)++ # 1. Crush.+ result = crusher.crush_array_json(original_json, "", 1.0)+ assert result["ccr_hash"] is not None, (+ f"expected lossy drop, got strategy={result['strategy_info']!r}"+ )+ hash_key = result["ccr_hash"]+ assert hash_key in result["dropped_summary"], (+ f"marker {result['dropped_summary']!r} should embed hash {hash_key}"+ )++ # 2. Retrieve by hash.+ retrieved_json = crusher.ccr_get(hash_key)+ assert retrieved_json is not None, f"hash {hash_key} not in store"++ # 3. Parse + compare element-for-element with the input.+ retrieved = json.loads(retrieved_json)+ assert retrieved == original, (+ f"roundtrip mismatch:\n"+ f" before ({len(original)} items): {original[:3]!r}...\n"+ f" after ({len(retrieved)} items): {retrieved[:3]!r}..."+ )+ assert len(retrieved) == len(original)+++def test_explicit_before_after_roundtrip_shim() -> None:+ """Same story, but through the Python shim (the proxy's actual+ entry point). Pins that nothing gets lost across the bridge."""+ from headroom.config import SmartCrusherConfig as PyConfig+ from headroom.transforms.smart_crusher import SmartCrusher++ crusher = SmartCrusher(PyConfig(), with_compaction=False)++ original = [{"event_id": f"e{i}", "user": f"u{i % 5}", "action": "click"} for i in range(40)]+ original_json = json.dumps(original)++ result = crusher.crush_array_json(original_json)+ assert result["ccr_hash"] is not None, result["strategy_info"]+ hash_key = result["ccr_hash"]++ retrieved_json = crusher.ccr_get(hash_key)+ assert retrieved_json is not None+ retrieved = json.loads(retrieved_json)++ # Element-for-element equality + length match.+ assert retrieved == original+ assert len(retrieved) == len(original)+ # Spot-check a specific item to make the contract tangible.+ assert retrieved[0] == {"event_id": "e0", "user": "u0", "action": "click"}+ assert retrieved[-1] == {"event_id": "e39", "user": "u4", "action": "click"}+++def test_kept_subset_is_subset_of_original() -> None:+ """The compressed view (what the LLM sees inline) is a proper+ subset of the original. Combined with `ccr_get` returning the+ full original, this proves: nothing is invented, nothing is lost."""+ from headroom._core import SmartCrusher, SmartCrusherConfig++ crusher = SmartCrusher(SmartCrusherConfig(lossless_min_savings_ratio=0.99))+ original = [{"id": i, "status": "ok"} for i in range(50)]++ result = crusher.crush_array_json(json.dumps(original))+ kept = json.loads(result["items"])++ assert len(kept) < len(original), "lossy path should drop rows"+ # Every kept row exists verbatim in the original.+ for item in kept:+ assert item in original, f"invented row: {item!r}"+ # And the original is fully recoverable from the store.+ retrieved = json.loads(crusher.ccr_get(result["ccr_hash"]))+ assert retrieved == original+++def test_distinct_payloads_have_distinct_hashes_and_separate_storage() -> None:+ """Two different payloads → two different hashes → both+ independently retrievable. Pins the per-payload isolation."""+ from headroom._core import SmartCrusher, SmartCrusherConfig++ crusher = SmartCrusher(SmartCrusherConfig(lossless_min_savings_ratio=0.99))++ a = [{"id": i, "tag": "alpha"} for i in range(50)]+ b = [{"id": i, "tag": "beta"} for i in range(50)]++ ra = crusher.crush_array_json(json.dumps(a))+ rb = crusher.crush_array_json(json.dumps(b))++ assert ra["ccr_hash"] != rb["ccr_hash"]++ # Each hash resolves to its own original.+ pa = json.loads(crusher.ccr_get(ra["ccr_hash"]))+ pb = json.loads(crusher.ccr_get(rb["ccr_hash"]))+ assert pa == a+ assert pb == b+ # And they don't cross-contaminate.+ assert pa != b+ assert pb != a