Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(rust): SmartCrusher struct — execute_plan + crush_array + crush_mixed_array
chopratejas4c44b95d9b14
diff --git a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs b/crates/headroom-core/src/transforms/smart_crusher/crusher.rsnew file mode 100644index 000000000..d1fb43ca0--- /dev/null+++ b/crates/headroom-core/src/transforms/smart_crusher/crusher.rs@@ -0,0 +1,593 @@+//! `SmartCrusher` struct — top-level entry point for compression.+//!+//! Owns the `config`, `anchor_selector`, `scorer`, and `analyzer`+//! singletons that every per-message call needs. Constructed once+//! per process; the struct is `Send + Sync` so it can sit behind an+//! `Arc` in a multi-threaded proxy.+//!+//! This module ports three Python entry points:+//!+//! - `_execute_plan` (line 3617) → `SmartCrusher::execute_plan`+//! - `_crush_array` (line 2400) → `SmartCrusher::crush_array`+//! - `_crush_mixed_array` (line 2914) → `SmartCrusher::crush_mixed_array`+//!+//! # Stubs that match Python's "everything-disabled" path+//!+//! Python's `_crush_array` calls into TOIN (cross-user pattern+//! learning), feedback (per-tool compression hints), CCR (compress-+//! cache-retrieve store), and telemetry. All four are large separate+//! systems with their own state. For the like-for-like port at Stage+//! 3c.1, we mirror Python's behavior **when those subsystems are+//! disabled**:+//!+//! - **TOIN**: never produces a recommendation, never overrides+//! `effective_max_items`, never injects preserve_fields/strategy/level.+//! - **Feedback**: never produces hints; default `effective_max_items`.+//! - **CCR**: `enabled=false`; result has `ccr_hash = None`.+//! - **Telemetry**: no-op.+//! - **`_compress_text_within_items`**: pass-through (returns input+//! unchanged) since text compression has its own port pipeline.+//! - **`summarize_dropped_items`**: empty string.+//!+//! Parity fixtures will be recorded with all four disabled on the+//! Python side, locking byte-equal output. The TOIN/CCR/feedback+//! integration ports happen later (Stage 3c.2 follow-ups).++use serde_json::Value;++use super::analyzer::SmartAnalyzer;+use super::config::SmartCrusherConfig;+use super::crushers::{compute_k_split, crush_string_array};+use super::planning::SmartCrusherPlanner;+use super::types::{CompressionPlan, CompressionStrategy};+use crate::relevance::{HybridScorer, RelevanceScorer};+use crate::transforms::adaptive_sizer::compute_optimal_k;+use crate::transforms::anchor_selector::{AnchorConfig, AnchorSelector};++/// Return type for `crush_array` — mirrors Python's+/// `(crushed_items, strategy_info, ccr_hash, dropped_summary)` tuple.+pub struct CrushArrayResult {+ pub items: Vec<Value>,+ /// Strategy debug string, e.g. `"smart_sample"`, `"top_n"`,+ /// `"none:adaptive_at_limit"`, `"skip:unique_entities_no_signal"`.+ pub strategy_info: String,+ /// CCR retrieval hash if caching is enabled. Stub: always `None`.+ pub ccr_hash: Option<String>,+ /// Categorical summary of dropped items. Stub: always empty.+ pub dropped_summary: String,+}++/// Top-level SmartCrusher.+pub struct SmartCrusher {+ pub config: SmartCrusherConfig,+ pub anchor_selector: AnchorSelector,+ pub scorer: Box<dyn RelevanceScorer + Send + Sync>,+ pub analyzer: SmartAnalyzer,+}++impl SmartCrusher {+ /// Construct with the default scorer (`HybridScorer`) and the+ /// default `AnchorConfig`. Mirrors Python's+ /// `SmartCrusher(config=...)` no-extra-args path with all+ /// optional subsystems disabled.+ pub fn new(config: SmartCrusherConfig) -> Self {+ let analyzer = SmartAnalyzer::new(config.clone());+ SmartCrusher {+ config,+ anchor_selector: AnchorSelector::new(AnchorConfig::default()),+ scorer: Box::new(HybridScorer::default()),+ analyzer,+ }+ }++ /// Construct with a custom scorer.+ pub fn with_scorer(+ config: SmartCrusherConfig,+ scorer: Box<dyn RelevanceScorer + Send + Sync>,+ ) -> Self {+ let analyzer = SmartAnalyzer::new(config.clone());+ SmartCrusher {+ config,+ anchor_selector: AnchorSelector::new(AnchorConfig::default()),+ scorer,+ analyzer,+ }+ }++ fn planner(&self) -> SmartCrusherPlanner<'_> {+ SmartCrusherPlanner::new(+ &self.config,+ &self.anchor_selector,+ &*self.scorer,+ &self.analyzer,+ )+ }++ /// Execute a `CompressionPlan` against `items`, returning the+ /// kept-items list in original-array order. Mirrors Python's+ /// `_execute_plan` (line 3617-3633).+ ///+ /// Schema-preserving: each kept item is cloned unchanged. No+ /// summary objects, generated fields, or wrapper metadata.+ pub fn execute_plan(&self, plan: &CompressionPlan, items: &[Value]) -> Vec<Value> {+ let mut indices = plan.keep_indices.clone();+ indices.sort_unstable();+ indices+ .into_iter()+ .filter(|&idx| idx < items.len())+ .map(|idx| items[idx].clone())+ .collect()+ }++ /// Compress an array of dict items.+ ///+ /// Direct port of `_crush_array` (Python line 2400-2687) with the+ /// optional subsystems (TOIN / CCR / feedback / telemetry) wired+ /// in their disabled-by-default behavior. See module-level docs+ /// for the rationale.+ ///+ /// # Pipeline+ ///+ /// 1. Compute `item_strings` once (used as input to adaptive+ /// sizing and downstream relevance scoring).+ /// 2. `compute_optimal_k` → `adaptive_k`.+ /// 3. If `n <= adaptive_k`, return passthrough.+ /// 4. `analyzer.analyze_array(items)` → `analysis`.+ /// 5. If `analysis.recommended_strategy == Skip`, return passthrough+ /// with a `skip:<reason>` strategy string.+ /// 6. `planner.create_plan(analysis, items, query_context, ...)`.+ /// 7. `execute_plan(plan, items)` → result.+ /// 8. Strategy info = `analysis.recommended_strategy.as_str()`.+ pub fn crush_array(&self, items: &[Value], query_context: &str, bias: f64) -> CrushArrayResult {+ let item_strings: Vec<String> = items+ .iter()+ .map(|i| serde_json::to_string(i).unwrap_or_default())+ .collect();+ let item_str_refs: Vec<&str> = item_strings.iter().map(|s| s.as_str()).collect();++ let max_k = if self.config.max_items_after_crush > 0 {+ Some(self.config.max_items_after_crush)+ } else {+ None+ };+ let adaptive_k = compute_optimal_k(&item_str_refs, bias, 3, max_k);++ // Tier-1 boundary: array already small enough.+ if items.len() <= adaptive_k {+ // Python branches into _compress_text_within_items here;+ // stubbed to passthrough (no within-item text compression+ // in this stage).+ return CrushArrayResult {+ items: items.to_vec(),+ strategy_info: "none:adaptive_at_limit".to_string(),+ ccr_hash: None,+ dropped_summary: String::new(),+ };+ }++ // TOIN/feedback paths stubbed → effective_max_items = adaptive_k.+ let effective_max_items = adaptive_k;++ let analysis = self.analyzer.analyze_array(items);++ // Crushability gate: not safe to crush → return original.+ if analysis.recommended_strategy == CompressionStrategy::Skip {+ let reason = match &analysis.crushability {+ Some(c) => format!("skip:{}", c.reason),+ None => String::new(),+ };+ return CrushArrayResult {+ items: items.to_vec(),+ strategy_info: reason,+ ccr_hash: None,+ dropped_summary: String::new(),+ };+ }++ // Plan + execute.+ let plan = self.planner().create_plan(+ &analysis,+ items,+ query_context,+ None, // preserve_fields (TOIN — stubbed)+ Some(effective_max_items),+ Some(&item_strings),+ );+ let result = self.execute_plan(&plan, items);++ // CCR/telemetry/TOIN-record paths stubbed.+ let strategy_info = analysis.recommended_strategy.as_str().to_string();++ CrushArrayResult {+ items: result,+ strategy_info,+ ccr_hash: None,+ dropped_summary: String::new(),+ }+ }++ /// Compress a mixed-type array by grouping items by type and+ /// compressing each group with the appropriate handler.+ ///+ /// Direct port of `_crush_mixed_array` (Python line 2914-3013).+ ///+ /// Strategy:+ /// 1. Group by type (dict / str / number / list / null / bool / other).+ /// 2. For groups with >= `min_items_to_analyze` items: apply the+ /// type-specific compressor.+ /// 3. For small groups: keep all items.+ /// 4. Reassemble in original order.+ ///+ /// Returns `(crushed_items, strategy_string)`.+ pub fn crush_mixed_array(+ &self,+ items: &[Value],+ query_context: &str,+ bias: f64,+ ) -> (Vec<Value>, String) {+ let n = items.len();+ if n <= 8 {+ return (items.to_vec(), "mixed:passthrough".to_string());+ }++ // Group by type, tracking original indices.+ let mut groups: GroupBuckets = GroupBuckets::default();+ for (i, item) in items.iter().enumerate() {+ groups.push(group_key(item), i, item.clone());+ }++ let mut keep_indices: std::collections::BTreeSet<usize> =+ std::collections::BTreeSet::new();+ let mut strategy_parts: Vec<String> = Vec::new();++ for (type_key, indices, values) in groups.into_iter() {+ // Small groups: keep all items.+ if values.len() < self.config.min_items_to_analyze {+ keep_indices.extend(&indices);+ continue;+ }++ match type_key {+ "dict" => {+ let CrushArrayResult {+ items: crushed, ..+ } = self.crush_array(&values, query_context, bias);+ // Find which original indices survived by matching+ // canonical-JSON serialization. Mirrors Python's+ // `json.dumps(c, sort_keys=True, default=str)`-keyed+ // set match.+ let crushed_keys: std::collections::HashSet<String> = crushed+ .iter()+ .map(canonical_json_for_match)+ .collect();+ for (i, idx) in indices.iter().enumerate() {+ if crushed_keys.contains(&canonical_json_for_match(&values[i])) {+ keep_indices.insert(*idx);+ }+ }+ strategy_parts.push(format!("dict:{}->{}", values.len(), crushed.len()));+ }+ "str" => {+ let strs: Vec<&str> = values.iter().filter_map(|v| v.as_str()).collect();+ let (crushed, _) = crush_string_array(&strs, &self.config, bias);+ let crushed_set: std::collections::HashSet<&str> =+ crushed.iter().map(|s| s.as_str()).collect();+ for (i, idx) in indices.iter().enumerate() {+ if let Some(s) = values[i].as_str() {+ if crushed_set.contains(s) {+ keep_indices.insert(*idx);+ }+ }+ }+ strategy_parts.push(format!("str:{}->{}", values.len(), crushed.len()));+ }+ "number" => {+ // Python: just adaptive sampling + outlier detection+ // (no summary prefix). Keeps first/last by index+ // and items >variance_threshold σ from mean.+ let item_strings: Vec<String> =+ values.iter().map(|v| v.to_string()).collect();+ let item_refs: Vec<&str> =+ item_strings.iter().map(|s| s.as_str()).collect();+ let (_kt, kf, kl, _) = compute_k_split(&item_refs, &self.config, bias);++ let kf = kf.min(values.len());+ let kl = kl.min(values.len().saturating_sub(kf));+ let first_idx: Vec<usize> = indices.iter().take(kf).copied().collect();+ let last_idx: Vec<usize> = indices+ .iter()+ .rev()+ .take(kl)+ .copied()+ .collect::<Vec<_>>();+ keep_indices.extend(&first_idx);+ keep_indices.extend(&last_idx);++ // Outliers via finite-only stats.+ let finite: Vec<f64> = values+ .iter()+ .filter_map(|v| v.as_f64().filter(|f| f.is_finite()))+ .collect();+ if finite.len() > 1 {+ if let Some(mean_v) =+ super::stats_math::mean(&finite)+ {+ if let Some(std_v) =+ super::stats_math::sample_stdev(&finite)+ {+ if std_v > 0.0 {+ let threshold = self.config.variance_threshold * std_v;+ for (i, val) in values.iter().enumerate() {+ if let Some(num) =+ val.as_f64().filter(|f| f.is_finite())+ {+ if (num - mean_v).abs() > threshold {+ keep_indices.insert(indices[i]);+ }+ }+ }+ }+ }+ }+ }+ strategy_parts.push(format!("num:{}", values.len()));+ }+ _ => {+ // list / bool / none / other → keep all items.+ keep_indices.extend(&indices);+ }+ }+ }++ // Reassemble in original order.+ let result: Vec<Value> = keep_indices.iter().map(|&i| items[i].clone()).collect();+ let strategy = format!(+ "mixed:adaptive({}->{},{})",+ n,+ result.len(),+ strategy_parts.join(",")+ );+ (result, strategy)+ }+}++// ---------- helpers ----------++/// Group key that mirrors Python's `_crush_mixed_array` switch on+/// `isinstance`. Note the bool-before-number ordering: in Python, bool+/// is a subclass of int, but JSON treats them as distinct types, so we+/// don't have the Python ordering hazard.+fn group_key(item: &Value) -> &'static str {+ match item {+ Value::Object(_) => "dict",+ Value::String(_) => "str",+ Value::Bool(_) => "bool",+ Value::Number(_) => "number",+ Value::Array(_) => "list",+ Value::Null => "none",+ }+}++/// Group buckets keyed by the type-string. Preserves first-occurrence+/// order across keys so dict/str/number/list/none/bool always come out+/// in the same order — matters because `keep_indices` is built+/// incrementally and Python iterates `groups.items()` (insertion order+/// in 3.7+).+#[derive(Default)]+struct GroupBuckets {+ entries: Vec<(&'static str, Vec<usize>, Vec<Value>)>,+ index_of: std::collections::HashMap<&'static str, usize>,+}++impl GroupBuckets {+ fn push(&mut self, key: &'static str, idx: usize, value: Value) {+ match self.index_of.get(key).copied() {+ Some(i) => {+ self.entries[i].1.push(idx);+ self.entries[i].2.push(value);+ }+ None => {+ self.index_of.insert(key, self.entries.len());+ self.entries.push((key, vec![idx], vec![value]));+ }+ }+ }+}++impl IntoIterator for GroupBuckets {+ type Item = (&'static str, Vec<usize>, Vec<Value>);+ type IntoIter = std::vec::IntoIter<Self::Item>;+ fn into_iter(self) -> Self::IntoIter {+ self.entries.into_iter()+ }+}++/// Serialize a `Value` for membership comparison. Mirrors Python's+/// `json.dumps(c, sort_keys=True, default=str)` used by+/// `_crush_mixed_array` to match crushed dict items back to their+/// original indices. The `default=str` fallback only matters for+/// non-JSON-serializable Python values; in serde_json land everything+/// is already JSON-native, so plain canonical JSON suffices.+fn canonical_json_for_match(value: &Value) -> String {+ crate::transforms::anchor_selector::python_json_dumps_sort_keys(value)+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ fn crusher() -> SmartCrusher {+ SmartCrusher::new(SmartCrusherConfig::default())+ }++ // ---------- execute_plan ----------++ #[test]+ fn execute_plan_empty_indices_returns_empty() {+ let c = crusher();+ let plan = CompressionPlan::default();+ let items: Vec<Value> = (0..5).map(|i| json!({"id": i})).collect();+ let result = c.execute_plan(&plan, &items);+ assert!(result.is_empty());+ }++ #[test]+ fn execute_plan_returns_items_in_sorted_index_order() {+ let c = crusher();+ let items: Vec<Value> = (0..10).map(|i| json!({"id": i})).collect();+ let plan = CompressionPlan {+ keep_indices: vec![5, 2, 8, 0],+ ..CompressionPlan::default()+ };+ let result = c.execute_plan(&plan, &items);+ assert_eq!(result.len(), 4);+ assert_eq!(result[0]["id"], 0);+ assert_eq!(result[1]["id"], 2);+ assert_eq!(result[2]["id"], 5);+ assert_eq!(result[3]["id"], 8);+ }++ #[test]+ fn execute_plan_skips_out_of_bounds() {+ let c = crusher();+ let items: Vec<Value> = (0..3).map(|i| json!({"id": i})).collect();+ let plan = CompressionPlan {+ keep_indices: vec![0, 5, 2],+ ..CompressionPlan::default()+ };+ let result = c.execute_plan(&plan, &items);+ assert_eq!(result.len(), 2);+ }++ // ---------- crush_array ----------++ #[test]+ fn crush_array_passthrough_when_below_adaptive_k() {+ let c = crusher();+ let items: Vec<Value> = (0..3).map(|i| json!({"id": i})).collect();+ let result = c.crush_array(&items, "", 1.0);+ assert_eq!(result.items.len(), 3);+ assert_eq!(result.strategy_info, "none:adaptive_at_limit");+ assert!(result.ccr_hash.is_none());+ }++ #[test]+ fn crush_array_skip_path_returns_original_items() {+ // 30 unique dict items with ID-like fields → analyzer should+ // detect "unique_entities_no_signal" and SKIP.+ let c = crusher();+ let items: Vec<Value> = (0..30)+ .map(|i| json!({"id": i, "name": format!("user_{}", i)}))+ .collect();+ let result = c.crush_array(&items, "", 1.0);+ // skip path returns the original items unchanged.+ assert_eq!(result.items.len(), 30);+ assert!(+ result.strategy_info.starts_with("skip:"),+ "expected skip:..., got {}",+ result.strategy_info+ );+ }++ #[test]+ fn crush_array_low_uniqueness_compresses() {+ // 30 items with status=ok across all → low_uniqueness path+ // (crushable, smart_sample strategy).+ let c = crusher();+ let items: Vec<Value> = (0..30).map(|_| json!({"status": "ok"})).collect();+ let result = c.crush_array(&items, "", 1.0);+ assert!(+ result.items.len() <= 30,+ "should not exceed original count"+ );+ }++ #[test]+ fn crush_array_keeps_error_items() {+ let c = crusher();+ let mut items: Vec<Value> = (0..30)+ .map(|i| json!({"id": i, "status": "ok"}))+ .collect();+ items.push(json!({"id": 30, "status": "error", "msg": "FATAL"}));+ let result = c.crush_array(&items, "", 1.0);+ // Whatever path is taken, the error item should survive.+ assert!(+ result.items.iter().any(|item| {+ item.get("status").and_then(|v| v.as_str()) == Some("error")+ }),+ "error item must survive crush_array"+ );+ }++ // ---------- crush_mixed_array ----------++ #[test]+ fn crush_mixed_passthrough_at_threshold() {+ let c = crusher();+ let items: Vec<Value> = vec![+ json!(1),+ json!("two"),+ json!({"k": "v"}),+ json!([1, 2]),+ json!(null),+ json!(true),+ json!(3),+ json!("four"),+ ];+ let (result, strat) = c.crush_mixed_array(&items, "", 1.0);+ assert_eq!(result.len(), 8);+ assert_eq!(strat, "mixed:passthrough");+ }++ #[test]+ fn crush_mixed_groups_and_compresses_dicts() {+ let c = crusher();+ // 25 dicts (large group → gets crushed) + 5 strings (small group → all kept).+ let mut items: Vec<Value> = (0..25)+ .map(|i| json!({"id": i, "status": "ok"}))+ .collect();+ for i in 0..5 {+ items.push(json!(format!("string_{}", i)));+ }+ let (result, strat) = c.crush_mixed_array(&items, "", 1.0);+ assert!(strat.starts_with("mixed:adaptive("));+ // The 5 strings (small group) all survive.+ let str_count = result.iter().filter(|v| v.is_string()).count();+ assert_eq!(str_count, 5);+ }++ #[test]+ fn crush_mixed_keeps_lists_and_nulls_unchanged() {+ let c = crusher();+ let mut items: Vec<Value> = vec![json!([1, 2]); 6];+ items.extend(vec![json!(null); 6]);+ items.extend(vec![json!({"k": 1}); 10]);+ let (result, _strat) = c.crush_mixed_array(&items, "", 1.0);+ // Lists and nulls (not dict/str/number) → fall through to "keep all".+ let list_count = result.iter().filter(|v| v.is_array()).count();+ let null_count = result.iter().filter(|v| v.is_null()).count();+ assert_eq!(list_count, 6);+ assert_eq!(null_count, 6);+ }++ #[test]+ fn crusher_construction_default() {+ let c = SmartCrusher::new(SmartCrusherConfig::default());+ assert_eq!(c.config.max_items_after_crush, 15);+ }++ #[test]+ fn crusher_with_custom_scorer() {+ use crate::relevance::BM25Scorer;+ let c = SmartCrusher::with_scorer(+ SmartCrusherConfig::default(),+ Box::new(BM25Scorer::default()),+ );+ // Sanity: crushing still works with a swapped scorer.+ let items: Vec<Value> = (0..30).map(|_| json!({"status": "ok"})).collect();+ let result = c.crush_array(&items, "anything", 1.0);+ assert!(result.items.len() <= 30);+ }++}diff --git a/crates/headroom-core/src/transforms/smart_crusher/mod.rs b/crates/headroom-core/src/transforms/smart_crusher/mod.rsindex bed801e16..3accbe556 100644--- a/crates/headroom-core/src/transforms/smart_crusher/mod.rs+++ b/crates/headroom-core/src/transforms/smart_crusher/mod.rs@@ -36,6 +36,7 @@ mod analyzer;mod anchors;mod classifier;mod config;+mod crusher;mod crushers;mod error_keywords;mod field_detect;@@ -51,6 +52,7 @@ pub use analyzer::SmartAnalyzer;pub use anchors::{extract_query_anchors, item_matches_anchors};pub use classifier::{classify_array, ArrayType};pub use config::SmartCrusherConfig;+pub use crusher::{CrushArrayResult, SmartCrusher};pub use crushers::{compute_k_split, crush_number_array, crush_object, crush_string_array};pub use error_keywords::ERROR_KEYWORDS;pub use field_detect::{detect_id_field_statistically, detect_score_field_statistically};