Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(rust): SmartCrusher struct — execute_plan + crush_array + crush_mixed_array

chopratejas4c44b95d9b14
diff --git a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs b/crates/headroom-core/src/transforms/smart_crusher/crusher.rs
new file mode 100644
index 000000000..d1fb43ca0
--- /dev/null
+++ b/crates/headroom-core/src/transforms/smart_crusher/crusher.rs
@@ -0,0 +1,593 @@
+//! `SmartCrusher` struct — top-level entry point for compression.
+//!
+//! Owns the `config`, `anchor_selector`, `scorer`, and `analyzer`
+//! singletons that every per-message call needs. Constructed once
+//! per process; the struct is `Send + Sync` so it can sit behind an
+//! `Arc` in a multi-threaded proxy.
+//!
+//! This module ports three Python entry points:
+//!
+//! - `_execute_plan` (line 3617) → `SmartCrusher::execute_plan`
+//! - `_crush_array` (line 2400) → `SmartCrusher::crush_array`
+//! - `_crush_mixed_array` (line 2914) → `SmartCrusher::crush_mixed_array`
+//!
+//! # Stubs that match Python's "everything-disabled" path
+//!
+//! Python's `_crush_array` calls into TOIN (cross-user pattern
+//! learning), feedback (per-tool compression hints), CCR (compress-
+//! cache-retrieve store), and telemetry. All four are large separate
+//! systems with their own state. For the like-for-like port at Stage
+//! 3c.1, we mirror Python's behavior **when those subsystems are
+//! disabled**:
+//!
+//! - **TOIN**: never produces a recommendation, never overrides
+//! `effective_max_items`, never injects preserve_fields/strategy/level.
+//! - **Feedback**: never produces hints; default `effective_max_items`.
+//! - **CCR**: `enabled=false`; result has `ccr_hash = None`.
+//! - **Telemetry**: no-op.
+//! - **`_compress_text_within_items`**: pass-through (returns input
+//! unchanged) since text compression has its own port pipeline.
+//! - **`summarize_dropped_items`**: empty string.
+//!
+//! Parity fixtures will be recorded with all four disabled on the
+//! Python side, locking byte-equal output. The TOIN/CCR/feedback
+//! integration ports happen later (Stage 3c.2 follow-ups).
+
+use serde_json::Value;
+
+use super::analyzer::SmartAnalyzer;
+use super::config::SmartCrusherConfig;
+use super::crushers::{compute_k_split, crush_string_array};
+use super::planning::SmartCrusherPlanner;
+use super::types::{CompressionPlan, CompressionStrategy};
+use crate::relevance::{HybridScorer, RelevanceScorer};
+use crate::transforms::adaptive_sizer::compute_optimal_k;
+use crate::transforms::anchor_selector::{AnchorConfig, AnchorSelector};
+
+/// Return type for `crush_array` — mirrors Python's
+/// `(crushed_items, strategy_info, ccr_hash, dropped_summary)` tuple.
+pub struct CrushArrayResult {
+ pub items: Vec<Value>,
+ /// Strategy debug string, e.g. `"smart_sample"`, `"top_n"`,
+ /// `"none:adaptive_at_limit"`, `"skip:unique_entities_no_signal"`.
+ pub strategy_info: String,
+ /// CCR retrieval hash if caching is enabled. Stub: always `None`.
+ pub ccr_hash: Option<String>,
+ /// Categorical summary of dropped items. Stub: always empty.
+ pub dropped_summary: String,
+}
+
+/// Top-level SmartCrusher.
+pub struct SmartCrusher {
+ pub config: SmartCrusherConfig,
+ pub anchor_selector: AnchorSelector,
+ pub scorer: Box<dyn RelevanceScorer + Send + Sync>,
+ pub analyzer: SmartAnalyzer,
+}
+
+impl SmartCrusher {
+ /// Construct with the default scorer (`HybridScorer`) and the
+ /// default `AnchorConfig`. Mirrors Python's
+ /// `SmartCrusher(config=...)` no-extra-args path with all
+ /// optional subsystems disabled.
+ pub fn new(config: SmartCrusherConfig) -> Self {
+ let analyzer = SmartAnalyzer::new(config.clone());
+ SmartCrusher {
+ config,
+ anchor_selector: AnchorSelector::new(AnchorConfig::default()),
+ scorer: Box::new(HybridScorer::default()),
+ analyzer,
+ }
+ }
+
+ /// Construct with a custom scorer.
+ pub fn with_scorer(
+ config: SmartCrusherConfig,
+ scorer: Box<dyn RelevanceScorer + Send + Sync>,
+ ) -> Self {
+ let analyzer = SmartAnalyzer::new(config.clone());
+ SmartCrusher {
+ config,
+ anchor_selector: AnchorSelector::new(AnchorConfig::default()),
+ scorer,
+ analyzer,
+ }
+ }
+
+ fn planner(&self) -> SmartCrusherPlanner<'_> {
+ SmartCrusherPlanner::new(
+ &self.config,
+ &self.anchor_selector,
+ &*self.scorer,
+ &self.analyzer,
+ )
+ }
+
+ /// Execute a `CompressionPlan` against `items`, returning the
+ /// kept-items list in original-array order. Mirrors Python's
+ /// `_execute_plan` (line 3617-3633).
+ ///
+ /// Schema-preserving: each kept item is cloned unchanged. No
+ /// summary objects, generated fields, or wrapper metadata.
+ pub fn execute_plan(&self, plan: &CompressionPlan, items: &[Value]) -> Vec<Value> {
+ let mut indices = plan.keep_indices.clone();
+ indices.sort_unstable();
+ indices
+ .into_iter()
+ .filter(|&idx| idx < items.len())
+ .map(|idx| items[idx].clone())
+ .collect()
+ }
+
+ /// Compress an array of dict items.
+ ///
+ /// Direct port of `_crush_array` (Python line 2400-2687) with the
+ /// optional subsystems (TOIN / CCR / feedback / telemetry) wired
+ /// in their disabled-by-default behavior. See module-level docs
+ /// for the rationale.
+ ///
+ /// # Pipeline
+ ///
+ /// 1. Compute `item_strings` once (used as input to adaptive
+ /// sizing and downstream relevance scoring).
+ /// 2. `compute_optimal_k` → `adaptive_k`.
+ /// 3. If `n <= adaptive_k`, return passthrough.
+ /// 4. `analyzer.analyze_array(items)` → `analysis`.
+ /// 5. If `analysis.recommended_strategy == Skip`, return passthrough
+ /// with a `skip:<reason>` strategy string.
+ /// 6. `planner.create_plan(analysis, items, query_context, ...)`.
+ /// 7. `execute_plan(plan, items)` → result.
+ /// 8. Strategy info = `analysis.recommended_strategy.as_str()`.
+ pub fn crush_array(&self, items: &[Value], query_context: &str, bias: f64) -> CrushArrayResult {
+ let item_strings: Vec<String> = items
+ .iter()
+ .map(|i| serde_json::to_string(i).unwrap_or_default())
+ .collect();
+ let item_str_refs: Vec<&str> = item_strings.iter().map(|s| s.as_str()).collect();
+
+ let max_k = if self.config.max_items_after_crush > 0 {
+ Some(self.config.max_items_after_crush)
+ } else {
+ None
+ };
+ let adaptive_k = compute_optimal_k(&item_str_refs, bias, 3, max_k);
+
+ // Tier-1 boundary: array already small enough.
+ if items.len() <= adaptive_k {
+ // Python branches into _compress_text_within_items here;
+ // stubbed to passthrough (no within-item text compression
+ // in this stage).
+ return CrushArrayResult {
+ items: items.to_vec(),
+ strategy_info: "none:adaptive_at_limit".to_string(),
+ ccr_hash: None,
+ dropped_summary: String::new(),
+ };
+ }
+
+ // TOIN/feedback paths stubbed → effective_max_items = adaptive_k.
+ let effective_max_items = adaptive_k;
+
+ let analysis = self.analyzer.analyze_array(items);
+
+ // Crushability gate: not safe to crush → return original.
+ if analysis.recommended_strategy == CompressionStrategy::Skip {
+ let reason = match &analysis.crushability {
+ Some(c) => format!("skip:{}", c.reason),
+ None => String::new(),
+ };
+ return CrushArrayResult {
+ items: items.to_vec(),
+ strategy_info: reason,
+ ccr_hash: None,
+ dropped_summary: String::new(),
+ };
+ }
+
+ // Plan + execute.
+ let plan = self.planner().create_plan(
+ &analysis,
+ items,
+ query_context,
+ None, // preserve_fields (TOIN — stubbed)
+ Some(effective_max_items),
+ Some(&item_strings),
+ );
+ let result = self.execute_plan(&plan, items);
+
+ // CCR/telemetry/TOIN-record paths stubbed.
+ let strategy_info = analysis.recommended_strategy.as_str().to_string();
+
+ CrushArrayResult {
+ items: result,
+ strategy_info,
+ ccr_hash: None,
+ dropped_summary: String::new(),
+ }
+ }
+
+ /// Compress a mixed-type array by grouping items by type and
+ /// compressing each group with the appropriate handler.
+ ///
+ /// Direct port of `_crush_mixed_array` (Python line 2914-3013).
+ ///
+ /// Strategy:
+ /// 1. Group by type (dict / str / number / list / null / bool / other).
+ /// 2. For groups with >= `min_items_to_analyze` items: apply the
+ /// type-specific compressor.
+ /// 3. For small groups: keep all items.
+ /// 4. Reassemble in original order.
+ ///
+ /// Returns `(crushed_items, strategy_string)`.
+ pub fn crush_mixed_array(
+ &self,
+ items: &[Value],
+ query_context: &str,
+ bias: f64,
+ ) -> (Vec<Value>, String) {
+ let n = items.len();
+ if n <= 8 {
+ return (items.to_vec(), "mixed:passthrough".to_string());
+ }
+
+ // Group by type, tracking original indices.
+ let mut groups: GroupBuckets = GroupBuckets::default();
+ for (i, item) in items.iter().enumerate() {
+ groups.push(group_key(item), i, item.clone());
+ }
+
+ let mut keep_indices: std::collections::BTreeSet<usize> =
+ std::collections::BTreeSet::new();
+ let mut strategy_parts: Vec<String> = Vec::new();
+
+ for (type_key, indices, values) in groups.into_iter() {
+ // Small groups: keep all items.
+ if values.len() < self.config.min_items_to_analyze {
+ keep_indices.extend(&indices);
+ continue;
+ }
+
+ match type_key {
+ "dict" => {
+ let CrushArrayResult {
+ items: crushed, ..
+ } = self.crush_array(&values, query_context, bias);
+ // Find which original indices survived by matching
+ // canonical-JSON serialization. Mirrors Python's
+ // `json.dumps(c, sort_keys=True, default=str)`-keyed
+ // set match.
+ let crushed_keys: std::collections::HashSet<String> = crushed
+ .iter()
+ .map(canonical_json_for_match)
+ .collect();
+ for (i, idx) in indices.iter().enumerate() {
+ if crushed_keys.contains(&canonical_json_for_match(&values[i])) {
+ keep_indices.insert(*idx);
+ }
+ }
+ strategy_parts.push(format!("dict:{}->{}", values.len(), crushed.len()));
+ }
+ "str" => {
+ let strs: Vec<&str> = values.iter().filter_map(|v| v.as_str()).collect();
+ let (crushed, _) = crush_string_array(&strs, &self.config, bias);
+ let crushed_set: std::collections::HashSet<&str> =
+ crushed.iter().map(|s| s.as_str()).collect();
+ for (i, idx) in indices.iter().enumerate() {
+ if let Some(s) = values[i].as_str() {
+ if crushed_set.contains(s) {
+ keep_indices.insert(*idx);
+ }
+ }
+ }
+ strategy_parts.push(format!("str:{}->{}", values.len(), crushed.len()));
+ }
+ "number" => {
+ // Python: just adaptive sampling + outlier detection
+ // (no summary prefix). Keeps first/last by index
+ // and items >variance_threshold σ from mean.
+ let item_strings: Vec<String> =
+ values.iter().map(|v| v.to_string()).collect();
+ let item_refs: Vec<&str> =
+ item_strings.iter().map(|s| s.as_str()).collect();
+ let (_kt, kf, kl, _) = compute_k_split(&item_refs, &self.config, bias);
+
+ let kf = kf.min(values.len());
+ let kl = kl.min(values.len().saturating_sub(kf));
+ let first_idx: Vec<usize> = indices.iter().take(kf).copied().collect();
+ let last_idx: Vec<usize> = indices
+ .iter()
+ .rev()
+ .take(kl)
+ .copied()
+ .collect::<Vec<_>>();
+ keep_indices.extend(&first_idx);
+ keep_indices.extend(&last_idx);
+
+ // Outliers via finite-only stats.
+ let finite: Vec<f64> = values
+ .iter()
+ .filter_map(|v| v.as_f64().filter(|f| f.is_finite()))
+ .collect();
+ if finite.len() > 1 {
+ if let Some(mean_v) =
+ super::stats_math::mean(&finite)
+ {
+ if let Some(std_v) =
+ super::stats_math::sample_stdev(&finite)
+ {
+ if std_v > 0.0 {
+ let threshold = self.config.variance_threshold * std_v;
+ for (i, val) in values.iter().enumerate() {
+ if let Some(num) =
+ val.as_f64().filter(|f| f.is_finite())
+ {
+ if (num - mean_v).abs() > threshold {
+ keep_indices.insert(indices[i]);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ strategy_parts.push(format!("num:{}", values.len()));
+ }
+ _ => {
+ // list / bool / none / other → keep all items.
+ keep_indices.extend(&indices);
+ }
+ }
+ }
+
+ // Reassemble in original order.
+ let result: Vec<Value> = keep_indices.iter().map(|&i| items[i].clone()).collect();
+ let strategy = format!(
+ "mixed:adaptive({}->{},{})",
+ n,
+ result.len(),
+ strategy_parts.join(",")
+ );
+ (result, strategy)
+ }
+}
+
+// ---------- helpers ----------
+
+/// Group key that mirrors Python's `_crush_mixed_array` switch on
+/// `isinstance`. Note the bool-before-number ordering: in Python, bool
+/// is a subclass of int, but JSON treats them as distinct types, so we
+/// don't have the Python ordering hazard.
+fn group_key(item: &Value) -> &'static str {
+ match item {
+ Value::Object(_) => "dict",
+ Value::String(_) => "str",
+ Value::Bool(_) => "bool",
+ Value::Number(_) => "number",
+ Value::Array(_) => "list",
+ Value::Null => "none",
+ }
+}
+
+/// Group buckets keyed by the type-string. Preserves first-occurrence
+/// order across keys so dict/str/number/list/none/bool always come out
+/// in the same order — matters because `keep_indices` is built
+/// incrementally and Python iterates `groups.items()` (insertion order
+/// in 3.7+).
+#[derive(Default)]
+struct GroupBuckets {
+ entries: Vec<(&'static str, Vec<usize>, Vec<Value>)>,
+ index_of: std::collections::HashMap<&'static str, usize>,
+}
+
+impl GroupBuckets {
+ fn push(&mut self, key: &'static str, idx: usize, value: Value) {
+ match self.index_of.get(key).copied() {
+ Some(i) => {
+ self.entries[i].1.push(idx);
+ self.entries[i].2.push(value);
+ }
+ None => {
+ self.index_of.insert(key, self.entries.len());
+ self.entries.push((key, vec![idx], vec![value]));
+ }
+ }
+ }
+}
+
+impl IntoIterator for GroupBuckets {
+ type Item = (&'static str, Vec<usize>, Vec<Value>);
+ type IntoIter = std::vec::IntoIter<Self::Item>;
+ fn into_iter(self) -> Self::IntoIter {
+ self.entries.into_iter()
+ }
+}
+
+/// Serialize a `Value` for membership comparison. Mirrors Python's
+/// `json.dumps(c, sort_keys=True, default=str)` used by
+/// `_crush_mixed_array` to match crushed dict items back to their
+/// original indices. The `default=str` fallback only matters for
+/// non-JSON-serializable Python values; in serde_json land everything
+/// is already JSON-native, so plain canonical JSON suffices.
+fn canonical_json_for_match(value: &Value) -> String {
+ crate::transforms::anchor_selector::python_json_dumps_sort_keys(value)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ fn crusher() -> SmartCrusher {
+ SmartCrusher::new(SmartCrusherConfig::default())
+ }
+
+ // ---------- execute_plan ----------
+
+ #[test]
+ fn execute_plan_empty_indices_returns_empty() {
+ let c = crusher();
+ let plan = CompressionPlan::default();
+ let items: Vec<Value> = (0..5).map(|i| json!({"id": i})).collect();
+ let result = c.execute_plan(&plan, &items);
+ assert!(result.is_empty());
+ }
+
+ #[test]
+ fn execute_plan_returns_items_in_sorted_index_order() {
+ let c = crusher();
+ let items: Vec<Value> = (0..10).map(|i| json!({"id": i})).collect();
+ let plan = CompressionPlan {
+ keep_indices: vec![5, 2, 8, 0],
+ ..CompressionPlan::default()
+ };
+ let result = c.execute_plan(&plan, &items);
+ assert_eq!(result.len(), 4);
+ assert_eq!(result[0]["id"], 0);
+ assert_eq!(result[1]["id"], 2);
+ assert_eq!(result[2]["id"], 5);
+ assert_eq!(result[3]["id"], 8);
+ }
+
+ #[test]
+ fn execute_plan_skips_out_of_bounds() {
+ let c = crusher();
+ let items: Vec<Value> = (0..3).map(|i| json!({"id": i})).collect();
+ let plan = CompressionPlan {
+ keep_indices: vec![0, 5, 2],
+ ..CompressionPlan::default()
+ };
+ let result = c.execute_plan(&plan, &items);
+ assert_eq!(result.len(), 2);
+ }
+
+ // ---------- crush_array ----------
+
+ #[test]
+ fn crush_array_passthrough_when_below_adaptive_k() {
+ let c = crusher();
+ let items: Vec<Value> = (0..3).map(|i| json!({"id": i})).collect();
+ let result = c.crush_array(&items, "", 1.0);
+ assert_eq!(result.items.len(), 3);
+ assert_eq!(result.strategy_info, "none:adaptive_at_limit");
+ assert!(result.ccr_hash.is_none());
+ }
+
+ #[test]
+ fn crush_array_skip_path_returns_original_items() {
+ // 30 unique dict items with ID-like fields → analyzer should
+ // detect "unique_entities_no_signal" and SKIP.
+ let c = crusher();
+ let items: Vec<Value> = (0..30)
+ .map(|i| json!({"id": i, "name": format!("user_{}", i)}))
+ .collect();
+ let result = c.crush_array(&items, "", 1.0);
+ // skip path returns the original items unchanged.
+ assert_eq!(result.items.len(), 30);
+ assert!(
+ result.strategy_info.starts_with("skip:"),
+ "expected skip:..., got {}",
+ result.strategy_info
+ );
+ }
+
+ #[test]
+ fn crush_array_low_uniqueness_compresses() {
+ // 30 items with status=ok across all → low_uniqueness path
+ // (crushable, smart_sample strategy).
+ let c = crusher();
+ let items: Vec<Value> = (0..30).map(|_| json!({"status": "ok"})).collect();
+ let result = c.crush_array(&items, "", 1.0);
+ assert!(
+ result.items.len() <= 30,
+ "should not exceed original count"
+ );
+ }
+
+ #[test]
+ fn crush_array_keeps_error_items() {
+ let c = crusher();
+ let mut items: Vec<Value> = (0..30)
+ .map(|i| json!({"id": i, "status": "ok"}))
+ .collect();
+ items.push(json!({"id": 30, "status": "error", "msg": "FATAL"}));
+ let result = c.crush_array(&items, "", 1.0);
+ // Whatever path is taken, the error item should survive.
+ assert!(
+ result.items.iter().any(|item| {
+ item.get("status").and_then(|v| v.as_str()) == Some("error")
+ }),
+ "error item must survive crush_array"
+ );
+ }
+
+ // ---------- crush_mixed_array ----------
+
+ #[test]
+ fn crush_mixed_passthrough_at_threshold() {
+ let c = crusher();
+ let items: Vec<Value> = vec![
+ json!(1),
+ json!("two"),
+ json!({"k": "v"}),
+ json!([1, 2]),
+ json!(null),
+ json!(true),
+ json!(3),
+ json!("four"),
+ ];
+ let (result, strat) = c.crush_mixed_array(&items, "", 1.0);
+ assert_eq!(result.len(), 8);
+ assert_eq!(strat, "mixed:passthrough");
+ }
+
+ #[test]
+ fn crush_mixed_groups_and_compresses_dicts() {
+ let c = crusher();
+ // 25 dicts (large group → gets crushed) + 5 strings (small group → all kept).
+ let mut items: Vec<Value> = (0..25)
+ .map(|i| json!({"id": i, "status": "ok"}))
+ .collect();
+ for i in 0..5 {
+ items.push(json!(format!("string_{}", i)));
+ }
+ let (result, strat) = c.crush_mixed_array(&items, "", 1.0);
+ assert!(strat.starts_with("mixed:adaptive("));
+ // The 5 strings (small group) all survive.
+ let str_count = result.iter().filter(|v| v.is_string()).count();
+ assert_eq!(str_count, 5);
+ }
+
+ #[test]
+ fn crush_mixed_keeps_lists_and_nulls_unchanged() {
+ let c = crusher();
+ let mut items: Vec<Value> = vec![json!([1, 2]); 6];
+ items.extend(vec![json!(null); 6]);
+ items.extend(vec![json!({"k": 1}); 10]);
+ let (result, _strat) = c.crush_mixed_array(&items, "", 1.0);
+ // Lists and nulls (not dict/str/number) → fall through to "keep all".
+ let list_count = result.iter().filter(|v| v.is_array()).count();
+ let null_count = result.iter().filter(|v| v.is_null()).count();
+ assert_eq!(list_count, 6);
+ assert_eq!(null_count, 6);
+ }
+
+ #[test]
+ fn crusher_construction_default() {
+ let c = SmartCrusher::new(SmartCrusherConfig::default());
+ assert_eq!(c.config.max_items_after_crush, 15);
+ }
+
+ #[test]
+ fn crusher_with_custom_scorer() {
+ use crate::relevance::BM25Scorer;
+ let c = SmartCrusher::with_scorer(
+ SmartCrusherConfig::default(),
+ Box::new(BM25Scorer::default()),
+ );
+ // Sanity: crushing still works with a swapped scorer.
+ let items: Vec<Value> = (0..30).map(|_| json!({"status": "ok"})).collect();
+ let result = c.crush_array(&items, "anything", 1.0);
+ assert!(result.items.len() <= 30);
+ }
+
+}
diff --git a/crates/headroom-core/src/transforms/smart_crusher/mod.rs b/crates/headroom-core/src/transforms/smart_crusher/mod.rs
index bed801e16..3accbe556 100644
--- a/crates/headroom-core/src/transforms/smart_crusher/mod.rs
+++ b/crates/headroom-core/src/transforms/smart_crusher/mod.rs
@@ -36,6 +36,7 @@ mod analyzer;
mod anchors;
mod classifier;
mod config;
+mod crusher;
mod crushers;
mod error_keywords;
mod field_detect;
@@ -51,6 +52,7 @@ pub use analyzer::SmartAnalyzer;
pub use anchors::{extract_query_anchors, item_matches_anchors};
pub use classifier::{classify_array, ArrayType};
pub use config::SmartCrusherConfig;
+pub use crusher::{CrushArrayResult, SmartCrusher};
pub use crushers::{compute_k_split, crush_number_array, crush_object, crush_string_array};
pub use error_keywords::ERROR_KEYWORDS;
pub use field_detect::{detect_id_field_statistically, detect_score_field_statistically};