Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(rust): smart_crusher universal crushers — string, number, object
chopratejas9d515fb78edf
diff --git a/Cargo.lock b/Cargo.lockindex 1c8fb3dc1..a3f46fc97 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -927,6 +927,7 @@ version = "0.1.0"dependencies = ["bytes","criterion",+ "flate2","hf-hub","md-5","proptest",diff --git a/crates/headroom-core/src/transforms/smart_crusher/crushers.rs b/crates/headroom-core/src/transforms/smart_crusher/crushers.rsnew file mode 100644index 000000000..4e3a14ace--- /dev/null+++ b/crates/headroom-core/src/transforms/smart_crusher/crushers.rs@@ -0,0 +1,737 @@+//! Three universal crushers for non-dict-array JSON shapes.+//!+//! Direct ports from `headroom/transforms/smart_crusher.py`:+//!+//! - `crush_string_array` ← `_crush_string_array` (line 2727)+//! - `crush_number_array` ← `_crush_number_array` (line 2810) — has BUG #1+//! - `crush_object` ← `_crush_object` (line 3015)+//!+//! Each takes a `&SmartCrusherConfig`, a `bias` multiplier, and returns+//! `(crushed_items, strategy_string)`. Schema-preserving: the output+//! contains only items/values from the original; no generated text or+//! summary objects sneak in.+//!+//! `_crush_array` (the dict-array orchestrator) and `_crush_mixed_array`+//! (the type-grouped fallback) live in a later commit because they pull+//! in the planning + execution + TOIN/CCR scaffolding.+//!+//! # BUG #1 — percentile off-by-one in `crush_number_array`+//!+//! Python's `_crush_number_array` computes p25/p75 as+//! `sorted_finite[len(sorted_finite) // 4]` and+//! `sorted_finite[3 * len(sorted_finite) // 4]`. For `len < 8`, those+//! integer-division indices land one position before where a proper+//! quantile would sit. The bug only affects the strategy debug string+//! (`f",p25={p25:.4g},p75={p75:.4g}"`); item-selection logic is+//! unaffected.+//!+//! We port the bug **as-is** so parity fixtures still byte-match.+//! Stage 3c.1 commit 7 fixes BOTH languages in lockstep — at that point+//! the bug-doc test below flips to pin the corrected behavior and the+//! fixtures are regenerated.++use serde_json::{Map, Value};+use std::collections::{BTreeSet, HashSet};++use super::config::SmartCrusherConfig;+use super::error_keywords::ERROR_KEYWORDS;+use super::stats_math::{format_g, mean, median, sample_stdev};+use crate::transforms::adaptive_sizer::compute_optimal_k;++/// Compute K split (total / first / last / importance) for adaptive+/// crushers. Mirrors `_compute_k_split` (Python `smart_crusher.py:2693-2725`).+///+/// Splits the Kneedle-derived `k_total` into:+/// - `k_first`: items kept from the start of the array.+/// - `k_last`: items kept from the end.+/// - `k_importance`: leftover budget for importance-driven items.+///+/// Returns `(k_total, k_first, k_last, k_importance)`.+pub fn compute_k_split(+ items: &[&str],+ config: &SmartCrusherConfig,+ bias: f64,+) -> (usize, usize, usize, usize) {+ let max_k = if config.max_items_after_crush > 0 {+ Some(config.max_items_after_crush)+ } else {+ None+ };+ let k_total = compute_optimal_k(items, bias, 3, max_k);+ // Python: `max(1, round(k_total * fraction))`. Python's round() uses+ // banker's rounding (round-half-to-even). Rust's+ // f64::round_ties_even() mirrors that exactly — was stabilized in+ // Rust 1.77 and is the right primitive for this parity port.+ let k_first = 1_usize.max(round_ties_even(k_total as f64 * config.first_fraction) as usize);+ let k_last = 1_usize.max(round_ties_even(k_total as f64 * config.last_fraction) as usize);+ let k_importance = k_total.saturating_sub(k_first + k_last);+ (k_total, k_first, k_last, k_importance)+}++/// Crush an array of strings.+///+/// Strategy (Python `_crush_string_array`):+/// 1. Adaptive K via Kneedle (passthrough on `n <= 8`).+/// 2. **Always keep**: error-keyword strings + length-anomaly strings.+/// 3. **Boundary keep**: first K_first + last K_last.+/// 4. **Stride-fill**: stride-based diverse sampling, dedup by content.+/// 5. Output preserves original array order.+///+/// `bias` is the compression-aggressiveness multiplier used by+/// `compute_optimal_k`.+pub fn crush_string_array(+ items: &[&str],+ config: &SmartCrusherConfig,+ bias: f64,+) -> (Vec<String>, String) {+ let n = items.len();+ if n <= 8 {+ return (+ items.iter().map(|s| (*s).to_string()).collect(),+ "string:passthrough".to_string(),+ );+ }++ // K split. Python serializes each item via json.dumps; for already-+ // string items that just wraps in quotes. We feed the raw &str refs+ // since adaptive_sizer's input is documented as "string repr in+ // importance order" — matches Python's intent.+ let (k_total, k_first, k_last, _k_importance) = compute_k_split(items, config, bias);++ // 1. Error-keyword indices.+ let mut error_indices: BTreeSet<usize> = BTreeSet::new();+ for (i, s) in items.iter().enumerate() {+ let lower = s.to_lowercase();+ if ERROR_KEYWORDS.iter().any(|kw| lower.contains(kw)) {+ error_indices.insert(i);+ }+ }++ // 2. Length anomaly indices.+ let lengths: Vec<f64> = items.iter().map(|s| s.chars().count() as f64).collect();+ let mut anomaly_indices: BTreeSet<usize> = BTreeSet::new();+ if lengths.len() > 1 {+ let mean_len = mean(&lengths).unwrap_or(0.0);+ // Python uses `statistics.stdev` here (sample stdev).+ let std_len = sample_stdev(&lengths).unwrap_or(0.0);+ if std_len > 0.0 {+ let threshold = config.variance_threshold * std_len;+ for (i, &length) in lengths.iter().enumerate() {+ if (length - mean_len).abs() > threshold {+ anomaly_indices.insert(i);+ }+ }+ }+ }++ // 3. Boundary indices.+ let first_indices: BTreeSet<usize> = (0..k_first.min(n)).collect();+ let last_start = n.saturating_sub(k_last);+ let last_indices: BTreeSet<usize> = (last_start..n).collect();++ // 4. Combine.+ let mut keep_indices: BTreeSet<usize> = BTreeSet::new();+ keep_indices.extend(error_indices.iter().copied());+ keep_indices.extend(anomaly_indices.iter().copied());+ keep_indices.extend(first_indices.iter().copied());+ keep_indices.extend(last_indices.iter().copied());++ // Pre-populate seen_strings from current keeps.+ let mut seen: HashSet<&str> = HashSet::new();+ for &i in &keep_indices {+ seen.insert(items[i]);+ }++ // 5. Stride-fill remaining budget.+ let mut dedup_count: usize = 0;+ let remaining_budget = k_total.saturating_sub(keep_indices.len());+ if remaining_budget > 0 {+ let stride = ((n.saturating_sub(1)) / (remaining_budget + 1)).max(1);+ // Python: cap = k_total + len(error_indices) + len(anomaly_indices)+ let cap = k_total + error_indices.len() + anomaly_indices.len();+ let mut i: usize = 0;+ while i < n {+ if keep_indices.len() >= cap {+ break;+ }+ if !keep_indices.contains(&i) {+ if !seen.contains(items[i]) {+ keep_indices.insert(i);+ seen.insert(items[i]);+ } else {+ dedup_count += 1;+ }+ }+ i += stride;+ }+ }++ // 6. Build output preserving original order.+ let result: Vec<String> = keep_indices.iter().map(|&i| items[i].to_string()).collect();++ let mut strategy = format!("string:adaptive({}->{}", n, result.len());+ if dedup_count > 0 {+ strategy.push_str(&format!(",dedup={}", dedup_count));+ }+ if !error_indices.is_empty() {+ strategy.push_str(&format!(",errors={}", error_indices.len()));+ }+ strategy.push(')');++ (result, strategy)+}++/// Crush an array of numbers.+///+/// Mirrors `_crush_number_array`. **Carries BUG #1** in the percentile+/// computation (see module-level doc); fix lands in commit 7.+pub fn crush_number_array(+ items: &[Value],+ config: &SmartCrusherConfig,+ bias: f64,+) -> (Vec<Value>, String) {+ let n = items.len();+ if n <= 8 {+ return (items.to_vec(), "number:passthrough".to_string());+ }++ // Filter to finite f64 only — Python: `isinstance(x, int|float) and math.isfinite(x)`.+ let finite: Vec<f64> = items+ .iter()+ .filter_map(|v| v.as_f64().filter(|f| f.is_finite()))+ .collect();+ if finite.is_empty() {+ return (items.to_vec(), "number:no_finite".to_string());+ }++ // K split. Python: `_compute_k_split(items, bias)` serializes via json.dumps+ // — for a number array that's just str(num).+ let item_strings: Vec<String> = items.iter().map(|v| v.to_string()).collect();+ let item_str_refs: Vec<&str> = item_strings.iter().map(|s| s.as_str()).collect();+ let (k_total, k_first, k_last, _) = compute_k_split(&item_str_refs, config, bias);++ // Statistics.+ let mean_val = mean(&finite).unwrap_or(0.0);+ let median_val = median(&finite).unwrap_or(0.0);+ let std_val = if finite.len() > 1 {+ sample_stdev(&finite).unwrap_or(0.0)+ } else {+ 0.0+ };++ // Sorted for percentiles.+ let mut sorted_finite: Vec<f64> = finite.clone();+ sorted_finite.sort_by(f64::total_cmp);++ // BUG #1 (faithful port): integer-division index. For `len(sorted_finite) < 4`,+ // fall back to min/max. Otherwise `sorted_finite[len/4]` / `sorted_finite[3*len/4]`.+ let p25 = if sorted_finite.len() >= 4 {+ sorted_finite[sorted_finite.len() / 4]+ } else {+ finite_min(&finite)+ };+ let p75 = if sorted_finite.len() >= 4 {+ sorted_finite[3 * sorted_finite.len() / 4]+ } else {+ finite_max(&finite)+ };++ // Outliers (>variance_threshold σ from mean).+ let mut outlier_indices: BTreeSet<usize> = BTreeSet::new();+ if std_val > 0.0 {+ let threshold = config.variance_threshold * std_val;+ for (i, val) in items.iter().enumerate() {+ if let Some(num) = val.as_f64().filter(|f| f.is_finite()) {+ if (num - mean_val).abs() > threshold {+ outlier_indices.insert(i);+ }+ }+ }+ }++ // Change points via window-mean comparison. Python guards on `n > 10`.+ let mut change_indices: BTreeSet<usize> = BTreeSet::new();+ if config.preserve_change_points && n > 10 {+ let window: usize = 5;+ for i in window..n.saturating_sub(window) {+ // Python collects only finite items in each window; it's possible+ // for windows to be empty if all items in a slice are non-finite.+ let left: Vec<f64> = items[i - window..i]+ .iter()+ .filter_map(|v| v.as_f64().filter(|f| f.is_finite()))+ .collect();+ let right: Vec<f64> = items[i..i + window]+ .iter()+ .filter_map(|v| v.as_f64().filter(|f| f.is_finite()))+ .collect();+ if !left.is_empty() && !right.is_empty() {+ let left_mean = mean(&left).unwrap_or(0.0);+ let right_mean = mean(&right).unwrap_or(0.0);+ if std_val > 0.0+ && (right_mean - left_mean).abs() > config.variance_threshold * std_val+ {+ change_indices.insert(i);+ }+ }+ }+ }++ // Boundary.+ let first_indices: BTreeSet<usize> = (0..k_first.min(n)).collect();+ let last_start = n.saturating_sub(k_last);+ let last_indices: BTreeSet<usize> = (last_start..n).collect();++ // Combine.+ let mut keep_indices: BTreeSet<usize> = BTreeSet::new();+ keep_indices.extend(outlier_indices.iter().copied());+ keep_indices.extend(change_indices.iter().copied());+ keep_indices.extend(first_indices.iter().copied());+ keep_indices.extend(last_indices.iter().copied());++ // Stride-fill. Cap = k_total + len(outlier_indices) (Python:+ // `keep_indices >= k_total + len(outlier_indices)` — note no+ // anomaly term here, unlike crush_string_array).+ let remaining_budget = k_total.saturating_sub(keep_indices.len());+ if remaining_budget > 0 {+ let stride = ((n.saturating_sub(1)) / (remaining_budget + 1)).max(1);+ let cap = k_total + outlier_indices.len();+ let mut i: usize = 0;+ while i < n {+ if keep_indices.len() >= cap {+ break;+ }+ if !keep_indices.contains(&i) {+ keep_indices.insert(i);+ }+ i += stride;+ }+ }++ // Build output: kept values only (schema-preserving — no summary prefix).+ let kept_values: Vec<Value> = keep_indices.iter().map(|&i| items[i].clone()).collect();++ let mn = finite_min(&finite);+ let mx = finite_max(&finite);+ let mut strategy = format!(+ "number:adaptive({}->{},min={},max={},mean={},median={},stddev={},p25={},p75={}",+ n,+ kept_values.len(),+ format_number_repr(mn),+ format_number_repr(mx),+ format_g(mean_val),+ format_g(median_val),+ format_g(std_val),+ format_g(p25),+ format_g(p75),+ );+ if !outlier_indices.is_empty() {+ strategy.push_str(&format!(",outliers={}", outlier_indices.len()));+ }+ if !change_indices.is_empty() {+ strategy.push_str(&format!(",change_points={}", change_indices.len()));+ }+ strategy.push(')');++ (kept_values, strategy)+}++/// Crush a JSON object by selecting the most informative keys.+///+/// Mirrors `_crush_object`. Treats key-value pairs as items and applies+/// `compute_optimal_k` directly on `f"{k}: {json.dumps(v)}"` strings.+/// Always-kept rules:+/// - keys whose value contains an error keyword.+/// - keys with small total token estimate (<=12 tokens via the rough+/// `len(str)/4 + len(key)/4 + 2` heuristic).+/// - first K_first and last K_last keys (insertion order — `IndexMap`+/// preserves it via the `serde_json/preserve_order` feature).+pub fn crush_object(+ obj: &Map<String, Value>,+ config: &SmartCrusherConfig,+ bias: f64,+) -> (Map<String, Value>, String) {+ let n = obj.len();+ if n <= 8 {+ return (obj.clone(), "object:passthrough".to_string());+ }++ // Estimate tokens per key-value pair. Python: `len(str)/4 + len(key)/4 + 2`.+ let mut kv_tokens: Vec<(String, usize)> = Vec::with_capacity(n);+ let mut total_tokens: usize = 0;+ for (key, val) in obj {+ let val_str = serde_json::to_string(val).unwrap_or_default();+ let tokens = val_str.len() / 4 + key.len() / 4 + 2;+ kv_tokens.push((key.clone(), tokens));+ total_tokens += tokens;+ }++ if total_tokens < config.min_tokens_to_crush {+ return (obj.clone(), "object:passthrough".to_string());+ }++ // Compute adaptive K on key-value string representations.+ let keys: Vec<&String> = obj.keys().collect();+ let kv_strings: Vec<String> = keys+ .iter()+ .map(|k| format!("{}: {}", k, serde_json::to_string(&obj[k.as_str()]).unwrap_or_default()))+ .collect();+ let kv_refs: Vec<&str> = kv_strings.iter().map(|s| s.as_str()).collect();++ let max_k = if config.max_items_after_crush > 0 {+ Some(config.max_items_after_crush)+ } else {+ None+ };+ let k_total = compute_optimal_k(&kv_refs, bias, 3, max_k);++ if k_total >= n {+ return (obj.clone(), "object:passthrough".to_string());+ }++ // Always keep: error-keyword values.+ let mut keep_keys: HashSet<String> = HashSet::new();+ for (key, val) in obj {+ let val_str = serde_json::to_string(val).unwrap_or_default().to_lowercase();+ if ERROR_KEYWORDS.iter().any(|kw| val_str.contains(kw)) {+ keep_keys.insert(key.clone());+ }+ }++ // Always keep: small values (cheap to keep).+ // Python: `if tokens <= small_threshold // 4` where small_threshold=50,+ // so tokens <= 12.+ let small_threshold_tokens = 50_usize / 4;+ for (key, tokens) in &kv_tokens {+ if *tokens <= small_threshold_tokens {+ keep_keys.insert(key.clone());+ }+ }++ // Boundary: first K_first and last K_last (over the key insertion order).+ let k_first = 1_usize.max(round_ties_even(k_total as f64 * config.first_fraction) as usize);+ let k_last = 1_usize.max(round_ties_even(k_total as f64 * config.last_fraction) as usize);+ for k in keys.iter().take(k_first) {+ keep_keys.insert((*k).clone());+ }+ for k in keys.iter().rev().take(k_last) {+ keep_keys.insert((*k).clone());+ }++ // Stride fill. Python's cap recomputes the error-keyword count each+ // iteration (inefficient but deterministic). We can compute once+ // because once a key is in keep_keys, the count of error-flagged+ // entries grows monotonically — which means the cap effectively+ // grows. Mirror Python's behavior by recomputing.+ let remaining = k_total.saturating_sub(keep_keys.len());+ if remaining > 0 {+ let stride = ((n.saturating_sub(1)) / (remaining + 1)).max(1);+ let mut i: usize = 0;+ while i < n {+ // Python: `if len(keep_keys) >= k_total + len([k for k in keep_keys if any(kw in json.dumps(obj[k]).lower() for kw in keywords)])`+ let error_kept_count = keep_keys+ .iter()+ .filter(|k| {+ let s = serde_json::to_string(&obj[k.as_str()])+ .unwrap_or_default()+ .to_lowercase();+ ERROR_KEYWORDS.iter().any(|kw| s.contains(kw))+ })+ .count();+ if keep_keys.len() >= k_total + error_kept_count {+ break;+ }+ keep_keys.insert(keys[i].clone());+ i += stride;+ }+ }++ // Build output preserving original key insertion order.+ let mut result: Map<String, Value> = Map::new();+ for k in &keys {+ if keep_keys.contains(k.as_str()) {+ result.insert((*k).clone(), obj[k.as_str()].clone());+ }+ }++ let strategy = format!("object:adaptive({}->{} keys)", n, result.len());+ (result, strategy)+}++// ---------- helpers ----------++fn finite_min(values: &[f64]) -> f64 {+ values.iter().cloned().reduce(f64::min).unwrap_or(0.0)+}++fn finite_max(values: &[f64]) -> f64 {+ values.iter().cloned().reduce(f64::max).unwrap_or(0.0)+}++/// Python's `round()` uses banker's rounding (round-half-to-even). Rust+/// stabilized `f64::round_ties_even()` in 1.77 — that's the right+/// primitive for parity. Wrapping it in a helper keeps the call sites+/// readable.+fn round_ties_even(x: f64) -> f64 {+ x.round_ties_even()+}++/// Format a number for Python's f-string default repr (no precision+/// specifier). `min(finite)` and `max(finite)` in Python's strategy+/// string fall here. Integers print without a decimal; floats print+/// with their natural decimal form. JSON Number doesn't preserve the+/// integer/float distinction once parsed via `as_f64`, so we approximate:+/// values exactly representable as `i64` get integer formatting.+fn format_number_repr(x: f64) -> String {+ if x.is_nan() {+ return "nan".to_string();+ }+ if x.is_infinite() {+ return if x > 0.0 { "inf".to_string() } else { "-inf".to_string() };+ }+ if x.fract() == 0.0 && x.abs() < 1e16 {+ return format!("{}", x as i64);+ }+ // Otherwise Python's `str(float)` — which is "shortest round-trip".+ // Rust's f64 Display is also shortest round-trip; should match for+ // typical inputs.+ format!("{}", x)+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ fn cfg() -> SmartCrusherConfig {+ SmartCrusherConfig::default()+ }++ // ---------- compute_k_split ----------++ #[test]+ fn k_split_below_threshold_returns_n() {+ // n <= 8 → adaptive_k = n. k_first/k_last = max(1, round(n * fraction)).+ let items = ["a", "b", "c", "d", "e"];+ let (kt, kf, kl, ki) = compute_k_split(&items, &cfg(), 1.0);+ assert_eq!(kt, 5);+ // round(5 * 0.3) = round(1.5) = banker's → 2+ assert_eq!(kf, 2);+ // round(5 * 0.15) = round(0.75) = 1+ assert_eq!(kl, 1);+ // 5 - 2 - 1 = 2+ assert_eq!(ki, 2);+ }++ #[test]+ fn k_split_low_diversity_returns_min_k() {+ // 10 identical items: tier-1 unique-by-simhash=1, returns max(min_k=3, 1)=3.+ // Then k_first = max(1, round_ties_even(3*0.3))=max(1, round(0.9))=max(1,1)=1.+ let items: [&str; 10] = ["x"; 10];+ let (kt, kf, kl, _) = compute_k_split(&items, &cfg(), 1.0);+ assert_eq!(kt, 3, "low-diversity → max(min_k, unique_count)=3");+ assert_eq!(kf, 1);+ assert_eq!(kl, 1);+ }++ // ---------- crush_string_array ----------++ #[test]+ fn string_array_passthrough_at_threshold() {+ let items: [&str; 8] = ["a", "b", "c", "d", "e", "f", "g", "h"];+ let (out, strat) = crush_string_array(&items, &cfg(), 1.0);+ assert_eq!(out.len(), 8);+ assert_eq!(strat, "string:passthrough");+ }++ #[test]+ fn string_array_keeps_error_strings() {+ let items: Vec<&str> = (0..30)+ .map(|i| if i == 15 { "FATAL: out of memory" } else { "ok" })+ .collect();+ let (out, strat) = crush_string_array(&items, &cfg(), 1.0);+ // Error item at index 15 must survive.+ assert!(out.iter().any(|s| s == "FATAL: out of memory"));+ assert!(strat.contains("errors=1"));+ }++ #[test]+ fn string_array_keeps_first_and_last() {+ let items: Vec<String> = (0..30).map(|i| format!("item_{}", i)).collect();+ let refs: Vec<&str> = items.iter().map(|s| s.as_str()).collect();+ let (out, _) = crush_string_array(&refs, &cfg(), 1.0);+ // First item (item_0) should always be kept (k_first >= 1).+ assert!(out.iter().any(|s| s == "item_0"));+ // Last item (item_29) should always be kept (k_last >= 1).+ assert!(out.iter().any(|s| s == "item_29"));+ }++ #[test]+ fn string_array_dedup_count_appears_in_strategy() {+ // Lots of duplicates that survive stride sampling get deduped.+ let items: Vec<&str> = std::iter::repeat("dup").take(50).collect();+ let (_out, strat) = crush_string_array(&items, &cfg(), 1.0);+ // 50 identical items: unique-by-simhash = 1, fast-path returns 3.+ // So k_total=3. Stride loop runs but every item is "dup" already+ // seen → dedup_count > 0.+ assert!(+ strat.contains("dedup="),+ "strategy {} should mention dedup",+ strat+ );+ }++ // ---------- crush_number_array ----------++ #[test]+ fn number_array_passthrough_at_threshold() {+ let items: Vec<Value> = (0..8).map(|i| json!(i)).collect();+ let (out, strat) = crush_number_array(&items, &cfg(), 1.0);+ assert_eq!(out.len(), 8);+ assert_eq!(strat, "number:passthrough");+ }++ #[test]+ fn number_array_no_finite_returns_passthrough() {+ // n > 8 but no finite values → "number:no_finite" strategy.+ // serde_json can't carry NaN, so use null values for non-numeric:+ // they're filtered out by `as_f64()`.+ let items: Vec<Value> = (0..15).map(|_| json!(null)).collect();+ let (out, strat) = crush_number_array(&items, &cfg(), 1.0);+ assert_eq!(out.len(), items.len());+ assert_eq!(strat, "number:no_finite");+ }++ #[test]+ fn number_array_keeps_outliers() {+ // 30 zeros + one 1000 → outlier should be kept.+ let mut items: Vec<Value> = vec![json!(0); 30];+ items.push(json!(1000));+ let (out, strat) = crush_number_array(&items, &cfg(), 1.0);+ assert!(out.iter().any(|v| v.as_f64() == Some(1000.0)));+ assert!(strat.contains("outliers="));+ }++ #[test]+ fn number_array_strategy_string_includes_summary() {+ let items: Vec<Value> = (1..=20).map(|i| json!(i)).collect();+ let (_out, strat) = crush_number_array(&items, &cfg(), 1.0);+ assert!(strat.starts_with("number:adaptive("));+ assert!(strat.contains("min=1"));+ assert!(strat.contains("max=20"));+ assert!(strat.contains("mean="));+ assert!(strat.contains("median="));+ assert!(strat.contains("p25="));+ assert!(strat.contains("p75="));+ }++ // ---------- crush_object ----------++ #[test]+ fn object_passthrough_when_few_keys() {+ let mut obj = Map::new();+ for i in 0..5 {+ obj.insert(format!("k{}", i), json!(i));+ }+ let (out, strat) = crush_object(&obj, &cfg(), 1.0);+ assert_eq!(out.len(), 5);+ assert_eq!(strat, "object:passthrough");+ }++ #[test]+ fn object_passthrough_when_total_tokens_below_min() {+ // Many tiny keys/values: total_tokens stays below+ // min_tokens_to_crush=200.+ let mut obj = Map::new();+ for i in 0..30 {+ obj.insert(format!("k{}", i), json!(i));+ }+ let (_out, strat) = crush_object(&obj, &cfg(), 1.0);+ assert_eq!(strat, "object:passthrough");+ }++ #[test]+ fn object_crushes_when_token_budget_exceeded() {+ // 30 keys, each with a long string value → total tokens > 200,+ // and unique k_total < n → actual crushing happens.+ let mut obj = Map::new();+ for i in 0..30 {+ obj.insert(+ format!("k{:02}", i),+ json!(format!(+ "this is a relatively long value string for entry number {} with content",+ i+ )),+ );+ }+ let (out, strat) = crush_object(&obj, &cfg(), 1.0);+ // Either the optimizer kept all (if it deems them all distinct+ // enough — strategy = passthrough), or it crushed.+ if strat == "object:passthrough" {+ assert_eq!(out.len(), 30);+ } else {+ assert!(strat.starts_with("object:adaptive("));+ assert!(out.len() <= 30);+ }+ }++ #[test]+ fn object_keeps_small_values() {+ // Mix of small + large values; small ones (<=12 tokens) always survive.+ let mut obj = Map::new();+ obj.insert("tiny".to_string(), json!(1));+ for i in 0..30 {+ obj.insert(+ format!("big{:02}", i),+ json!(format!(+ "this is a long string with content for entry number {} that exceeds the small threshold",+ i+ )),+ );+ }+ let (out, _) = crush_object(&obj, &cfg(), 1.0);+ assert!(out.contains_key("tiny"), "tiny key (small value) must survive");+ }++ #[test]+ fn object_keeps_error_keywords() {+ let mut obj = Map::new();+ obj.insert(+ "msg1".to_string(),+ json!(format!("FATAL: {}", "x".repeat(200))),+ );+ for i in 0..30 {+ obj.insert(+ format!("k{:02}", i),+ json!(format!("padding content for entry {} with text", i)),+ );+ }+ let (out, _) = crush_object(&obj, &cfg(), 1.0);+ assert!(+ out.contains_key("msg1"),+ "key with error-keyword value must survive"+ );+ }++ // ---------- BUG #1 documentation test ----------++ #[test]+ fn bug1_percentile_off_by_one_documented() {+ // Pin Python's buggy index choice for 9-element finite list:+ // sorted_finite[9 // 4] = sorted_finite[2] (3rd element).+ // sorted_finite[3 * 9 // 4] = sorted_finite[6] (7th element).+ //+ // For sorted [1,2,3,4,5,6,7,8,9]:+ // buggy p25 = sorted[2] = 3+ // buggy p75 = sorted[6] = 7+ //+ // Stage 3c.1 commit 7 will fix this in BOTH languages and+ // regenerate fixtures. Until then, this test pins parity.+ let mut items: Vec<Value> = (1..=9).map(|i| json!(i)).collect();+ // Pad past n>8 threshold and keep finite list exactly 9.+ items.extend(vec![json!(null); 5]); // nulls drop out of `finite`+ let (_out, strat) = crush_number_array(&items, &cfg(), 1.0);+ // Verify the buggy p25/p75 land in the strategy string.+ assert!(strat.contains("p25=3"), "got: {}", strat);+ assert!(strat.contains("p75=7"), "got: {}", strat);+ }+}diff --git a/crates/headroom-core/src/transforms/smart_crusher/mod.rs b/crates/headroom-core/src/transforms/smart_crusher/mod.rsindex 9574a97ac..8d4a91b73 100644--- a/crates/headroom-core/src/transforms/smart_crusher/mod.rs+++ b/crates/headroom-core/src/transforms/smart_crusher/mod.rs@@ -36,6 +36,7 @@ mod analyzer;mod anchors;mod classifier;mod config;+mod crushers;mod error_keywords;mod field_detect;mod hashing;@@ -48,6 +49,7 @@ pub use analyzer::SmartAnalyzer;pub use anchors::{extract_query_anchors, item_matches_anchors};pub use classifier::{classify_array, ArrayType};pub use config::SmartCrusherConfig;+pub use crushers::{compute_k_split, crush_number_array, crush_object, crush_string_array};pub use error_keywords::ERROR_KEYWORDS;pub use field_detect::{detect_id_field_statistically, detect_score_field_statistically};pub use hashing::hash_field_name;@@ -55,7 +57,7 @@ pub use outliers::{detect_error_items_for_preservation, detect_rare_status_values, detect_structural_outliers,};pub use statistics::{calculate_string_entropy, detect_sequential_pattern, is_uuid_format};-pub use stats_math::{mean, sample_stdev, sample_variance};+pub use stats_math::{format_g, mean, median, sample_stdev, sample_variance};pub use types::{ArrayAnalysis, CompressionPlan, CompressionStrategy, CrushResult, CrushabilityAnalysis,FieldStats,diff --git a/crates/headroom-core/src/transforms/smart_crusher/stats_math.rs b/crates/headroom-core/src/transforms/smart_crusher/stats_math.rsindex 7fe8b3b9a..1e8ff1d9e 100644--- a/crates/headroom-core/src/transforms/smart_crusher/stats_math.rs+++ b/crates/headroom-core/src/transforms/smart_crusher/stats_math.rs@@ -53,6 +53,94 @@ pub fn sample_stdev(values: &[f64]) -> Option<f64> {sample_variance(values).map(f64::sqrt)}+/// Median (Python `statistics.median`). Returns the middle element for+/// odd-count input, mean of two middles for even-count. Returns `None`+/// on empty input — Python raises `StatisticsError`.+///+/// Caller must pre-filter NaN/Inf if undesired (Python's median with+/// NaN gives indeterminate ordering; we sort with `total_cmp` to keep+/// behavior deterministic).+pub fn median(values: &[f64]) -> Option<f64> {+ if values.is_empty() {+ return None;+ }+ let mut sorted: Vec<f64> = values.to_vec();+ sorted.sort_by(f64::total_cmp);+ let n = sorted.len();+ if n % 2 == 0 {+ // Mean of the two middle elements.+ let lo = sorted[n / 2 - 1];+ let hi = sorted[n / 2];+ Some((lo + hi) / 2.0)+ } else {+ Some(sorted[n / 2])+ }+}++/// Approximation of Python's `f"{x:.4g}"` general-purpose float format.+///+/// Rules:+/// - 4 significant digits.+/// - Scientific notation when `exponent < -4` OR `exponent >= 4`.+/// - Trailing zeros stripped (and the `.` if all decimals removed).+/// - Scientific exponent padded to at least 2 digits with explicit sign+/// (`1.234e+04`, `1e-05`).+///+/// Used for crusher strategy debug strings. Strategy strings are+/// fixture-locked at the parity stage; if Rust's float formatting drifts+/// from CPython on some edge case, we'll catch it then. Documented+/// edge cases:+///+/// - **Banker's rounding (round half-to-even)**: Python uses banker's;+/// Rust's `format!("{:.*}", ...)` also rounds-half-to-even (per the+/// "round-to-nearest-even" IEEE 754 default). Should match.+/// - **NaN/Inf**: Python prints `nan`, `inf`, `-inf`. We mirror.+pub fn format_g(x: f64) -> String {+ if x.is_nan() {+ return "nan".to_string();+ }+ if x.is_infinite() {+ return if x > 0.0 { "inf".to_string() } else { "-inf".to_string() };+ }+ if x == 0.0 {+ return "0".to_string();+ }++ let abs = x.abs();+ let exp = abs.log10().floor() as i32;++ if !(-4..4).contains(&exp) {+ // Scientific. Python uses 4 sig figs → 3 digits after decimal in mantissa.+ let s = format!("{:.3e}", x);+ normalize_scientific_exp(&s)+ } else {+ let digits_after = (3 - exp).max(0) as usize;+ let s = format!("{:.*}", digits_after, x);+ if s.contains('.') {+ // Trim trailing zeros and a dangling decimal point.+ s.trim_end_matches('0').trim_end_matches('.').to_string()+ } else {+ s+ }+ }+}++fn normalize_scientific_exp(s: &str) -> String {+ let Some(epos) = s.find('e') else {+ return s.to_string();+ };+ let (mantissa, rest) = s.split_at(epos);+ let exp_part = &rest[1..];+ let exp_num: i32 = exp_part.parse().unwrap_or(0);+ let mantissa_clean = if mantissa.contains('.') {+ mantissa.trim_end_matches('0').trim_end_matches('.').to_string()+ } else {+ mantissa.to_string()+ };+ let sign = if exp_num >= 0 { "+" } else { "-" };+ format!("{}e{}{:02}", mantissa_clean, sign, exp_num.abs())+}+#[cfg(test)]mod tests {use super::*;@@ -131,4 +219,62 @@ mod tests {let huge = 1e200;assert_eq!(sample_stdev(&[huge, -huge]), None);}++ #[test]+ fn median_empty_is_none() {+ assert_eq!(median(&[]), None);+ }++ #[test]+ fn median_odd_count() {+ assert_eq!(median(&[3.0, 1.0, 2.0]), Some(2.0));+ }++ #[test]+ fn median_even_count_mean_of_middles() {+ // Python: statistics.median([1, 2, 3, 4]) == 2.5+ assert_eq!(median(&[4.0, 1.0, 2.0, 3.0]), Some(2.5));+ }++ #[test]+ fn median_single_element() {+ assert_eq!(median(&[42.0]), Some(42.0));+ }++ // ---------- format_g ----------++ #[test]+ fn format_g_zero_and_special() {+ assert_eq!(format_g(0.0), "0");+ assert_eq!(format_g(-0.0), "0");+ assert_eq!(format_g(f64::NAN), "nan");+ assert_eq!(format_g(f64::INFINITY), "inf");+ assert_eq!(format_g(f64::NEG_INFINITY), "-inf");+ }++ #[test]+ fn format_g_fixed_range() {+ // Python: f"{1.5:.4g}" = "1.5"+ assert_eq!(format_g(1.5), "1.5");+ // Python: f"{1.0:.4g}" = "1"+ assert_eq!(format_g(1.0), "1");+ // Python: f"{1234.0:.4g}" = "1234"+ assert_eq!(format_g(1234.0), "1234");+ // Python: f"{0.123456:.4g}" = "0.1235"+ assert_eq!(format_g(0.123456), "0.1235");+ }++ #[test]+ fn format_g_scientific_range() {+ // Python: f"{12345.678:.4g}" = "1.235e+04"+ assert_eq!(format_g(12345.678), "1.235e+04");+ // Python: f"{0.00001234:.4g}" = "1.234e-05"+ assert_eq!(format_g(0.00001234), "1.234e-05");+ }++ #[test]+ fn format_g_negative() {+ assert_eq!(format_g(-1.5), "-1.5");+ assert_eq!(format_g(-12345.678), "-1.235e+04");+ }}