Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(rust): smart_crusher self-review — Python parity for f64 overflow path
chopratejasab4d29448748
diff --git a/crates/headroom-core/src/transforms/smart_crusher/analyzer.rs b/crates/headroom-core/src/transforms/smart_crusher/analyzer.rsindex 425dde696..f525d6aac 100644--- a/crates/headroom-core/src/transforms/smart_crusher/analyzer.rs+++ b/crates/headroom-core/src/transforms/smart_crusher/analyzer.rs@@ -225,16 +225,45 @@ impl SmartAnalyzer {.filter_map(|v| v.as_f64().filter(|f| f.is_finite())).collect();if !nums.is_empty() {- stats.min_val = nums.iter().cloned().reduce(f64::min);- stats.max_val = nums.iter().cloned().reduce(f64::max);- stats.mean_val = mean(&nums);+ let min_val = nums.iter().cloned().reduce(f64::min);+ let max_val = nums.iter().cloned().reduce(f64::max);+ let mean_val = mean(&nums);// `variance = 0` when n < 2 (Python: `if len(nums) > 1`).- stats.variance = if nums.len() > 1 {+ let variance = if nums.len() > 1 {sample_variance(&nums)} else {Some(0.0)};- stats.change_points = self.detect_change_points(&nums, 5);+ // Python wraps the numeric-stats block in+ // `try/except (OverflowError, ValueError)` and resets+ // ALL fields to None on failure. Mirror that+ // all-or-nothing reset: if any computed stat is non-+ // finite (or computation returned None), drop the+ // entire numeric stats block and leave change_points+ // empty.+ let all_finite = mean_val.map(f64::is_finite).unwrap_or(false)+ && variance.map(f64::is_finite).unwrap_or(false)+ && min_val.map(f64::is_finite).unwrap_or(false)+ && max_val.map(f64::is_finite).unwrap_or(false);+ if all_finite {+ stats.min_val = min_val;+ stats.max_val = max_val;+ stats.mean_val = mean_val;+ stats.variance = variance;+ stats.change_points = self.detect_change_points(&nums, 5);+ } else {+ // Python parity: the except block sets `variance = 0`+ // (int literal) but min/max/mean to None. Downstream+ // truthiness checks (`if stats.variance:` and+ // `(variance or 0) > 0`) treat 0 the same as None,+ // but the FieldStats serialization shape matters+ // for parity fixtures. Pin variance to Some(0.0).+ stats.min_val = None;+ stats.max_val = None;+ stats.mean_val = None;+ stats.variance = Some(0.0);+ stats.change_points = Vec::new();+ }}}"string" => {@@ -879,6 +908,30 @@ mod tests {assert!((v - 9.166666666666666).abs() < 1e-9);}+ #[test]+ fn analyze_field_numeric_overflow_resets_all_stats_to_none() {+ // Python parity: when stats computation overflows, the+ // `try/except (OverflowError, ValueError)` block resets ALL+ // numeric fields to None. We mirror by checking finiteness across+ // the bundle and dropping the whole numeric stats group on+ // failure.+ let huge = 1e200;+ // Two extreme opposite values: variance overflows.+ let items = vec![json!({"n": huge}), json!({"n": -huge})];+ let s = analyzer().analyze_field("n", &items);+ assert_eq!(s.field_type, "numeric");+ // Per Python: min/max/mean reset to None; variance = 0 (int);+ // change_points empty.+ assert_eq!(s.min_val, None);+ assert_eq!(s.max_val, None);+ assert_eq!(s.mean_val, None);+ assert_eq!(s.variance, Some(0.0));+ assert!(s.change_points.is_empty());+ // Non-numeric stats (count, unique, is_constant) should still hold.+ assert_eq!(s.count, 2);+ assert_eq!(s.unique_count, 2);+ }+#[test]fn analyze_field_numeric_filters_nan_and_inf() {// Tricky: serde_json doesn't allow NaN/Inf in JSON, so we build adiff --git a/crates/headroom-core/src/transforms/smart_crusher/stats_math.rs b/crates/headroom-core/src/transforms/smart_crusher/stats_math.rsindex 27aae5c2c..7fe8b3b9a 100644--- a/crates/headroom-core/src/transforms/smart_crusher/stats_math.rs+++ b/crates/headroom-core/src/transforms/smart_crusher/stats_math.rs@@ -10,28 +10,45 @@/// Arithmetic mean. Returns `None` on empty input — Python's/// `statistics.mean([])` raises `StatisticsError`; we model that as/// "no value to return", and callers must handle it.+///+/// Also returns `None` if the result is non-finite (Inf/NaN). Python's+/// numeric-stats path in `_analyze_field` is wrapped in+/// `try/except (OverflowError, ValueError)` and resets all stats to+/// `None` on failure; mirroring that here keeps parity on extreme floats.pub fn mean(values: &[f64]) -> Option<f64> {if values.is_empty() {return None;}let sum: f64 = values.iter().sum();- Some(sum / values.len() as f64)+ let m = sum / values.len() as f64;+ if m.is_finite() {+ Some(m)+ } else {+ None+ }}/// Sample variance with `n-1` denominator (Python `statistics.variance`)./// Requires at least 2 values; returns `None` for fewer (mirrors-/// Python which raises `StatisticsError` for n < 2).+/// Python which raises `StatisticsError` for n < 2). Also returns `None`+/// on non-finite results — see `mean` for rationale.pub fn sample_variance(values: &[f64]) -> Option<f64> {if values.len() < 2 {return None;}let m = mean(values)?;let sum_sq_diff: f64 = values.iter().map(|v| (v - m).powi(2)).sum();- Some(sum_sq_diff / (values.len() - 1) as f64)+ let var = sum_sq_diff / (values.len() - 1) as f64;+ if var.is_finite() {+ Some(var)+ } else {+ None+ }}/// Sample standard deviation — sqrt of `sample_variance`. Same n>=2-/// requirement as the variance helper.+/// requirement as the variance helper. `None` propagates from+/// `sample_variance`, including on non-finite inputs.pub fn sample_stdev(values: &[f64]) -> Option<f64> {sample_variance(values).map(f64::sqrt)}@@ -88,4 +105,30 @@ mod tests {let v = sample_variance(&[7.0, 7.0, 7.0]).unwrap();assert!(approx_eq(v, 0.0));}++ #[test]+ fn mean_non_finite_overflow_returns_none() {+ // Python parity: extreme inputs that overflow during sum cause+ // `statistics.mean` to raise OverflowError, which `_analyze_field`+ // treats as "no stats". We mirror by returning None.+ let huge = f64::MAX / 2.0;+ let nums = vec![huge, huge, huge, huge];+ // Sum overflows to +Inf, mean = Inf — non-finite, must be None.+ assert_eq!(mean(&nums), None);+ }++ #[test]+ fn sample_variance_non_finite_returns_none() {+ // Squared-diff sum overflows. Mirrors Python's OverflowError path.+ let huge = 1e200;+ let v = sample_variance(&[huge, -huge]);+ // (1e200 - 0)² + (-1e200 - 0)² overflows → variance is Inf → None.+ assert_eq!(v, None);+ }++ #[test]+ fn sample_stdev_non_finite_returns_none() {+ let huge = 1e200;+ assert_eq!(sample_stdev(&[huge, -huge]), None);+ }}