Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(rust): smart_crusher self-review — Python parity for f64 overflow path

chopratejasab4d29448748
diff --git a/crates/headroom-core/src/transforms/smart_crusher/analyzer.rs b/crates/headroom-core/src/transforms/smart_crusher/analyzer.rs
index 425dde696..f525d6aac 100644
--- a/crates/headroom-core/src/transforms/smart_crusher/analyzer.rs
+++ b/crates/headroom-core/src/transforms/smart_crusher/analyzer.rs
@@ -225,16 +225,45 @@ impl SmartAnalyzer {
.filter_map(|v| v.as_f64().filter(|f| f.is_finite()))
.collect();
if !nums.is_empty() {
- stats.min_val = nums.iter().cloned().reduce(f64::min);
- stats.max_val = nums.iter().cloned().reduce(f64::max);
- stats.mean_val = mean(&nums);
+ let min_val = nums.iter().cloned().reduce(f64::min);
+ let max_val = nums.iter().cloned().reduce(f64::max);
+ let mean_val = mean(&nums);
// `variance = 0` when n < 2 (Python: `if len(nums) > 1`).
- stats.variance = if nums.len() > 1 {
+ let variance = if nums.len() > 1 {
sample_variance(&nums)
} else {
Some(0.0)
};
- stats.change_points = self.detect_change_points(&nums, 5);
+ // Python wraps the numeric-stats block in
+ // `try/except (OverflowError, ValueError)` and resets
+ // ALL fields to None on failure. Mirror that
+ // all-or-nothing reset: if any computed stat is non-
+ // finite (or computation returned None), drop the
+ // entire numeric stats block and leave change_points
+ // empty.
+ let all_finite = mean_val.map(f64::is_finite).unwrap_or(false)
+ && variance.map(f64::is_finite).unwrap_or(false)
+ && min_val.map(f64::is_finite).unwrap_or(false)
+ && max_val.map(f64::is_finite).unwrap_or(false);
+ if all_finite {
+ stats.min_val = min_val;
+ stats.max_val = max_val;
+ stats.mean_val = mean_val;
+ stats.variance = variance;
+ stats.change_points = self.detect_change_points(&nums, 5);
+ } else {
+ // Python parity: the except block sets `variance = 0`
+ // (int literal) but min/max/mean to None. Downstream
+ // truthiness checks (`if stats.variance:` and
+ // `(variance or 0) > 0`) treat 0 the same as None,
+ // but the FieldStats serialization shape matters
+ // for parity fixtures. Pin variance to Some(0.0).
+ stats.min_val = None;
+ stats.max_val = None;
+ stats.mean_val = None;
+ stats.variance = Some(0.0);
+ stats.change_points = Vec::new();
+ }
}
}
"string" => {
@@ -879,6 +908,30 @@ mod tests {
assert!((v - 9.166666666666666).abs() < 1e-9);
}
+ #[test]
+ fn analyze_field_numeric_overflow_resets_all_stats_to_none() {
+ // Python parity: when stats computation overflows, the
+ // `try/except (OverflowError, ValueError)` block resets ALL
+ // numeric fields to None. We mirror by checking finiteness across
+ // the bundle and dropping the whole numeric stats group on
+ // failure.
+ let huge = 1e200;
+ // Two extreme opposite values: variance overflows.
+ let items = vec![json!({"n": huge}), json!({"n": -huge})];
+ let s = analyzer().analyze_field("n", &items);
+ assert_eq!(s.field_type, "numeric");
+ // Per Python: min/max/mean reset to None; variance = 0 (int);
+ // change_points empty.
+ assert_eq!(s.min_val, None);
+ assert_eq!(s.max_val, None);
+ assert_eq!(s.mean_val, None);
+ assert_eq!(s.variance, Some(0.0));
+ assert!(s.change_points.is_empty());
+ // Non-numeric stats (count, unique, is_constant) should still hold.
+ assert_eq!(s.count, 2);
+ assert_eq!(s.unique_count, 2);
+ }
+
#[test]
fn analyze_field_numeric_filters_nan_and_inf() {
// Tricky: serde_json doesn't allow NaN/Inf in JSON, so we build a
diff --git a/crates/headroom-core/src/transforms/smart_crusher/stats_math.rs b/crates/headroom-core/src/transforms/smart_crusher/stats_math.rs
index 27aae5c2c..7fe8b3b9a 100644
--- a/crates/headroom-core/src/transforms/smart_crusher/stats_math.rs
+++ b/crates/headroom-core/src/transforms/smart_crusher/stats_math.rs
@@ -10,28 +10,45 @@
/// Arithmetic mean. Returns `None` on empty input — Python's
/// `statistics.mean([])` raises `StatisticsError`; we model that as
/// "no value to return", and callers must handle it.
+///
+/// Also returns `None` if the result is non-finite (Inf/NaN). Python's
+/// numeric-stats path in `_analyze_field` is wrapped in
+/// `try/except (OverflowError, ValueError)` and resets all stats to
+/// `None` on failure; mirroring that here keeps parity on extreme floats.
pub fn mean(values: &[f64]) -> Option<f64> {
if values.is_empty() {
return None;
}
let sum: f64 = values.iter().sum();
- Some(sum / values.len() as f64)
+ let m = sum / values.len() as f64;
+ if m.is_finite() {
+ Some(m)
+ } else {
+ None
+ }
}
/// Sample variance with `n-1` denominator (Python `statistics.variance`).
/// Requires at least 2 values; returns `None` for fewer (mirrors
-/// Python which raises `StatisticsError` for n < 2).
+/// Python which raises `StatisticsError` for n < 2). Also returns `None`
+/// on non-finite results — see `mean` for rationale.
pub fn sample_variance(values: &[f64]) -> Option<f64> {
if values.len() < 2 {
return None;
}
let m = mean(values)?;
let sum_sq_diff: f64 = values.iter().map(|v| (v - m).powi(2)).sum();
- Some(sum_sq_diff / (values.len() - 1) as f64)
+ let var = sum_sq_diff / (values.len() - 1) as f64;
+ if var.is_finite() {
+ Some(var)
+ } else {
+ None
+ }
}
/// Sample standard deviation — sqrt of `sample_variance`. Same n>=2
-/// requirement as the variance helper.
+/// requirement as the variance helper. `None` propagates from
+/// `sample_variance`, including on non-finite inputs.
pub fn sample_stdev(values: &[f64]) -> Option<f64> {
sample_variance(values).map(f64::sqrt)
}
@@ -88,4 +105,30 @@ mod tests {
let v = sample_variance(&[7.0, 7.0, 7.0]).unwrap();
assert!(approx_eq(v, 0.0));
}
+
+ #[test]
+ fn mean_non_finite_overflow_returns_none() {
+ // Python parity: extreme inputs that overflow during sum cause
+ // `statistics.mean` to raise OverflowError, which `_analyze_field`
+ // treats as "no stats". We mirror by returning None.
+ let huge = f64::MAX / 2.0;
+ let nums = vec![huge, huge, huge, huge];
+ // Sum overflows to +Inf, mean = Inf — non-finite, must be None.
+ assert_eq!(mean(&nums), None);
+ }
+
+ #[test]
+ fn sample_variance_non_finite_returns_none() {
+ // Squared-diff sum overflows. Mirrors Python's OverflowError path.
+ let huge = 1e200;
+ let v = sample_variance(&[huge, -huge]);
+ // (1e200 - 0)² + (-1e200 - 0)² overflows → variance is Inf → None.
+ assert_eq!(v, None);
+ }
+
+ #[test]
+ fn sample_stdev_non_finite_returns_none() {
+ let huge = 1e200;
+ assert_eq!(sample_stdev(&[huge, -huge]), None);
+ }
}