Apache DataFusion SQL Query Engine
fix: correct weight handling in approx_percentile_cont_with_weight (#19941)
Sergio Estevescad38654c75f
diff --git a/datafusion/functions-aggregate-common/src/tdigest.rs b/datafusion/functions-aggregate-common/src/tdigest.rsindex 225c61b719..a7450f0eb5 100644--- a/datafusion/functions-aggregate-common/src/tdigest.rs+++ b/datafusion/functions-aggregate-common/src/tdigest.rs@@ -49,17 +49,6 @@ macro_rules! cast_scalar_f64 {};}-// Cast a non-null [`ScalarValue::UInt64`] to an [`u64`], or-// panic.-macro_rules! cast_scalar_u64 {- ($value:expr ) => {- match &$value {- ScalarValue::UInt64(Some(v)) => *v,- v => panic!("invalid type {}", v),- }- };-}-/// Centroid implementation to the cluster mentioned in the paper.#[derive(Debug, PartialEq, Clone)]pub struct Centroid {@@ -110,7 +99,7 @@ pub struct TDigest {centroids: Vec<Centroid>,max_size: usize,sum: f64,- count: u64,+ count: f64,max: f64,min: f64,}@@ -120,8 +109,8 @@ impl TDigest {TDigest {centroids: Vec::new(),max_size,- sum: 0_f64,- count: 0,+ sum: 0.0,+ count: 0.0,max: f64::NAN,min: f64::NAN,}@@ -133,14 +122,14 @@ impl TDigest {centroids: vec![centroid.clone()],max_size,sum: centroid.mean * centroid.weight,- count: 1,+ count: centroid.weight,max: centroid.mean,min: centroid.mean,}}#[inline]- pub fn count(&self) -> u64 {+ pub fn count(&self) -> f64 {self.count}@@ -170,8 +159,8 @@ impl Default for TDigest {TDigest {centroids: Vec::new(),max_size: 100,- sum: 0_f64,- count: 0,+ sum: 0.0,+ count: 0.0,max: f64::NAN,min: f64::NAN,}@@ -216,12 +205,12 @@ impl TDigest {}let mut result = TDigest::new(self.max_size());- result.count = self.count() + sorted_values.len() as u64;+ result.count = self.count() + sorted_values.len() as f64;let maybe_min = *sorted_values.first().unwrap();let maybe_max = *sorted_values.last().unwrap();- if self.count() > 0 {+ if self.count() > 0.0 {result.min = self.min.min(maybe_min);result.max = self.max.max(maybe_max);} else {@@ -233,7 +222,7 @@ impl TDigest {let mut k_limit: u64 = 1;let mut q_limit_times_count =- Self::k_to_q(k_limit, self.max_size) * result.count() as f64;+ Self::k_to_q(k_limit, self.max_size) * result.count();k_limit += 1;let mut iter_centroids = self.centroids.iter().peekable();@@ -281,7 +270,7 @@ impl TDigest {compressed.push(curr.clone());q_limit_times_count =- Self::k_to_q(k_limit, self.max_size) * result.count() as f64;+ Self::k_to_q(k_limit, self.max_size) * result.count();k_limit += 1;curr = next;}@@ -353,7 +342,7 @@ impl TDigest {let mut centroids: Vec<Centroid> = Vec::with_capacity(n_centroids);let mut starts: Vec<usize> = Vec::with_capacity(digests.len());- let mut count = 0;+ let mut count = 0.0;let mut min = f64::INFINITY;let mut max = f64::NEG_INFINITY;@@ -362,7 +351,7 @@ impl TDigest {starts.push(start);let curr_count = digest.count();- if curr_count > 0 {+ if curr_count > 0.0 {min = min.min(digest.min);max = max.max(digest.max);count += curr_count;@@ -373,6 +362,11 @@ impl TDigest {}}+ // If no centroids were added (all digests had zero count), return default+ if centroids.is_empty() {+ return TDigest::default();+ }+let mut digests_per_block: usize = 1;while digests_per_block < starts.len() {for i in (0..starts.len()).step_by(digests_per_block * 2) {@@ -397,7 +391,7 @@ impl TDigest {let mut compressed: Vec<Centroid> = Vec::with_capacity(max_size);let mut k_limit = 1;- let mut q_limit_times_count = Self::k_to_q(k_limit, max_size) * count as f64;+ let mut q_limit_times_count = Self::k_to_q(k_limit, max_size) * count;let mut iter_centroids = centroids.iter_mut();let mut curr = iter_centroids.next().unwrap();@@ -416,7 +410,7 @@ impl TDigest {sums_to_merge = 0_f64;weights_to_merge = 0_f64;compressed.push(curr.clone());- q_limit_times_count = Self::k_to_q(k_limit, max_size) * count as f64;+ q_limit_times_count = Self::k_to_q(k_limit, max_size) * count;k_limit += 1;curr = centroid;}@@ -440,7 +434,7 @@ impl TDigest {return 0.0;}- let rank = q * self.count as f64;+ let rank = q * self.count;let mut pos: usize;let mut t;@@ -450,7 +444,7 @@ impl TDigest {}pos = 0;- t = self.count as f64;+ t = self.count;for (k, centroid) in self.centroids.iter().enumerate().rev() {t -= centroid.weight();@@ -563,7 +557,7 @@ impl TDigest {vec![ScalarValue::UInt64(Some(self.max_size as u64)),ScalarValue::Float64(Some(self.sum)),- ScalarValue::UInt64(Some(self.count)),+ ScalarValue::Float64(Some(self.count)),ScalarValue::Float64(Some(self.max)),ScalarValue::Float64(Some(self.min)),ScalarValue::List(arr),@@ -611,7 +605,7 @@ impl TDigest {Self {max_size,sum: cast_scalar_f64!(state[1]),- count: cast_scalar_u64!(&state[2]),+ count: cast_scalar_f64!(state[2]),max,min,centroids,diff --git a/datafusion/functions-aggregate/src/approx_median.rs b/datafusion/functions-aggregate/src/approx_median.rsindex 739e333b54..2205b009ec 100644--- a/datafusion/functions-aggregate/src/approx_median.rs+++ b/datafusion/functions-aggregate/src/approx_median.rs@@ -110,7 +110,7 @@ impl AggregateUDFImpl for ApproxMedian {Ok(vec![Field::new(format_state_name(args.name, "max_size"), UInt64, false),Field::new(format_state_name(args.name, "sum"), Float64, false),- Field::new(format_state_name(args.name, "count"), UInt64, false),+ Field::new(format_state_name(args.name, "count"), Float64, false),Field::new(format_state_name(args.name, "max"), Float64, false),Field::new(format_state_name(args.name, "min"), Float64, false),Field::new_list(diff --git a/datafusion/functions-aggregate/src/approx_percentile_cont.rs b/datafusion/functions-aggregate/src/approx_percentile_cont.rsindex b1e649ec02..392a044d01 100644--- a/datafusion/functions-aggregate/src/approx_percentile_cont.rs+++ b/datafusion/functions-aggregate/src/approx_percentile_cont.rs@@ -259,7 +259,7 @@ impl AggregateUDFImpl for ApproxPercentileCont {),Field::new(format_state_name(args.name, "count"),- DataType::UInt64,+ DataType::Float64,false,),Field::new(@@ -436,7 +436,7 @@ impl Accumulator for ApproxPercentileAccumulator {}fn evaluate(&mut self) -> Result<ScalarValue> {- if self.digest.count() == 0 {+ if self.digest.count() == 0.0 {return ScalarValue::try_from(self.return_type.clone());}let q = self.digest.estimate_quantile(self.percentile);@@ -513,8 +513,8 @@ mod tests {ApproxPercentileAccumulator::new_with_max_size(0.5, DataType::Float64, 100);accumulator.merge_digests(&[t1]);- assert_eq!(accumulator.digest.count(), 50_000);+ assert_eq!(accumulator.digest.count(), 50_000.0);accumulator.merge_digests(&[t2]);- assert_eq!(accumulator.digest.count(), 100_000);+ assert_eq!(accumulator.digest.count(), 100_000.0);}}diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.sltindex 036bb93283..4c49fae4dc 100644--- a/datafusion/sqllogictest/test_files/aggregate.slt+++ b/datafusion/sqllogictest/test_files/aggregate.slt@@ -2029,11 +2029,12 @@ statement okINSERT INTO t1 VALUES (TRUE);# ISSUE: https://github.com/apache/datafusion/issues/12716-# This test verifies that approx_percentile_cont_with_weight does not panic when given 'NaN' and returns 'inf'+# This test verifies that approx_percentile_cont_with_weight does not panic when given 'NaN'+# With weight=0, the data point does not contribute, so result is NULLquery RSELECT approx_percentile_cont_with_weight(0, 0) WITHIN GROUP (ORDER BY 'NaN'::DOUBLE) FROM t1 WHERE t1.v1;-----Infinity+NULLstatement okDROP TABLE t1;@@ -2352,21 +2353,21 @@ e 115query TISELECT c1, approx_percentile_cont_with_weight(c2, 0.95) WITHIN GROUP (ORDER BY c3) AS c3_p95 FROM aggregate_test_100 GROUP BY 1 ORDER BY 1-----a 74+a 65b 68-c 123-d 124-e 115+c 122+d 123+e 110# approx_percentile_cont_with_weight with centroidsquery TISELECT c1, approx_percentile_cont_with_weight(c2, 0.95, 200) WITHIN GROUP (ORDER BY c3) AS c3_p95 FROM aggregate_test_100 GROUP BY 1 ORDER BY 1-----a 74+a 65b 68-c 123-d 124-e 115+c 122+d 123+e 110# csv_query_sum_crossjoinquery TTI