Apache DataFusion SQL Query Engine
feat: Handle edge case with `corr` with single row and `NaN` (#18677)
Oleks V2b3b220839a2
diff --git a/datafusion/functions-aggregate/src/correlation.rs b/datafusion/functions-aggregate/src/correlation.rsindex f2a464de41..4cac159ebb 100644--- a/datafusion/functions-aggregate/src/correlation.rs+++ b/datafusion/functions-aggregate/src/correlation.rs@@ -196,15 +196,24 @@ impl Accumulator for CorrelationAccumulator {}fn evaluate(&mut self) -> Result<ScalarValue> {- let n = self.covar.get_count();- if n < 2 {- return Ok(ScalarValue::Float64(None));- }-let covar = self.covar.evaluate()?;let stddev1 = self.stddev1.evaluate()?;let stddev2 = self.stddev2.evaluate()?;+ // First check if we have NaN values by examining the internal state+ // This handles the case where both inputs are NaN even with count=1+ let mean1 = self.covar.get_mean1();+ let mean2 = self.covar.get_mean2();++ // If both means are NaN, then both input columns contain only NaN values+ if mean1.is_nan() && mean2.is_nan() {+ return Ok(ScalarValue::Float64(Some(f64::NAN)));+ }+ let n = self.covar.get_count();+ if mean1.is_nan() || mean2.is_nan() || n < 2 {+ return Ok(ScalarValue::Float64(None));+ }+if let ScalarValue::Float64(Some(c)) = covar {if let ScalarValue::Float64(Some(s1)) = stddev1 {if let ScalarValue::Float64(Some(s2)) = stddev2 {@@ -402,54 +411,6 @@ impl GroupsAccumulator for CorrelationGroupsAccumulator {Ok(())}- fn merge_batch(- &mut self,- values: &[ArrayRef],- group_indices: &[usize],- opt_filter: Option<&BooleanArray>,- total_num_groups: usize,- ) -> Result<()> {- // Resize vectors to accommodate total number of groups- self.count.resize(total_num_groups, 0);- self.sum_x.resize(total_num_groups, 0.0);- self.sum_y.resize(total_num_groups, 0.0);- self.sum_xy.resize(total_num_groups, 0.0);- self.sum_xx.resize(total_num_groups, 0.0);- self.sum_yy.resize(total_num_groups, 0.0);-- // Extract arrays from input values- let partial_counts = values[0].as_primitive::<UInt64Type>();- let partial_sum_x = values[1].as_primitive::<Float64Type>();- let partial_sum_y = values[2].as_primitive::<Float64Type>();- let partial_sum_xy = values[3].as_primitive::<Float64Type>();- let partial_sum_xx = values[4].as_primitive::<Float64Type>();- let partial_sum_yy = values[5].as_primitive::<Float64Type>();-- assert!(opt_filter.is_none(), "aggregate filter should be applied in partial stage, there should be no filter in final stage");-- accumulate_correlation_states(- group_indices,- (- partial_counts,- partial_sum_x,- partial_sum_y,- partial_sum_xy,- partial_sum_xx,- partial_sum_yy,- ),- |group_index, count, values| {- self.count[group_index] += count;- self.sum_x[group_index] += values[0];- self.sum_y[group_index] += values[1];- self.sum_xy[group_index] += values[2];- self.sum_xx[group_index] += values[3];- self.sum_yy[group_index] += values[4];- },- );-- Ok(())- }-fn evaluate(&mut self, emit_to: EmitTo) -> Result<ArrayRef> {let n = match emit_to {EmitTo::All => self.count.len(),@@ -465,14 +426,10 @@ impl GroupsAccumulator for CorrelationGroupsAccumulator {// - Correlation can't be calculated when a group only has 1 record, or when// the `denominator` state is 0. In these cases, the final aggregation// result should be `Null` (according to PostgreSQL's behavior).+ // - However, if any of the accumulated values contain NaN, the result should+ // be NaN regardless of the count (even for single-row groups).//for i in 0..n {- if self.count[i] < 2 {- values.push(0.0);- nulls.append_null();- continue;- }-let count = self.count[i];let sum_x = self.sum_x[i];let sum_y = self.sum_y[i];@@ -480,6 +437,20 @@ impl GroupsAccumulator for CorrelationGroupsAccumulator {let sum_xx = self.sum_xx[i];let sum_yy = self.sum_yy[i];+ // If BOTH sum_x AND sum_y are NaN, then both input values are NaN → return NaN+ // If only ONE of them is NaN, then only one input value is NaN → return NULL+ if sum_x.is_nan() && sum_y.is_nan() {+ // Both inputs are NaN → return NaN+ values.push(f64::NAN);+ nulls.append_non_null();+ continue;+ } else if count < 2 || sum_x.is_nan() || sum_y.is_nan() {+ // Only one input is NaN → return NULL+ values.push(0.0);+ nulls.append_null();+ continue;+ }+let mean_x = sum_x / count as f64;let mean_y = sum_y / count as f64;@@ -515,6 +486,54 @@ impl GroupsAccumulator for CorrelationGroupsAccumulator {])}+ fn merge_batch(+ &mut self,+ values: &[ArrayRef],+ group_indices: &[usize],+ opt_filter: Option<&BooleanArray>,+ total_num_groups: usize,+ ) -> Result<()> {+ // Resize vectors to accommodate total number of groups+ self.count.resize(total_num_groups, 0);+ self.sum_x.resize(total_num_groups, 0.0);+ self.sum_y.resize(total_num_groups, 0.0);+ self.sum_xy.resize(total_num_groups, 0.0);+ self.sum_xx.resize(total_num_groups, 0.0);+ self.sum_yy.resize(total_num_groups, 0.0);++ // Extract arrays from input values+ let partial_counts = values[0].as_primitive::<UInt64Type>();+ let partial_sum_x = values[1].as_primitive::<Float64Type>();+ let partial_sum_y = values[2].as_primitive::<Float64Type>();+ let partial_sum_xy = values[3].as_primitive::<Float64Type>();+ let partial_sum_xx = values[4].as_primitive::<Float64Type>();+ let partial_sum_yy = values[5].as_primitive::<Float64Type>();++ assert!(opt_filter.is_none(), "aggregate filter should be applied in partial stage, there should be no filter in final stage");++ accumulate_correlation_states(+ group_indices,+ (+ partial_counts,+ partial_sum_x,+ partial_sum_y,+ partial_sum_xy,+ partial_sum_xx,+ partial_sum_yy,+ ),+ |group_index, count, values| {+ self.count[group_index] += count;+ self.sum_x[group_index] += values[0];+ self.sum_y[group_index] += values[1];+ self.sum_xy[group_index] += values[2];+ self.sum_xx[group_index] += values[3];+ self.sum_yy[group_index] += values[4];+ },+ );++ Ok(())+ }+fn size(&self) -> usize {size_of_val(&self.count)+ size_of_val(&self.sum_x)diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.sltindex 8dce114ab5..e81bfb72a0 100644--- a/datafusion/sqllogictest/test_files/aggregate.slt+++ b/datafusion/sqllogictest/test_files/aggregate.slt@@ -607,6 +607,70 @@ from data----1+# group correlation_query_with_nans_f32+query IR+select id, corr(f, b)+from values+ (1, 1, 'nan'::float),+ (2, 'nan'::float, 1),+ (3, 'nan'::float, null),+ (4, null, 'nan'::float),+ (5, 'nan'::float, 'nan'::float),+ (5, 1, 1),+ (5, 2, 2),+ (6, 'nan'::float, 'nan'::float) t(id, f, b)+group by id+order by id+----+1 NULL+2 NULL+3 NULL+4 NULL+5 NaN+6 NaN++# correlation_query_with_nans_f32+query RR+with data as (+ select 'nan'::float as f, 'nan'::float as b+)+select corr(f, b), corr('nan'::float, 'nan'::float)+from data+----+NaN NaN++# group correlation_query_with_nans_f64+query IR+select id, corr(f, b)+from values+ (1, 1, 'nan'::double),+ (2, 'nan'::double, 1),+ (3, 'nan'::double, null),+ (4, null, 'nan'::float),+ (5, 'nan'::double, 'nan'::double),+ (5, 1, 1),+ (5, 2, 2),+ (6, 'nan'::double, 'nan'::double) t(id, f, b)+group by id+order by id+----+1 NULL+2 NULL+3 NULL+4 NULL+5 NaN+6 NaN++# correlation_query_with_nans_f64+query RR+with data as (+ select 'nan'::double as f, 'nan'::double as b+)+select corr(f, b), corr('nan'::double, 'nan'::double)+from data+----+NaN NaN+# csv_query_variance_1query RSELECT var_pop(c2) FROM aggregate_test_100