Apache DataFusion SQL Query Engine
feat: fix AVG sliding windows wrong results with NULLs (#22139)
Oleks V7f2f78d48b6d
diff --git a/datafusion/functions-aggregate/src/average.rs b/datafusion/functions-aggregate/src/average.rsindex bcccea3813..24f2777797 100644--- a/datafusion/functions-aggregate/src/average.rs+++ b/datafusion/functions-aggregate/src/average.rs@@ -519,9 +519,16 @@ impl Accumulator for AvgAccumulator {}fn evaluate(&mut self) -> Result<ScalarValue> {- Ok(ScalarValue::Float64(- self.sum.map(|f| f / self.count as f64),- ))+ // In sliding-window mode `retract_batch` can bring `count` back to 0+ // while `sum` remains `Some(..)` (possibly zero or a floating-point+ // residual). Guard against that so the frame with no non-NULL values+ // yields NULL rather than NaN / ±Inf.+ let avg = if self.count == 0 {+ None+ } else {+ self.sum.map(|f| f / self.count as f64)+ };+ Ok(ScalarValue::Float64(avg))}fn size(&self) -> usize {@@ -584,17 +591,23 @@ impl<T: DecimalType + ArrowNumericType + Debug> Accumulator for DecimalAvgAccumu}fn evaluate(&mut self) -> Result<ScalarValue> {- let v = self- .sum- .map(|v| {- DecimalAverager::<T>::try_new(- self.sum_scale,- self.target_precision,- self.target_scale,- )?- .avg(v, T::Native::from_usize(self.count as usize).unwrap())- })- .transpose()?;+ // `count == 0` can occur in sliding-window mode after `retract_batch`+ // removes every contributing value. Return NULL rather than dividing+ // by zero (which would panic for integer decimal types).+ let v = if self.count == 0 {+ None+ } else {+ self.sum+ .map(|v| {+ DecimalAverager::<T>::try_new(+ self.sum_scale,+ self.target_precision,+ self.target_scale,+ )?+ .avg(v, T::Native::from_usize(self.count as usize).unwrap())+ })+ .transpose()?+ };ScalarValue::new_primitive::<T>(v,@@ -670,7 +683,14 @@ impl Accumulator for DurationAvgAccumulator {}fn evaluate(&mut self) -> Result<ScalarValue> {- let avg = self.sum.map(|sum| sum / self.count as i64);+ // Guard against `count == 0` which can happen in sliding-window mode+ // after every contributing value has been retracted. Without this+ // check we would integer-divide by zero.+ let avg = if self.count == 0 {+ None+ } else {+ self.sum.map(|sum| sum / self.count as i64)+ };match self.result_unit {TimeUnit::Second => Ok(ScalarValue::DurationSecond(avg)),diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.sltindex 2a74660fe9..96b811093e 100644--- a/datafusion/sqllogictest/test_files/window.slt+++ b/datafusion/sqllogictest/test_files/window.slt@@ -6456,6 +6456,54 @@ FROM (2 13 1+# AVG over a sliding window must yield NULL when the frame has no non-NULL+# values — including frames that became empty via `retract_batch`. Covers+# Float64, Decimal, and the narrow-frame retract-to-empty case.+query IR+SELECT i, AVG(v) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)+FROM (VALUES(1,1),(2,2),(3,CAST(NULL AS INT)),(4,CAST(NULL AS INT))) t(i,v)+ORDER BY i;+----+1 1.5+2 2+3 NULL+4 NULL++# All-NULL input — every frame is empty.+query IR+SELECT i, AVG(v) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)+FROM (VALUES(1,CAST(NULL AS INT)),(2,CAST(NULL AS INT))) t(i,v)+ORDER BY i;+----+1 NULL+2 NULL++# Narrow sliding frame that drains to empty each row.+query IR+SELECT i, AVG(v) OVER (ORDER BY i ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING)+FROM (VALUES(1,CAST(NULL AS INT)),(2,1),(3,CAST(NULL AS INT)),(4,CAST(NULL AS INT))) t(i,v)+ORDER BY i;+----+1 NULL+2 NULL+3 1+4 NULL++# Decimal variant — the integer-division path would otherwise panic on an+# empty frame.+query IR+SELECT i, AVG(v) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)+FROM (VALUES(1,CAST(1.5 AS DECIMAL(10,2))),+ (2,CAST(2.5 AS DECIMAL(10,2))),+ (3,CAST(NULL AS DECIMAL(10,2))),+ (4,CAST(NULL AS DECIMAL(10,2)))) t(i,v)+ORDER BY i;+----+1 2+2 2.5+3 NULL+4 NULL+# Config resetstatement okreset datafusion.execution.batch_size;