Apache DataFusion SQL Query Engine
fix: sliding window `min()` returns wrong value for all-NULL windows (#23874)
Neil Conway2028927e1f5f
diff --git a/datafusion/functions-aggregate/src/min_max.rs b/datafusion/functions-aggregate/src/min_max.rsindex b306e7db2f..41643747e8 100644--- a/datafusion/functions-aggregate/src/min_max.rs+++ b/datafusion/functions-aggregate/src/min_max.rs@@ -381,7 +381,8 @@ impl AggregateUDFImpl for Max {#[derive(Debug)]pub struct SlidingMaxAccumulator {- max: ScalarValue,+ /// Typed NULL returned when the window contains no non-null values+ empty_value: ScalarValue,moving_max: MovingMax<ScalarValue>,}@@ -389,30 +390,38 @@ impl SlidingMaxAccumulator {/// new max accumulatorpub fn try_new(datatype: &DataType) -> Result<Self> {Ok(Self {- max: ScalarValue::try_from(datatype)?,+ empty_value: ScalarValue::try_from(datatype)?,moving_max: MovingMax::<ScalarValue>::new(),})}++ fn current_max(&self) -> ScalarValue {+ match self.moving_max.max() {+ Some(res) => res.clone(),+ None => self.empty_value.clone(),+ }+ }}impl Accumulator for SlidingMaxAccumulator {fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> {for idx in 0..values[0].len() {let val = ScalarValue::try_from_array(&values[0], idx)?;- self.moving_max.push(val);- }- if let Some(res) = self.moving_max.max() {- self.max = res.clone();+ if !val.is_null() {+ self.moving_max.push(val);+ }}Ok(())}fn retract_batch(&mut self, values: &[ArrayRef]) -> Result<()> {- for _idx in 0..values[0].len() {- (self.moving_max).pop();- }- if let Some(res) = self.moving_max.max() {- self.max = res.clone();+ // We assume that values are retracted in the order they were added, so+ // the retracted values must be the oldest elements of `moving_max`.+ // NULLs are never pushed, so be sure to only pop once per non-NULL+ // value.+ let valid_count = values[0].len() - values[0].logical_null_count();+ for _ in 0..valid_count {+ self.moving_max.pop();}Ok(())}@@ -422,11 +431,11 @@ impl Accumulator for SlidingMaxAccumulator {}fn state(&mut self) -> Result<Vec<ScalarValue>> {- Ok(vec![self.max.clone()])+ Ok(vec![self.current_max()])}fn evaluate(&mut self) -> Result<ScalarValue> {- Ok(self.max.clone())+ Ok(self.current_max())}fn supports_retract_batch(&self) -> bool {@@ -434,8 +443,8 @@ impl Accumulator for SlidingMaxAccumulator {}fn size(&self) -> usize {- size_of_val(self) - size_of_val(&self.max)- + self.max.size()+ size_of_val(self) - size_of_val(&self.empty_value)+ + self.empty_value.size()+ self.moving_max.heap_size(|sv| sv.size() - size_of_val(sv))}}@@ -667,22 +676,30 @@ impl AggregateUDFImpl for Min {#[derive(Debug)]pub struct SlidingMinAccumulator {- min: ScalarValue,+ /// Typed NULL returned when the window contains no non-null values+ empty_value: ScalarValue,moving_min: MovingMin<ScalarValue>,}impl SlidingMinAccumulator {pub fn try_new(datatype: &DataType) -> Result<Self> {Ok(Self {- min: ScalarValue::try_from(datatype)?,+ empty_value: ScalarValue::try_from(datatype)?,moving_min: MovingMin::<ScalarValue>::new(),})}++ fn current_min(&self) -> ScalarValue {+ match self.moving_min.min() {+ Some(res) => res.clone(),+ None => self.empty_value.clone(),+ }+ }}impl Accumulator for SlidingMinAccumulator {fn state(&mut self) -> Result<Vec<ScalarValue>> {- Ok(vec![self.min.clone()])+ Ok(vec![self.current_min()])}fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> {@@ -692,21 +709,17 @@ impl Accumulator for SlidingMinAccumulator {self.moving_min.push(val);}}- if let Some(res) = self.moving_min.min() {- self.min = res.clone();- }Ok(())}fn retract_batch(&mut self, values: &[ArrayRef]) -> Result<()> {- for idx in 0..values[0].len() {- let val = ScalarValue::try_from_array(&values[0], idx)?;- if !val.is_null() {- (self.moving_min).pop();- }- }- if let Some(res) = self.moving_min.min() {- self.min = res.clone();+ // We assume that values are retracted in the order they were added, so+ // the retracted values must be the oldest elements of `moving_min`.+ // NULLs are never pushed, so be sure to only pop once per non-NULL+ // value.+ let valid_count = values[0].len() - values[0].logical_null_count();+ for _ in 0..valid_count {+ self.moving_min.pop();}Ok(())}@@ -716,7 +729,7 @@ impl Accumulator for SlidingMinAccumulator {}fn evaluate(&mut self) -> Result<ScalarValue> {- Ok(self.min.clone())+ Ok(self.current_min())}fn supports_retract_batch(&self) -> bool {@@ -724,8 +737,8 @@ impl Accumulator for SlidingMinAccumulator {}fn size(&self) -> usize {- size_of_val(self) - size_of_val(&self.min)- + self.min.size()+ size_of_val(self) - size_of_val(&self.empty_value)+ + self.empty_value.size()+ self.moving_min.heap_size(|sv| sv.size() - size_of_val(sv))}}@@ -1193,6 +1206,58 @@ mod tests {Ok(())}+ #[test]+ fn sliding_min_all_null_window() -> Result<()> {+ let mut min_acc = SlidingMinAccumulator::try_new(&DataType::Int32)?;++ let values: ArrayRef = Arc::new(Int32Array::from(vec![Some(3), None]));+ min_acc.update_batch(&[Arc::clone(&values)])?;+ assert_eq!(min_acc.evaluate()?, ScalarValue::Int32(Some(3)));++ // Retract `3`; the window now contains only the NULL+ let retracted: ArrayRef = Arc::new(Int32Array::from(vec![Some(3)]));+ min_acc.retract_batch(&[Arc::clone(&retracted)])?;+ assert_eq!(min_acc.evaluate()?, ScalarValue::Int32(None));++ // A subsequent non-null value must be picked up again+ let update: ArrayRef = Arc::new(Int32Array::from(vec![Some(7)]));+ min_acc.update_batch(&[Arc::clone(&update)])?;+ assert_eq!(min_acc.evaluate()?, ScalarValue::Int32(Some(7)));++ // Retracting the NULL row must not pop the remaining value+ let null_row: ArrayRef = Arc::new(Int32Array::from(vec![None::<i32>]));+ min_acc.retract_batch(&[Arc::clone(&null_row)])?;+ assert_eq!(min_acc.evaluate()?, ScalarValue::Int32(Some(7)));++ Ok(())+ }++ #[test]+ fn sliding_max_all_null_window() -> Result<()> {+ let mut max_acc = SlidingMaxAccumulator::try_new(&DataType::Int32)?;++ let values: ArrayRef = Arc::new(Int32Array::from(vec![Some(3), None]));+ max_acc.update_batch(&[Arc::clone(&values)])?;+ assert_eq!(max_acc.evaluate()?, ScalarValue::Int32(Some(3)));++ // Retract `3`; the window now contains only the NULL+ let retracted: ArrayRef = Arc::new(Int32Array::from(vec![Some(3)]));+ max_acc.retract_batch(&[Arc::clone(&retracted)])?;+ assert_eq!(max_acc.evaluate()?, ScalarValue::Int32(None));++ // A subsequent non-null value must be picked up again+ let update: ArrayRef = Arc::new(Int32Array::from(vec![Some(7)]));+ max_acc.update_batch(&[Arc::clone(&update)])?;+ assert_eq!(max_acc.evaluate()?, ScalarValue::Int32(Some(7)));++ // Retracting the NULL row must not disturb the remaining value+ let null_row: ArrayRef = Arc::new(Int32Array::from(vec![None::<i32>]));+ max_acc.retract_batch(&[Arc::clone(&null_row)])?;+ assert_eq!(max_acc.evaluate()?, ScalarValue::Int32(Some(7)));++ Ok(())+ }+#[test]fn moving_min_tests() -> Result<()> {moving_min_i32(100, 10)?;diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.sltindex cbbd9b74df..fd477a3386 100644--- a/datafusion/sqllogictest/test_files/window.slt+++ b/datafusion/sqllogictest/test_files/window.slt@@ -6842,3 +6842,17 @@ DROP TABLE issue_20194_t1;statement okDROP TABLE issue_20194_t2;++# Sliding-window MIN/MAX over a frame whose non-NULL values have all been+# retracted should yield NULL.+query IIII+SELECT id, x,+ MIN(x) OVER (ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS min_x,+ MAX(x) OVER (ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS max_x+FROM (VALUES (1, 3), (2, NULL), (3, NULL), (4, 7)) t(id, x)+ORDER BY id+----+1 3 3 3+2 NULL 3 3+3 NULL NULL NULL+4 7 7 7