Apache DataFusion SQL Query Engine
fix: return NULL from `bit_xor` when a sliding frame has no non-NULL values (#26485)
Cheng-Yi Shihcee411484627
diff --git a/datafusion/functions-aggregate/src/bit_and_or_xor.rs b/datafusion/functions-aggregate/src/bit_and_or_xor.rsindex 1928f5c51c..3353d958f3 100644--- a/datafusion/functions-aggregate/src/bit_and_or_xor.rs+++ b/datafusion/functions-aggregate/src/bit_and_or_xor.rs@@ -433,7 +433,11 @@ where}struct BitXorAccumulator<T: ArrowNumericType> {- value: Option<T::Native>,+ /// Running XOR of every value seen so far, starting from the identity `0`.+ /// Only meaningful when `non_null_count` is non-zero.+ value: T::Native,+ /// Number of non-NULL values currently accumulated (inserted minus retracted)+ non_null_count: usize,}impl<T: ArrowNumericType> std::fmt::Debug for BitXorAccumulator<T> {@@ -444,7 +448,10 @@ impl<T: ArrowNumericType> std::fmt::Debug for BitXorAccumulator<T> {impl<T: ArrowNumericType> Default for BitXorAccumulator<T> {fn default() -> Self {- Self { value: None }+ Self {+ value: T::Native::usize_as(0),+ non_null_count: 0,+ }}}@@ -454,15 +461,20 @@ where{fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> {if let Some(x) = arrow::compute::bit_xor(values[0].as_primitive::<T>()) {- let v = self.value.get_or_insert_with(|| T::Native::usize_as(0));- *v = *v ^ x;+ self.value = self.value ^ x;}+ self.non_null_count += values[0].len() - values[0].null_count();Ok(())}fn retract_batch(&mut self, values: &[ArrayRef]) -> Result<()> {- // XOR is it's own inverse- self.update_batch(values)+ // XOR is its own inverse, so the value is updated as in `update_batch`;+ // only the count of non-NULL values moves the other way+ if let Some(x) = arrow::compute::bit_xor(values[0].as_primitive::<T>()) {+ self.value = self.value ^ x;+ }+ self.non_null_count -= values[0].len() - values[0].null_count();+ Ok(())}fn supports_retract_batch(&self) -> bool {@@ -470,7 +482,9 @@ where}fn evaluate(&mut self) -> Result<ScalarValue> {- ScalarValue::new_primitive::<T>(self.value, &T::DATA_TYPE)+ // All non-NULL values may have been retracted, in which case the result is NULL+ let value = (self.non_null_count > 0).then_some(self.value);+ ScalarValue::new_primitive::<T>(value, &T::DATA_TYPE)}fn size(&self) -> usize {@@ -580,7 +594,7 @@ mod tests {#[test]fn test_bit_xor_accumulator() {- let mut accumulator = BitXorAccumulator::<UInt64Type> { value: None };+ let mut accumulator = BitXorAccumulator::<UInt64Type>::default();let batches: Vec<_> = vec![vec![1, 2], vec![1]].into_iter().map(|b| Arc::new(b.into_iter().collect::<UInt64Array>()) as ArrayRef)@@ -603,4 +617,51 @@ mod tests {ScalarValue::UInt64(Some(2)));}++ #[test]+ fn test_bit_xor_accumulator_retract_to_all_null() {+ let mut accumulator = BitXorAccumulator::<UInt64Type>::default();+ let five = [Arc::new(UInt64Array::from(vec![Some(5)])) as ArrayRef];+ let nulls = [Arc::new(UInt64Array::from(vec![None])) as ArrayRef];+ let seven = [Arc::new(UInt64Array::from(vec![Some(7)])) as ArrayRef];++ accumulator.update_batch(&five).unwrap();+ assert_eq!(+ accumulator.evaluate().unwrap(),+ ScalarValue::UInt64(Some(5))+ );++ accumulator.update_batch(&nulls).unwrap();+ assert_eq!(+ accumulator.evaluate().unwrap(),+ ScalarValue::UInt64(Some(5))+ );++ // All non-NULL values left the frame, so the result is NULL+ accumulator.retract_batch(&five).unwrap();+ assert_eq!(accumulator.evaluate().unwrap(), ScalarValue::UInt64(None));++ // Retracting NULLs must not move the count+ accumulator.retract_batch(&nulls).unwrap();+ assert_eq!(accumulator.evaluate().unwrap(), ScalarValue::UInt64(None));++ accumulator.update_batch(&seven).unwrap();+ assert_eq!(+ accumulator.evaluate().unwrap(),+ ScalarValue::UInt64(Some(7))+ );+ }++ #[test]+ fn test_bit_xor_accumulator_legitimate_zero() {+ let mut accumulator = BitXorAccumulator::<UInt64Type>::default();+ let values = [Arc::new(UInt64Array::from(vec![Some(5), Some(5)])) as ArrayRef];++ // 5 ^ 5 is a real zero, not NULL+ accumulator.update_batch(&values).unwrap();+ assert_eq!(+ accumulator.evaluate().unwrap(),+ ScalarValue::UInt64(Some(0))+ );+ }}diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.sltindex 7c92514e72..cea8060c13 100644--- a/datafusion/sqllogictest/test_files/window.slt+++ b/datafusion/sqllogictest/test_files/window.slt@@ -6736,6 +6736,55 @@ ORDER BY i;3 14 NULL+# bit_xor over a sliding window must yield NULL when the frame has no non-NULL+# values, matching the result over the whole partition (apache/datafusion#26470).+# Before the fix the sliding result was 0 on the NULL row.+query III+SELECT x,+ bit_xor(x) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND CURRENT ROW) AS sliding,+ bit_xor(x) OVER (PARTITION BY i) AS whole+FROM (VALUES (1, 5), (2, CAST(NULL AS INT)), (3, 6)) t(i, x)+ORDER BY i;+----+5 5 5+NULL NULL NULL+6 6 6++# Narrow sliding frame that drains to empty each row.+query II+SELECT i, bit_xor(v) OVER (ORDER BY i ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING)+FROM (VALUES(1,CAST(NULL AS INT)),(2,1),(3,CAST(NULL AS INT)),(4,CAST(NULL AS INT))) t(i,v)+ORDER BY i;+----+1 NULL+2 NULL+3 1+4 NULL++# All-NULL input — every frame is empty.+query II+SELECT i, bit_xor(v) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)+FROM (VALUES(1,CAST(NULL AS INT)),(2,CAST(NULL AS INT))) t(i,v)+ORDER BY i;+----+1 NULL+2 NULL++# A legitimate 0 must not be turned into NULL, and NULL must follow it.+# Row 2's frame is [5, 5], whose XOR is genuinely 0. Row 4's frame is+# [NULL, NULL] and must be NULL even though the accumulated value was already 0.+# Row 5 shows the accumulator recovering once a non-NULL value re-enters.+query II+SELECT i, bit_xor(v) OVER (ORDER BY i ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)+FROM (VALUES(1,5),(2,5),(3,CAST(NULL AS INT)),(4,CAST(NULL AS INT)),(5,3)) t(i,v)+ORDER BY i;+----+1 5+2 0+3 5+4 NULL+5 3+# Covariance/correlation sliding-window regression test. Verifies correct# results across row removals and a NULL-gap empty-frame transition.query IRRR