Apache DataFusion SQL Query Engine
fix: xxhash64 of a NULL struct ignores the struct fields (#25774)
mrhard9090f56ae979e29f
diff --git a/datafusion/spark/src/function/hash/utils.rs b/datafusion/spark/src/function/hash/utils.rsindex e7918d33ec..68ee9e8337 100644--- a/datafusion/spark/src/function/hash/utils.rs+++ b/datafusion/spark/src/function/hash/utils.rs@@ -984,7 +984,20 @@ macro_rules! create_hashes_internal {// Hash each field of the struct - Spark hashes all fields recursivelylet columns: Vec<ArrayRef> = struct_array.columns().to_vec();if !columns.is_empty() {- $recursive_hash_method(&columns, $hashes_buffer)?;+ match struct_array.nulls().filter(|nulls| nulls.null_count() > 0) {+ // A NULL struct leaves the hash unchanged like any other+ // NULL, whatever its fields still hold.+ Some(nulls) => {+ let before = $hashes_buffer.to_vec();+ $recursive_hash_method(&columns, $hashes_buffer)?;+ for (i, hash) in $hashes_buffer.iter_mut().enumerate() {+ if nulls.is_null(i) {+ *hash = before[i];+ }+ }+ }+ None => $recursive_hash_method(&columns, $hashes_buffer)?,+ }}}DataType::Map(field, _) => {diff --git a/datafusion/spark/src/function/hash/xxhash64.rs b/datafusion/spark/src/function/hash/xxhash64.rsindex 9d02a51b22..d1d62d6e4a 100644--- a/datafusion/spark/src/function/hash/xxhash64.rs+++ b/datafusion/spark/src/function/hash/xxhash64.rs@@ -423,6 +423,34 @@ mod tests {assert_ne!(hashes[1], hashes[2]);}+ #[test]+ fn test_xxhash64_null_struct_ignores_fields() {+ use arrow::array::StructArray;+ use arrow::buffer::NullBuffer;+ use arrow::datatypes::{Field, Fields};++ // Row 1 is NULL but its fields still hold values.+ let fields = Fields::from(vec![+ Field::new("a", DataType::Int32, false),+ Field::new("b", DataType::Utf8, false),+ ]);+ let struct_array = StructArray::new(+ fields,+ vec![+ Arc::new(Int32Array::from(vec![1, 1])) as ArrayRef,+ Arc::new(StringArray::from(vec!["x", "x"])) as ArrayRef,+ ],+ Some(NullBuffer::from(vec![true, false])),+ );+ let array_ref: ArrayRef = Arc::new(struct_array);++ let mut hashes = vec![DEFAULT_SEED; 2];+ create_xxhash64_hashes(&[array_ref], &mut hashes).unwrap();++ assert_ne!(hashes[0], DEFAULT_SEED);+ assert_eq!(hashes[1], DEFAULT_SEED);+ }+#[test]fn test_xxhash64_list() {use arrow::array::ListArray;diff --git a/datafusion/sqllogictest/test_files/spark/hash/xxhash64.slt b/datafusion/sqllogictest/test_files/spark/hash/xxhash64.sltindex cbc14967c4..c4b5ad0d59 100644--- a/datafusion/sqllogictest/test_files/spark/hash/xxhash64.slt+++ b/datafusion/sqllogictest/test_files/spark/hash/xxhash64.slt@@ -195,3 +195,19 @@ SELECT id, xxhash64(id, name) FROM xxhash_test ORDER BY id;statement okDROP TABLE xxhash_test;++# A NULL struct hashes like any other NULL, whatever its fields still hold+query BII+SELECT s IS NULL,+ xxhash64(s),+ xxhash64(arrow_cast(NULL, 'Struct("a": Int64, "b": Utf8)'))+FROM (SELECT nullif(s, s) AS s FROM (SELECT named_struct('a', 1, 'b', 'x') AS s));+----+true 42 42++# A NULL struct in a later argument leaves the incoming hash unchanged+query II+SELECT xxhash64(7, nullif(s, s)), xxhash64(7)+FROM (SELECT named_struct('a', 1, 'b', 'x') AS s);+----+-2864211556195259663 -2864211556195259663