Skip to content

Apache DataFusion SQL Query Engine

fix: xxhash64 of a NULL struct ignores the struct fields (#25774)

mrhard9090f56ae979e29f
diff --git a/datafusion/spark/src/function/hash/utils.rs b/datafusion/spark/src/function/hash/utils.rs
index e7918d33ec..68ee9e8337 100644
--- a/datafusion/spark/src/function/hash/utils.rs
+++ b/datafusion/spark/src/function/hash/utils.rs
@@ -984,7 +984,20 @@ macro_rules! create_hashes_internal {
// Hash each field of the struct - Spark hashes all fields recursively
let columns: Vec<ArrayRef> = struct_array.columns().to_vec();
if !columns.is_empty() {
- $recursive_hash_method(&columns, $hashes_buffer)?;
+ match struct_array.nulls().filter(|nulls| nulls.null_count() > 0) {
+ // A NULL struct leaves the hash unchanged like any other
+ // NULL, whatever its fields still hold.
+ Some(nulls) => {
+ let before = $hashes_buffer.to_vec();
+ $recursive_hash_method(&columns, $hashes_buffer)?;
+ for (i, hash) in $hashes_buffer.iter_mut().enumerate() {
+ if nulls.is_null(i) {
+ *hash = before[i];
+ }
+ }
+ }
+ None => $recursive_hash_method(&columns, $hashes_buffer)?,
+ }
}
}
DataType::Map(field, _) => {
diff --git a/datafusion/spark/src/function/hash/xxhash64.rs b/datafusion/spark/src/function/hash/xxhash64.rs
index 9d02a51b22..d1d62d6e4a 100644
--- a/datafusion/spark/src/function/hash/xxhash64.rs
+++ b/datafusion/spark/src/function/hash/xxhash64.rs
@@ -423,6 +423,34 @@ mod tests {
assert_ne!(hashes[1], hashes[2]);
}
+ #[test]
+ fn test_xxhash64_null_struct_ignores_fields() {
+ use arrow::array::StructArray;
+ use arrow::buffer::NullBuffer;
+ use arrow::datatypes::{Field, Fields};
+
+ // Row 1 is NULL but its fields still hold values.
+ let fields = Fields::from(vec![
+ Field::new("a", DataType::Int32, false),
+ Field::new("b", DataType::Utf8, false),
+ ]);
+ let struct_array = StructArray::new(
+ fields,
+ vec![
+ Arc::new(Int32Array::from(vec![1, 1])) as ArrayRef,
+ Arc::new(StringArray::from(vec!["x", "x"])) as ArrayRef,
+ ],
+ Some(NullBuffer::from(vec![true, false])),
+ );
+ let array_ref: ArrayRef = Arc::new(struct_array);
+
+ let mut hashes = vec![DEFAULT_SEED; 2];
+ create_xxhash64_hashes(&[array_ref], &mut hashes).unwrap();
+
+ assert_ne!(hashes[0], DEFAULT_SEED);
+ assert_eq!(hashes[1], DEFAULT_SEED);
+ }
+
#[test]
fn test_xxhash64_list() {
use arrow::array::ListArray;
diff --git a/datafusion/sqllogictest/test_files/spark/hash/xxhash64.slt b/datafusion/sqllogictest/test_files/spark/hash/xxhash64.slt
index cbc14967c4..c4b5ad0d59 100644
--- a/datafusion/sqllogictest/test_files/spark/hash/xxhash64.slt
+++ b/datafusion/sqllogictest/test_files/spark/hash/xxhash64.slt
@@ -195,3 +195,19 @@ SELECT id, xxhash64(id, name) FROM xxhash_test ORDER BY id;
statement ok
DROP TABLE xxhash_test;
+
+# A NULL struct hashes like any other NULL, whatever its fields still hold
+query BII
+SELECT s IS NULL,
+ xxhash64(s),
+ xxhash64(arrow_cast(NULL, 'Struct("a": Int64, "b": Utf8)'))
+FROM (SELECT nullif(s, s) AS s FROM (SELECT named_struct('a', 1, 'b', 'x') AS s));
+----
+true 42 42
+
+# A NULL struct in a later argument leaves the incoming hash unchanged
+query II
+SELECT xxhash64(7, nullif(s, s)), xxhash64(7)
+FROM (SELECT named_struct('a', 1, 'b', 'x') AS s);
+----
+-2864211556195259663 -2864211556195259663