Apache DataFusion SQL Query Engine
feat: add Spark-compatible xxhash64 function (#21967)
Andy Grove6b134ddaf565
diff --git a/Cargo.lock b/Cargo.lockindex 79c6054726..b378253455 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2550,6 +2550,7 @@ dependencies = ["serde_json","sha1 0.11.0","sha2",+ "twox-hash","url",]@@ -6446,6 +6447,9 @@ name = "twox-hash"version = "2.1.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"+dependencies = [+ "rand 0.9.4",+][[package]]name = "typenum"diff --git a/datafusion/spark/Cargo.toml b/datafusion/spark/Cargo.tomlindex b03a732107..14f9396d76 100644--- a/datafusion/spark/Cargo.toml+++ b/datafusion/spark/Cargo.toml@@ -64,6 +64,7 @@ rand = { workspace = true }serde_json = { workspace = true }sha1 = "0.11"sha2 = { workspace = true }+twox-hash = "2.1"url = { workspace = true }[dev-dependencies]diff --git a/datafusion/spark/src/function/hash/mod.rs b/datafusion/spark/src/function/hash/mod.rsindex 5860596ac7..351f5d2d50 100644--- a/datafusion/spark/src/function/hash/mod.rs+++ b/datafusion/spark/src/function/hash/mod.rs@@ -18,6 +18,8 @@pub mod crc32;pub mod sha1;pub mod sha2;+pub(crate) mod utils;+pub mod xxhash64;use datafusion_expr::ScalarUDF;use datafusion_functions::make_udf_function;@@ -26,16 +28,18 @@ use std::sync::Arc;make_udf_function!(crc32::SparkCrc32, crc32);make_udf_function!(sha1::SparkSha1, sha1);make_udf_function!(sha2::SparkSha2, sha2);+make_udf_function!(xxhash64::SparkXxhash64, xxhash64);pub mod expr_fn {use datafusion_functions::export_functions;export_functions!((crc32, "crc32(expr) - Returns a cyclic redundancy check value of the expr as a bigint.", arg1),(sha1, "sha1(expr) - Returns a SHA-1 hash value of the expr as a hex string.", arg1),- (sha2, "sha2(expr, bitLength) - Returns a checksum of SHA-2 family as a hex string of expr. SHA-224, SHA-256, SHA-384, and SHA-512 are supported. Bit length of 0 is equivalent to 256.", arg1 arg2)+ (sha2, "sha2(expr, bitLength) - Returns a checksum of SHA-2 family as a hex string of expr. SHA-224, SHA-256, SHA-384, and SHA-512 are supported. Bit length of 0 is equivalent to 256.", arg1 arg2),+ (xxhash64, "xxhash64(expr1, expr2, ...) - Returns a 64-bit hash value of the arguments using xxHash.", args));}pub fn functions() -> Vec<Arc<ScalarUDF>> {- vec![crc32(), sha1(), sha2()]+ vec![crc32(), sha1(), sha2(), xxhash64()]}diff --git a/datafusion/spark/src/function/hash/utils.rs b/datafusion/spark/src/function/hash/utils.rsnew file mode 100644index 0000000000..e7918d33ec--- /dev/null+++ b/datafusion/spark/src/function/hash/utils.rs@@ -0,0 +1,1005 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Shared helpers for Spark-compatible hash functions.+//!+//! Generic helpers ([`hash_primitive_values`], [`hash_bytes_values`]) cover+//! the cases where a function-based abstraction is clean; less uniform paths+//! (booleans, decimals, list/map dispatch) remain as macros.+//!+//! Ported from Apache DataFusion Comet:+//! <https://github.com/apache/datafusion-comet/blob/main/native/spark-expr/src/hash_funcs/utils.rs>++use arrow::array::{Array, PrimitiveArray};+use arrow::datatypes::ArrowPrimitiveType;+use datafusion_common::Result;++/// Hash the values of a primitive array, calling `transform` to convert each+/// value into a byte slice before feeding it to `hash_method`.+#[inline]+pub(crate) fn hash_primitive_values<P, F, B, H>(+ array: &PrimitiveArray<P>,+ hashes: &mut [u64],+ transform: F,+ hash_method: H,+) where+ P: ArrowPrimitiveType,+ F: Fn(P::Native) -> B,+ B: AsRef<[u8]>,+ H: Fn(B, u64) -> u64,+{+ let values = array.values();+ if array.null_count() == 0 {+ // Fast path: no nulls, skip null checks+ for (i, hash) in hashes.iter_mut().enumerate() {+ *hash = hash_method(transform(values[i]), *hash);+ }+ } else {+ // Slow path: check each row for null+ for (i, hash) in hashes.iter_mut().enumerate() {+ if !array.is_null(i) {+ *hash = hash_method(transform(values[i]), *hash);+ }+ }+ }+}++/// Hash an array by calling `value_at(array, i)` to produce a byte slice for+/// each non-null row. Used for byte-slice arrays (Utf8/Binary/FixedSizeBinary+/// and their Large/View variants) and for arrays whose values need a small+/// transform (e.g. booleans widened to `i32`, decimals to `i128` bytes).+#[inline]+pub(crate) fn hash_bytes_values<A, B, H>(+ array: &A,+ hashes: &mut [u64],+ value_at: impl Fn(&A, usize) -> B,+ hash_method: H,+) where+ A: Array,+ B: AsRef<[u8]>,+ H: Fn(B, u64) -> u64,+{+ if array.null_count() == 0 {+ for (i, hash) in hashes.iter_mut().enumerate() {+ *hash = hash_method(value_at(array, i), *hash);+ }+ } else {+ for (i, hash) in hashes.iter_mut().enumerate() {+ if !array.is_null(i) {+ *hash = hash_method(value_at(array, i), *hash);+ }+ }+ }+}++/// Fallible variant of [`hash_bytes_values`]: `value_at` may return an error+/// (used by the small-decimal path, where values may not fit in `i64`).+#[inline]+pub(crate) fn try_hash_bytes_values<A, B, H>(+ array: &A,+ hashes: &mut [u64],+ value_at: impl Fn(&A, usize) -> Result<B>,+ hash_method: H,+) -> Result<()>+where+ A: Array,+ B: AsRef<[u8]>,+ H: Fn(B, u64) -> u64,+{+ if array.null_count() == 0 {+ for (i, hash) in hashes.iter_mut().enumerate() {+ *hash = hash_method(value_at(array, i)?, *hash);+ }+ } else {+ for (i, hash) in hashes.iter_mut().enumerate() {+ if !array.is_null(i) {+ *hash = hash_method(value_at(array, i)?, *hash);+ }+ }+ }+ Ok(())+}++/// Downcast `$column` to `$array_type`, panicking with a useful error if the+/// type does not match.+#[macro_export]+#[doc(hidden)]+macro_rules! downcast_array {+ ($column:ident, $array_type:ident) => {+ $column+ .as_any()+ .downcast_ref::<$array_type>()+ .unwrap_or_else(|| {+ panic!(+ "Failed to downcast column to {}. Actual data type: {:?}.",+ stringify!($array_type),+ $column.data_type()+ )+ })+ };+}++/// Hash a byte-slice-accessor array (strings, binary, fixed-size binary, ...)+/// where `value(i)` returns the item directly.+///+/// Kept as a macro because `a.value(i)` returns a borrow whose lifetime+/// depends on the array, which Rust's closure inference does not propagate+/// through a `Fn(&A, usize) -> B` bound with a free type parameter `B`.+#[macro_export]+#[doc(hidden)]+macro_rules! hash_array {+ ($array_type:ident, $column:ident, $hashes:ident, $hash_method:ident) => {+ let array = $crate::downcast_array!($column, $array_type);+ if array.null_count() == 0 {+ for i in 0..$hashes.len() {+ $hashes[i] = $hash_method(&array.value(i), $hashes[i]);+ }+ } else {+ for i in 0..$hashes.len() {+ if !array.is_null(i) {+ $hashes[i] = $hash_method(&array.value(i), $hashes[i]);+ }+ }+ }+ };+}++/// Hash a `BooleanArray`, widening each value to `$hash_input_type` first.+#[macro_export]+#[doc(hidden)]+macro_rules! hash_array_boolean {+ ($array_type:ident, $column:ident, $hash_input_type:ident, $hashes:ident, $hash_method:ident) => {+ let array = $crate::downcast_array!($column, $array_type);+ $crate::function::hash::utils::hash_bytes_values(+ array,+ $hashes,+ |a, i| $hash_input_type::from(a.value(i)).to_le_bytes(),+ $hash_method,+ );+ };+}++/// Hash a primitive array, widening each native value to `$ty` before hashing.+#[macro_export]+#[doc(hidden)]+macro_rules! hash_array_primitive {+ ($array_type:ident, $column:ident, $ty:ident, $hashes:ident, $hash_method:ident) => {+ let array = $crate::downcast_array!($column, $array_type);+ $crate::function::hash::utils::hash_primitive_values(+ array,+ $hashes,+ |v| (v as $ty).to_le_bytes(),+ $hash_method,+ );+ };+}++/// Hash a floating-point primitive array, normalizing `-0.0` to `0.0` per Spark.+#[macro_export]+#[doc(hidden)]+macro_rules! hash_array_primitive_float {+ ($array_type:ident, $column:ident, $ty:ident, $ty2:ident, $hashes:ident, $hash_method:ident) => {+ let array = $crate::downcast_array!($column, $array_type);+ $crate::function::hash::utils::hash_primitive_values(+ array,+ $hashes,+ |v| {+ // Spark uses 0 as hash for -0.0, see `Murmur3Hash` expression.+ if v == 0.0 && v.is_sign_negative() {+ (0 as $ty2).to_le_bytes()+ } else {+ (v as $ty).to_le_bytes()+ }+ },+ $hash_method,+ );+ };+}++/// Hash a small (precision <= 18) decimal array by reducing each value to `i64`.+/// Errors via `?` if a value does not fit in `i64`.+#[macro_export]+#[doc(hidden)]+macro_rules! hash_array_small_decimal {+ ($array_type:ident, $column:ident, $hashes:ident, $hash_method:ident) => {+ let array = $crate::downcast_array!($column, $array_type);+ $crate::function::hash::utils::try_hash_bytes_values(+ array,+ $hashes,+ |a, i| {+ i64::try_from(a.value(i))+ .map(|v| v.to_le_bytes())+ .map_err(|e| DataFusionError::Execution(e.to_string()))+ },+ $hash_method,+ )?;+ };+}++/// Hash a wide decimal array using the raw 128-bit byte representation.+#[macro_export]+#[doc(hidden)]+macro_rules! hash_array_decimal {+ ($array_type:ident, $column:ident, $hashes:ident, $hash_method:ident) => {+ let array = $crate::downcast_array!($column, $array_type);+ $crate::function::hash::utils::hash_bytes_values(+ array,+ $hashes,+ |a, i| a.value(i).to_le_bytes(),+ $hash_method,+ );+ };+}++/// Hash a list array with primitive elements by directly accessing the underlying buffer.+/// This avoids the overhead of slicing and recursive calls for common cases.+/// Supports both variable-length lists (with offsets) and fixed-size lists.+#[macro_export]+#[doc(hidden)]+macro_rules! hash_list_primitive {+ // Variable-length list variant (List/LargeList)+ (offsets: $offsets:expr, $list_array:ident, $elem_array:ident, $hashes:ident, $hash_method:ident, $value_transform:expr) => {+ if $list_array.null_count() == 0 && $elem_array.null_count() == 0 {+ for (row_idx, hash) in $hashes.iter_mut().enumerate() {+ let start = $offsets[row_idx] as usize;+ let end = $offsets[row_idx + 1] as usize;+ for elem_idx in start..end {+ let value = $elem_array.value(elem_idx);+ *hash = $hash_method($value_transform(value), *hash);+ }+ }+ } else {+ for (row_idx, hash) in $hashes.iter_mut().enumerate() {+ if !$list_array.is_null(row_idx) {+ let start = $offsets[row_idx] as usize;+ let end = $offsets[row_idx + 1] as usize;+ for elem_idx in start..end {+ if !$elem_array.is_null(elem_idx) {+ let value = $elem_array.value(elem_idx);+ *hash = $hash_method($value_transform(value), *hash);+ }+ }+ }+ }+ }+ };+ // Fixed-size list variant+ (fixed_size: $list_size:expr, $list_array:ident, $elem_array:ident, $hashes:ident, $hash_method:ident, $value_transform:expr) => {+ if $list_array.null_count() == 0 && $elem_array.null_count() == 0 {+ for (row_idx, hash) in $hashes.iter_mut().enumerate() {+ let start = row_idx * $list_size;+ for elem_idx in 0..$list_size {+ let value = $elem_array.value(start + elem_idx);+ *hash = $hash_method($value_transform(value), *hash);+ }+ }+ } else {+ for (row_idx, hash) in $hashes.iter_mut().enumerate() {+ if !$list_array.is_null(row_idx) {+ let start = row_idx * $list_size;+ for elem_idx in 0..$list_size {+ if !$elem_array.is_null(start + elem_idx) {+ let value = $elem_array.value(start + elem_idx);+ *hash = $hash_method($value_transform(value), *hash);+ }+ }+ }+ }+ }+ };+}++/// Hash a list array by recursively hashing each element.+/// For each row, we hash all elements in the list.+/// Spark hashes arrays by recursively hashing each element, where each+/// element's hash is computed using the previous element's hash as the seed.+/// This creates a chain: hash(elem_n, hash(elem_n-1, ... hash(elem_0, seed)...))+/// Dispatches hash operations for List/LargeList/FixedSizeList arrays with primitive element types.+/// This macro eliminates duplication by handling the type-to-array mapping for all supported primitives.+#[macro_export]+#[doc(hidden)]+macro_rules! hash_list_with_primitive_elements {+ // Variant for List/LargeList with offsets+ (offsets: $list_array_type:ident, $list_array:ident, $values:ident, $offsets:ident, $field:expr, $hashes_buffer:ident, $hash_method:ident, $recursive_hash_method:ident, $fallback_offset_type:ty, $col:ident) => {+ match $field.data_type() {+ DataType::Int8 => {+ let elem_array = $values.as_any().downcast_ref::<Int8Array>().unwrap();+ $crate::hash_list_primitive!(offsets: $offsets, $list_array, elem_array, $hashes_buffer, $hash_method, |v: i8| (v as i32).to_le_bytes());+ }+ DataType::Int16 => {+ let elem_array = $values.as_any().downcast_ref::<Int16Array>().unwrap();+ $crate::hash_list_primitive!(offsets: $offsets, $list_array, elem_array, $hashes_buffer, $hash_method, |v: i16| (v as i32).to_le_bytes());+ }+ DataType::Int32 => {+ let elem_array = $values.as_any().downcast_ref::<Int32Array>().unwrap();+ $crate::hash_list_primitive!(offsets: $offsets, $list_array, elem_array, $hashes_buffer, $hash_method, |v: i32| v.to_le_bytes());+ }+ DataType::Int64 => {+ let elem_array = $values.as_any().downcast_ref::<Int64Array>().unwrap();+ $crate::hash_list_primitive!(offsets: $offsets, $list_array, elem_array, $hashes_buffer, $hash_method, |v: i64| v.to_le_bytes());+ }+ DataType::Float32 => {+ let elem_array = $values.as_any().downcast_ref::<Float32Array>().unwrap();+ $crate::hash_list_primitive!(offsets: $offsets, $list_array, elem_array, $hashes_buffer, $hash_method,+ |v: f32| if v == 0.0 && v.is_sign_negative() { (0_i32).to_le_bytes() } else { v.to_le_bytes() });+ }+ DataType::Float64 => {+ let elem_array = $values.as_any().downcast_ref::<Float64Array>().unwrap();+ $crate::hash_list_primitive!(offsets: $offsets, $list_array, elem_array, $hashes_buffer, $hash_method,+ |v: f64| if v == 0.0 && v.is_sign_negative() { (0_i64).to_le_bytes() } else { v.to_le_bytes() });+ }+ DataType::Boolean => {+ let elem_array = $values.as_any().downcast_ref::<BooleanArray>().unwrap();+ $crate::hash_list_primitive!(offsets: $offsets, $list_array, elem_array, $hashes_buffer, $hash_method, |v: bool| (i32::from(v)).to_le_bytes());+ }+ DataType::Utf8 => {+ let elem_array = $values.as_any().downcast_ref::<StringArray>().unwrap();+ if $list_array.null_count() == 0 && elem_array.null_count() == 0 {+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {+ let start = $offsets[row_idx] as usize;+ let end = $offsets[row_idx + 1] as usize;+ for elem_idx in start..end {+ *hash = $hash_method(elem_array.value(elem_idx), *hash);+ }+ }+ } else {+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {+ if !$list_array.is_null(row_idx) {+ let start = $offsets[row_idx] as usize;+ let end = $offsets[row_idx + 1] as usize;+ for elem_idx in start..end {+ if !elem_array.is_null(elem_idx) {+ *hash = $hash_method(elem_array.value(elem_idx), *hash);+ }+ }+ }+ }+ }+ }+ DataType::Binary => {+ let elem_array = $values.as_any().downcast_ref::<BinaryArray>().unwrap();+ if $list_array.null_count() == 0 && elem_array.null_count() == 0 {+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {+ let start = $offsets[row_idx] as usize;+ let end = $offsets[row_idx + 1] as usize;+ for elem_idx in start..end {+ *hash = $hash_method(elem_array.value(elem_idx), *hash);+ }+ }+ } else {+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {+ if !$list_array.is_null(row_idx) {+ let start = $offsets[row_idx] as usize;+ let end = $offsets[row_idx + 1] as usize;+ for elem_idx in start..end {+ if !elem_array.is_null(elem_idx) {+ *hash = $hash_method(elem_array.value(elem_idx), *hash);+ }+ }+ }+ }+ }+ }+ DataType::Date32 => {+ let elem_array = $values.as_any().downcast_ref::<Date32Array>().unwrap();+ $crate::hash_list_primitive!(offsets: $offsets, $list_array, elem_array, $hashes_buffer, $hash_method, |v: i32| v.to_le_bytes());+ }+ DataType::Timestamp(TimeUnit::Microsecond, _) => {+ let elem_array = $values.as_any().downcast_ref::<TimestampMicrosecondArray>().unwrap();+ $crate::hash_list_primitive!(offsets: $offsets, $list_array, elem_array, $hashes_buffer, $hash_method, |v: i64| v.to_le_bytes());+ }+ _ => {+ // Fall back to recursive approach for complex element types+ $crate::hash_list_array!($list_array_type, $fallback_offset_type, $col, $hashes_buffer, $recursive_hash_method);+ }+ }+ };+ // Variant for FixedSizeList with fixed size+ (fixed_size: $list_array:ident, $values:ident, $list_size:ident, $field:expr, $hashes_buffer:ident, $hash_method:ident, $recursive_hash_method:ident) => {+ match $field.data_type() {+ DataType::Int8 => {+ let elem_array = $values.as_any().downcast_ref::<Int8Array>().unwrap();+ $crate::hash_list_primitive!(fixed_size: $list_size, $list_array, elem_array, $hashes_buffer, $hash_method, |v: i8| (v as i32).to_le_bytes());+ }+ DataType::Int16 => {+ let elem_array = $values.as_any().downcast_ref::<Int16Array>().unwrap();+ $crate::hash_list_primitive!(fixed_size: $list_size, $list_array, elem_array, $hashes_buffer, $hash_method, |v: i16| (v as i32).to_le_bytes());+ }+ DataType::Int32 => {+ let elem_array = $values.as_any().downcast_ref::<Int32Array>().unwrap();+ $crate::hash_list_primitive!(fixed_size: $list_size, $list_array, elem_array, $hashes_buffer, $hash_method, |v: i32| v.to_le_bytes());+ }+ DataType::Int64 => {+ let elem_array = $values.as_any().downcast_ref::<Int64Array>().unwrap();+ $crate::hash_list_primitive!(fixed_size: $list_size, $list_array, elem_array, $hashes_buffer, $hash_method, |v: i64| v.to_le_bytes());+ }+ DataType::Float32 => {+ let elem_array = $values.as_any().downcast_ref::<Float32Array>().unwrap();+ $crate::hash_list_primitive!(fixed_size: $list_size, $list_array, elem_array, $hashes_buffer, $hash_method,+ |v: f32| if v == 0.0 && v.is_sign_negative() { (0_i32).to_le_bytes() } else { v.to_le_bytes() });+ }+ DataType::Float64 => {+ let elem_array = $values.as_any().downcast_ref::<Float64Array>().unwrap();+ $crate::hash_list_primitive!(fixed_size: $list_size, $list_array, elem_array, $hashes_buffer, $hash_method,+ |v: f64| if v == 0.0 && v.is_sign_negative() { (0_i64).to_le_bytes() } else { v.to_le_bytes() });+ }+ DataType::Boolean => {+ let elem_array = $values.as_any().downcast_ref::<BooleanArray>().unwrap();+ $crate::hash_list_primitive!(fixed_size: $list_size, $list_array, elem_array, $hashes_buffer, $hash_method, |v: bool| (i32::from(v)).to_le_bytes());+ }+ DataType::Utf8 => {+ let elem_array = $values.as_any().downcast_ref::<StringArray>().unwrap();+ if $list_array.null_count() == 0 && elem_array.null_count() == 0 {+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {+ let start = row_idx * $list_size;+ for elem_idx in 0..$list_size {+ *hash = $hash_method(elem_array.value(start + elem_idx), *hash);+ }+ }+ } else {+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {+ if !$list_array.is_null(row_idx) {+ let start = row_idx * $list_size;+ for elem_idx in 0..$list_size {+ if !elem_array.is_null(start + elem_idx) {+ *hash = $hash_method(elem_array.value(start + elem_idx), *hash);+ }+ }+ }+ }+ }+ }+ DataType::Binary => {+ let elem_array = $values.as_any().downcast_ref::<BinaryArray>().unwrap();+ if $list_array.null_count() == 0 && elem_array.null_count() == 0 {+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {+ let start = row_idx * $list_size;+ for elem_idx in 0..$list_size {+ *hash = $hash_method(elem_array.value(start + elem_idx), *hash);+ }+ }+ } else {+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {+ if !$list_array.is_null(row_idx) {+ let start = row_idx * $list_size;+ for elem_idx in 0..$list_size {+ if !elem_array.is_null(start + elem_idx) {+ *hash = $hash_method(elem_array.value(start + elem_idx), *hash);+ }+ }+ }+ }+ }+ }+ DataType::Date32 => {+ let elem_array = $values.as_any().downcast_ref::<Date32Array>().unwrap();+ $crate::hash_list_primitive!(fixed_size: $list_size, $list_array, elem_array, $hashes_buffer, $hash_method, |v: i32| v.to_le_bytes());+ }+ DataType::Timestamp(TimeUnit::Microsecond, _) => {+ let elem_array = $values.as_any().downcast_ref::<TimestampMicrosecondArray>().unwrap();+ $crate::hash_list_primitive!(fixed_size: $list_size, $list_array, elem_array, $hashes_buffer, $hash_method, |v: i64| v.to_le_bytes());+ }+ _ => {+ // Fall back to recursive approach for complex element types+ if $list_array.null_count() == 0 {+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {+ let start = row_idx * $list_size;+ for elem_idx in 0..$list_size {+ let elem_array = $values.slice(start + elem_idx, 1);+ let mut single_hash = [*hash];+ $recursive_hash_method(&[elem_array], &mut single_hash)?;+ *hash = single_hash[0];+ }+ }+ } else {+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {+ if !$list_array.is_null(row_idx) {+ let start = row_idx * $list_size;+ for elem_idx in 0..$list_size {+ let elem_array = $values.slice(start + elem_idx, 1);+ let mut single_hash = [*hash];+ $recursive_hash_method(&[elem_array], &mut single_hash)?;+ *hash = single_hash[0];+ }+ }+ }+ }+ }+ }+ };+}++/// Hash a map array by hashing its key/value entries in order.+///+/// Specializes for common key/value type combinations so the hot path stays+/// monomorphic; falls back to recursive hashing for arbitrary nested types.+#[macro_export]+#[doc(hidden)]+macro_rules! hash_map_array {+ // Specialized variant: typed key and value arrays, byte transforms.+ ($map_array:ident, $key_array:ident, $value_array:ident, $offsets:ident, $hashes_buffer:ident, $hash_method:ident, $key_transform:expr, $value_transform:expr) => {+ if $map_array.null_count() == 0+ && $key_array.null_count() == 0+ && $value_array.null_count() == 0+ {+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {+ let start = $offsets[row_idx] as usize;+ let end = $offsets[row_idx + 1] as usize;+ for entry_idx in start..end {+ *hash =+ $hash_method($key_transform($key_array.value(entry_idx)), *hash);+ *hash = $hash_method(+ $value_transform($value_array.value(entry_idx)),+ *hash,+ );+ }+ }+ } else {+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {+ if !$map_array.is_null(row_idx) {+ let start = $offsets[row_idx] as usize;+ let end = $offsets[row_idx + 1] as usize;+ for entry_idx in start..end {+ if !$key_array.is_null(entry_idx) {+ *hash = $hash_method(+ $key_transform($key_array.value(entry_idx)),+ *hash,+ );+ }+ if !$value_array.is_null(entry_idx) {+ *hash = $hash_method(+ $value_transform($value_array.value(entry_idx)),+ *hash,+ );+ }+ }+ }+ }+ }+ };+ // Fallback variant: recursively hash each key/value entry.+ (recursive: $map_array:ident, $keys:ident, $values:ident, $offsets:ident, $hashes_buffer:ident, $recursive_hash_method:ident) => {+ if $map_array.null_count() == 0 {+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {+ let start = $offsets[row_idx] as usize;+ let end = $offsets[row_idx + 1] as usize;+ for entry_idx in start..end {+ let key_array = $keys.slice(entry_idx, 1);+ let mut single_hash = [*hash];+ $recursive_hash_method(&[key_array], &mut single_hash)?;+ *hash = single_hash[0];++ let value_array = $values.slice(entry_idx, 1);+ single_hash = [*hash];+ $recursive_hash_method(&[value_array], &mut single_hash)?;+ *hash = single_hash[0];+ }+ }+ } else {+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {+ if !$map_array.is_null(row_idx) {+ let start = $offsets[row_idx] as usize;+ let end = $offsets[row_idx + 1] as usize;+ for entry_idx in start..end {+ let key_array = $keys.slice(entry_idx, 1);+ let mut single_hash = [*hash];+ $recursive_hash_method(&[key_array], &mut single_hash)?;+ *hash = single_hash[0];++ let value_array = $values.slice(entry_idx, 1);+ single_hash = [*hash];+ $recursive_hash_method(&[value_array], &mut single_hash)?;+ *hash = single_hash[0];+ }+ }+ }+ }+ };+}++/// Dispatch over a `MapArray`'s key/value types, calling [`hash_map_array!`]+/// with the right specialization (or the recursive fallback).+#[macro_export]+#[doc(hidden)]+macro_rules! hash_map_with_typed_entries {+ ($col:ident, $field:expr, $hashes_buffer:ident, $hash_method:ident, $recursive_hash_method:ident) => {+ let map_array = $col.as_any().downcast_ref::<MapArray>().unwrap();+ let keys = map_array.keys();+ let values = map_array.values();+ let offsets = map_array.offsets();++ if let DataType::Struct(fields) = $field.data_type() {+ let key_type = &fields[0].data_type();+ let value_type = &fields[1].data_type();++ match (key_type, value_type) {+ (DataType::Utf8, DataType::Int32) => {+ let key_array = keys.as_any().downcast_ref::<StringArray>().unwrap();+ let value_array = values.as_any().downcast_ref::<Int32Array>().unwrap();+ $crate::hash_map_array!(+ map_array, key_array, value_array, offsets, $hashes_buffer,+ $hash_method, |v| v, |v: i32| v.to_le_bytes()+ );+ }+ (DataType::Int32, DataType::Utf8) => {+ let key_array = keys.as_any().downcast_ref::<Int32Array>().unwrap();+ let value_array = values.as_any().downcast_ref::<StringArray>().unwrap();+ $crate::hash_map_array!(+ map_array, key_array, value_array, offsets, $hashes_buffer,+ $hash_method, |v: i32| v.to_le_bytes(), |v| v+ );+ }+ (DataType::Utf8, DataType::Utf8) => {+ let key_array = keys.as_any().downcast_ref::<StringArray>().unwrap();+ let value_array = values.as_any().downcast_ref::<StringArray>().unwrap();+ $crate::hash_map_array!(+ map_array, key_array, value_array, offsets, $hashes_buffer,+ $hash_method, |v| v, |v| v+ );+ }+ (DataType::Int32, DataType::Int32) => {+ let key_array = keys.as_any().downcast_ref::<Int32Array>().unwrap();+ let value_array = values.as_any().downcast_ref::<Int32Array>().unwrap();+ $crate::hash_map_array!(+ map_array, key_array, value_array, offsets, $hashes_buffer,+ $hash_method, |v: i32| v.to_le_bytes(), |v: i32| v.to_le_bytes()+ );+ }+ _ => {+ $crate::hash_map_array!(+ recursive: map_array, keys, values, offsets,+ $hashes_buffer, $recursive_hash_method+ );+ }+ }+ } else {+ return Err(DataFusionError::Internal(format!(+ "Map field type must be a struct, got: {}",+ $field.data_type()+ )));+ }+ };+}++#[macro_export]+#[doc(hidden)]+macro_rules! hash_list_array {+ ($array_type:ident, $offset_type:ty, $column: ident, $hashes: ident, $recursive_hash_method: ident) => {+ let list_array = $column+ .as_any()+ .downcast_ref::<$array_type>()+ .unwrap_or_else(|| {+ panic!(+ "Failed to downcast column to {}. Actual data type: {:?}.",+ stringify!($array_type),+ $column.data_type()+ )+ });++ let values = list_array.values();+ let offsets = list_array.offsets();++ if list_array.null_count() == 0 {+ // Fast path: no nulls, skip null checks+ for (row_idx, hash) in $hashes.iter_mut().enumerate() {+ let start = offsets[row_idx] as usize;+ let end = offsets[row_idx + 1] as usize;+ let len = end - start;+ // Hash each element in sequence, chaining the hash values+ for elem_idx in 0..len {+ let elem_array = values.slice(start + elem_idx, 1);+ let mut single_hash = [*hash];+ $recursive_hash_method(&[elem_array], &mut single_hash)?;+ *hash = single_hash[0];+ }+ }+ } else {+ // Slow path: array has nulls, check each row+ for (row_idx, hash) in $hashes.iter_mut().enumerate() {+ if !list_array.is_null(row_idx) {+ let start = offsets[row_idx] as usize;+ let end = offsets[row_idx + 1] as usize;+ let len = end - start;+ // Hash each element in sequence, chaining the hash values+ for elem_idx in 0..len {+ let elem_array = values.slice(start + elem_idx, 1);+ let mut single_hash = [*hash];+ $recursive_hash_method(&[elem_array], &mut single_hash)?;+ *hash = single_hash[0];+ }+ }+ }+ }+ };+}++/// Creates hash values for every row, based on the values in the+/// columns.+///+/// The number of rows to hash is determined by `hashes_buffer.len()`.+/// `hashes_buffer` should be pre-sized appropriately+///+/// `hash_method` is the hash function to use.+/// `create_dictionary_hash_method` is the function to create hashes for dictionary arrays input.+/// `recursive_hash_method` is the function to call for recursive hashing of complex types.+#[macro_export]+#[doc(hidden)]+macro_rules! create_hashes_internal {+ ($arrays: ident, $hashes_buffer: ident, $hash_method: ident, $create_dictionary_hash_method: ident, $recursive_hash_method: ident) => {+ use arrow::array::{types::*, *};+ use arrow::datatypes::{DataType, TimeUnit};+ use datafusion_common::DataFusionError;++ for (i, col) in $arrays.iter().enumerate() {+ let first_col = i == 0;+ match col.data_type() {+ DataType::Boolean => {+ $crate::hash_array_boolean!(+ BooleanArray,+ col,+ i32,+ $hashes_buffer,+ $hash_method+ );+ }+ DataType::Int8 => {+ $crate::hash_array_primitive!(+ Int8Array,+ col,+ i32,+ $hashes_buffer,+ $hash_method+ );+ }+ DataType::Int16 => {+ $crate::hash_array_primitive!(+ Int16Array,+ col,+ i32,+ $hashes_buffer,+ $hash_method+ );+ }+ DataType::Int32 => {+ $crate::hash_array_primitive!(+ Int32Array,+ col,+ i32,+ $hashes_buffer,+ $hash_method+ );+ }+ DataType::Int64 => {+ $crate::hash_array_primitive!(+ Int64Array,+ col,+ i64,+ $hashes_buffer,+ $hash_method+ );+ }+ DataType::Float32 => {+ $crate::hash_array_primitive_float!(+ Float32Array,+ col,+ f32,+ i32,+ $hashes_buffer,+ $hash_method+ );+ }+ DataType::Float64 => {+ $crate::hash_array_primitive_float!(+ Float64Array,+ col,+ f64,+ i64,+ $hashes_buffer,+ $hash_method+ );+ }+ DataType::Timestamp(TimeUnit::Second, _) => {+ $crate::hash_array_primitive!(+ TimestampSecondArray,+ col,+ i64,+ $hashes_buffer,+ $hash_method+ );+ }+ DataType::Timestamp(TimeUnit::Millisecond, _) => {+ $crate::hash_array_primitive!(+ TimestampMillisecondArray,+ col,+ i64,+ $hashes_buffer,+ $hash_method+ );+ }+ DataType::Timestamp(TimeUnit::Microsecond, _) => {+ $crate::hash_array_primitive!(+ TimestampMicrosecondArray,+ col,+ i64,+ $hashes_buffer,+ $hash_method+ );+ }+ DataType::Timestamp(TimeUnit::Nanosecond, _) => {+ $crate::hash_array_primitive!(+ TimestampNanosecondArray,+ col,+ i64,+ $hashes_buffer,+ $hash_method+ );+ }+ DataType::Date32 => {+ $crate::hash_array_primitive!(+ Date32Array,+ col,+ i32,+ $hashes_buffer,+ $hash_method+ );+ }+ DataType::Date64 => {+ $crate::hash_array_primitive!(+ Date64Array,+ col,+ i64,+ $hashes_buffer,+ $hash_method+ );+ }+ DataType::Utf8 => {+ $crate::hash_array!(StringArray, col, $hashes_buffer, $hash_method);+ }+ DataType::LargeUtf8 => {+ $crate::hash_array!(LargeStringArray, col, $hashes_buffer, $hash_method);+ }+ DataType::Utf8View => {+ $crate::hash_array!(StringViewArray, col, $hashes_buffer, $hash_method);+ }+ DataType::Binary => {+ $crate::hash_array!(BinaryArray, col, $hashes_buffer, $hash_method);+ }+ DataType::LargeBinary => {+ $crate::hash_array!(LargeBinaryArray, col, $hashes_buffer, $hash_method);+ }+ DataType::BinaryView => {+ $crate::hash_array!(BinaryViewArray, col, $hashes_buffer, $hash_method);+ }+ DataType::FixedSizeBinary(_) => {+ $crate::hash_array!(FixedSizeBinaryArray, col, $hashes_buffer, $hash_method);+ }+ DataType::Null => {+ // Nulls don't update the hash+ }+ // Apache Spark: if it's a small decimal, i.e. precision <= 18, turn it into long and hash it.+ // Else, turn it into bytes and hash it.+ DataType::Decimal128(precision, _) if *precision <= 18 => {+ $crate::hash_array_small_decimal!(Decimal128Array, col, $hashes_buffer, $hash_method);+ }+ DataType::Decimal128(_, _) => {+ $crate::hash_array_decimal!(Decimal128Array, col, $hashes_buffer, $hash_method);+ }+ DataType::Dictionary(index_type, _) => match **index_type {+ DataType::Int8 => {+ $create_dictionary_hash_method::<Int8Type>(col, $hashes_buffer, first_col)?;+ }+ DataType::Int16 => {+ $create_dictionary_hash_method::<Int16Type>(+ col,+ $hashes_buffer,+ first_col,+ )?;+ }+ DataType::Int32 => {+ $create_dictionary_hash_method::<Int32Type>(+ col,+ $hashes_buffer,+ first_col,+ )?;+ }+ DataType::Int64 => {+ $create_dictionary_hash_method::<Int64Type>(+ col,+ $hashes_buffer,+ first_col,+ )?;+ }+ DataType::UInt8 => {+ $create_dictionary_hash_method::<UInt8Type>(+ col,+ $hashes_buffer,+ first_col,+ )?;+ }+ DataType::UInt16 => {+ $create_dictionary_hash_method::<UInt16Type>(+ col,+ $hashes_buffer,+ first_col,+ )?;+ }+ DataType::UInt32 => {+ $create_dictionary_hash_method::<UInt32Type>(+ col,+ $hashes_buffer,+ first_col,+ )?;+ }+ DataType::UInt64 => {+ $create_dictionary_hash_method::<UInt64Type>(+ col,+ $hashes_buffer,+ first_col,+ )?;+ }+ _ => {+ return Err(DataFusionError::Internal(format!(+ "Unsupported dictionary type in hasher hashing: {}",+ col.data_type(),+ )))+ }+ },+ DataType::List(field) => {+ let list_array = col.as_any().downcast_ref::<ListArray>().unwrap();+ let values = list_array.values();+ let offsets = list_array.offsets();++ $crate::hash_list_with_primitive_elements!(offsets: ListArray, list_array, values, offsets, field, $hashes_buffer, $hash_method, $recursive_hash_method, i32, col);+ }+ DataType::LargeList(field) => {+ let list_array = col.as_any().downcast_ref::<LargeListArray>().unwrap();+ let values = list_array.values();+ let offsets = list_array.offsets();++ $crate::hash_list_with_primitive_elements!(offsets: LargeListArray, list_array, values, offsets, field, $hashes_buffer, $hash_method, $recursive_hash_method, i64, col);+ }+ DataType::FixedSizeList(field, size) => {+ let list_array = col.as_any().downcast_ref::<FixedSizeListArray>().unwrap();+ let values = list_array.values();+ let list_size = *size as usize;++ $crate::hash_list_with_primitive_elements!(fixed_size: list_array, values, list_size, field, $hashes_buffer, $hash_method, $recursive_hash_method);+ }+ DataType::Struct(_) => {+ let struct_array = col.as_any().downcast_ref::<StructArray>().unwrap();+ // Hash each field of the struct - Spark hashes all fields recursively+ let columns: Vec<ArrayRef> = struct_array.columns().to_vec();+ if !columns.is_empty() {+ $recursive_hash_method(&columns, $hashes_buffer)?;+ }+ }+ DataType::Map(field, _) => {+ $crate::hash_map_with_typed_entries!(+ col, field, $hashes_buffer, $hash_method, $recursive_hash_method+ );+ }+ _ => {+ // This is internal because we should have caught this before.+ return Err(DataFusionError::Internal(format!(+ "Unsupported data type in hasher: {}",+ col.data_type()+ )));+ }+ }+ }+ };+}diff --git a/datafusion/spark/src/function/hash/xxhash64.rs b/datafusion/spark/src/function/hash/xxhash64.rsnew file mode 100644index 0000000000..5dca47bcb8--- /dev/null+++ b/datafusion/spark/src/function/hash/xxhash64.rs@@ -0,0 +1,445 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use std::sync::Arc;++use arrow::array::{+ Array, ArrayRef, DictionaryArray, Int64Array, types::ArrowDictionaryKeyType,+};+use arrow::buffer::{Buffer, ScalarBuffer};+use arrow::compute::take;+use arrow::datatypes::{ArrowNativeType, DataType, Field, FieldRef};+use datafusion_common::{Result, ScalarValue, internal_err};+use datafusion_expr::{+ ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature,+ Volatility,+};+use twox_hash::XxHash64;++use crate::create_hashes_internal;++const DEFAULT_SEED: u64 = 42;++/// Spark-compatible xxhash64 function.+/// <https://spark.apache.org/docs/latest/api/sql/index.html#xxhash64>+#[derive(Debug, PartialEq, Eq, Hash)]+pub struct SparkXxhash64 {+ signature: Signature,+}++impl Default for SparkXxhash64 {+ fn default() -> Self {+ Self::new()+ }+}++impl SparkXxhash64 {+ pub fn new() -> Self {+ Self {+ signature: Signature::variadic_any(Volatility::Immutable),+ }+ }+}++impl ScalarUDFImpl for SparkXxhash64 {+ fn name(&self) -> &str {+ "xxhash64"+ }++ fn signature(&self) -> &Signature {+ &self.signature+ }++ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {+ internal_err!("return_field_from_args should be used instead")+ }++ fn return_field_from_args(&self, _args: ReturnFieldArgs) -> Result<FieldRef> {+ // Spark's HashExpression overrides nullable to false: NULL inputs are+ // skipped and the seed is used, so the result is never null.+ Ok(Arc::new(Field::new(self.name(), DataType::Int64, false)))+ }++ fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {+ let num_rows = args.number_rows;+ let mut hashes: Vec<u64> = vec![DEFAULT_SEED; num_rows];++ let arrays = ColumnarValue::values_to_arrays(&args.args)?;+ create_xxhash64_hashes(&arrays, &mut hashes)?;++ if num_rows == 1 {+ Ok(ColumnarValue::Scalar(ScalarValue::Int64(Some(+ hashes[0] as i64,+ ))))+ } else {+ // Reinterpret Vec<u64> as ScalarBuffer<i64> without copying — both+ // types have identical layout, and `as i64` is a bitcast.+ let buffer = ScalarBuffer::<i64>::from(Buffer::from_vec(hashes));+ Ok(ColumnarValue::Array(Arc::new(Int64Array::new(+ buffer, None,+ ))))+ }+ }+}++#[inline]+fn spark_compatible_xxhash64<T: AsRef<[u8]>>(data: T, seed: u64) -> u64 {+ XxHash64::oneshot(seed, data.as_ref())+}++/// Hash the values in a dictionary array using xxhash64.+fn create_xxhash64_hashes_dictionary<K: ArrowDictionaryKeyType>(+ array: &ArrayRef,+ hashes_buffer: &mut [u64],+ first_col: bool,+) -> Result<()> {+ let dict_array = array.as_any().downcast_ref::<DictionaryArray<K>>().unwrap();+ if !first_col {+ let unpacked = take(dict_array.values().as_ref(), dict_array.keys(), None)?;+ create_xxhash64_hashes(&[unpacked], hashes_buffer)?;+ } else {+ // Hash each dictionary value once, then look up by key. This avoids+ // redundant hashing of large dictionary entries (e.g. long strings).+ let dict_values = Arc::clone(dict_array.values());+ let mut dict_hashes = vec![DEFAULT_SEED; dict_values.len()];+ create_xxhash64_hashes(&[dict_values], &mut dict_hashes)?;++ for (hash, key) in hashes_buffer.iter_mut().zip(dict_array.keys().iter()) {+ if let Some(key) = key {+ *hash = dict_hashes[key.as_usize()]+ }+ // No update for Null keys, consistent with other types.+ }+ }+ Ok(())+}++/// Create xxhash64 hash values for every row, based on the values in the columns.+///+/// The number of rows to hash is determined by `hashes_buffer.len()`.+/// `hashes_buffer` should be pre-sized appropriately and seeded with the+/// initial hash value (Spark uses `42`).+fn create_xxhash64_hashes(arrays: &[ArrayRef], hashes_buffer: &mut [u64]) -> Result<()> {+ create_hashes_internal!(+ arrays,+ hashes_buffer,+ spark_compatible_xxhash64,+ create_xxhash64_hashes_dictionary,+ create_xxhash64_hashes+ );+ Ok(())+}++#[cfg(test)]+mod tests {+ use super::*;+ use arrow::array::{FixedSizeBinaryArray, Int32Array, StringArray};++ #[test]+ fn test_xxhash64_nullability() -> Result<()> {+ let func = SparkXxhash64::new();++ // Spark's xxhash64 is never null (NULL args are skipped, seed is returned),+ // so the output field is non-nullable regardless of input nullability.+ let nullable: FieldRef = Arc::new(Field::new("a", DataType::Int32, true));+ let non_nullable: FieldRef = Arc::new(Field::new("b", DataType::Int32, false));++ let out = func.return_field_from_args(ReturnFieldArgs {+ arg_fields: &[Arc::clone(&nullable), Arc::clone(&non_nullable)],+ scalar_arguments: &[None, None],+ })?;+ assert!(!out.is_nullable());+ assert_eq!(out.data_type(), &DataType::Int64);++ Ok(())+ }++ #[test]+ fn test_xxhash64_i32() {+ let seed = 42u64;+ assert_eq!(+ spark_compatible_xxhash64(1i32.to_le_bytes(), seed),+ 0xa309b38455455929+ );+ assert_eq!(+ spark_compatible_xxhash64(0i32.to_le_bytes(), seed),+ 0x3229fbc4681e48f3+ );+ assert_eq!(+ spark_compatible_xxhash64((-1i32).to_le_bytes(), seed),+ 0x1bfdda8861c06e45+ );+ }++ #[test]+ fn test_xxhash64_i32_boundary() {+ let seed = 42u64;+ let h = spark_compatible_xxhash64(i32::MAX.to_le_bytes(), seed);+ assert_ne!(h, seed);+ let h = spark_compatible_xxhash64(i32::MIN.to_le_bytes(), seed);+ assert_ne!(h, seed);+ }++ #[test]+ fn test_xxhash64_i8() {+ let seed = 42u64;+ // i8 is widened to i32 before hashing+ assert_eq!(+ spark_compatible_xxhash64((1i8 as i32).to_le_bytes(), seed),+ spark_compatible_xxhash64(1i32.to_le_bytes(), seed),+ );+ }++ #[test]+ fn test_xxhash64_i64() {+ let seed = 42u64;+ assert_eq!(+ spark_compatible_xxhash64(1i64.to_le_bytes(), seed),+ 0x9ed50fd59358d232+ );+ assert_eq!(+ spark_compatible_xxhash64(0i64.to_le_bytes(), seed),+ 0xb71b47ebda15746c+ );+ assert_eq!(+ spark_compatible_xxhash64((-1i64).to_le_bytes(), seed),+ 0x358ae035bfb46fd2+ );+ }++ #[test]+ fn test_xxhash64_i64_boundary() {+ let seed = 42u64;+ let h = spark_compatible_xxhash64(i64::MAX.to_le_bytes(), seed);+ assert_ne!(h, seed);+ let h = spark_compatible_xxhash64(i64::MIN.to_le_bytes(), seed);+ assert_ne!(h, seed);+ }++ /// Spark normalizes `-0.0` to `0.0` before hashing, so both produce+ /// the same hash. Exercise the dispatch through `create_xxhash64_hashes`+ /// to cover the `hash_array_primitive_float!` normalization path.+ #[test]+ fn test_xxhash64_negative_zero_f32() {+ use arrow::array::Float32Array;+ let array: ArrayRef = Arc::new(Float32Array::from(vec![0.0f32, -0.0f32]));+ let mut hashes = vec![DEFAULT_SEED; 2];+ create_xxhash64_hashes(&[array], &mut hashes).unwrap();+ assert_eq!(hashes[0], hashes[1]);+ assert_eq!(hashes[0], spark_compatible_xxhash64(0i32.to_le_bytes(), 42));+ }++ #[test]+ fn test_xxhash64_negative_zero_f64() {+ use arrow::array::Float64Array;+ let array: ArrayRef = Arc::new(Float64Array::from(vec![0.0f64, -0.0f64]));+ let mut hashes = vec![DEFAULT_SEED; 2];+ create_xxhash64_hashes(&[array], &mut hashes).unwrap();+ assert_eq!(hashes[0], hashes[1]);+ assert_eq!(hashes[0], spark_compatible_xxhash64(0i64.to_le_bytes(), 42));+ }++ #[test]+ fn test_xxhash64_string() {+ let seed = 42u64;+ assert_eq!(spark_compatible_xxhash64("hello", seed), 0xc3629e6318d53932);+ assert_eq!(spark_compatible_xxhash64("", seed), 0x98b1582b0977e704);+ assert_eq!(spark_compatible_xxhash64("abc", seed), 0x13c1d910702770e6);+ }++ #[test]+ fn test_xxhash64_string_emoji_cjk() {+ let seed = 42u64;+ let h1 = spark_compatible_xxhash64("😁", seed);+ assert_ne!(h1, seed);+ let h2 = spark_compatible_xxhash64("天地", seed);+ assert_ne!(h2, seed);+ assert_ne!(h1, h2);+ }++ #[test]+ fn test_xxhash64_dictionary_string() {+ use arrow::array::DictionaryArray;+ use arrow::datatypes::Int32Type;++ let dict_array: DictionaryArray<Int32Type> =+ vec!["hello", "world", "abc", "hello", "world"]+ .into_iter()+ .collect();+ let array_ref: ArrayRef = Arc::new(dict_array);++ let mut hashes = vec![DEFAULT_SEED; 5];+ create_xxhash64_hashes(&[array_ref], &mut hashes).unwrap();++ assert_eq!(hashes[0], spark_compatible_xxhash64("hello", 42));+ assert_eq!(hashes[1], spark_compatible_xxhash64("world", 42));+ assert_eq!(hashes[2], spark_compatible_xxhash64("abc", 42));+ assert_eq!(hashes[3], hashes[0]);+ assert_eq!(hashes[4], hashes[1]);+ }++ #[test]+ fn test_xxhash64_dictionary_int() {+ use arrow::array::DictionaryArray;+ use arrow::datatypes::Int32Type;++ let keys = Int32Array::from(vec![0, 1, 2, 0, 1]);+ let values = Int32Array::from(vec![100, 200, 300]);+ let dict_array =+ DictionaryArray::<Int32Type>::try_new(keys, Arc::new(values)).unwrap();+ let array_ref: ArrayRef = Arc::new(dict_array);++ let mut hashes = vec![DEFAULT_SEED; 5];+ create_xxhash64_hashes(&[array_ref], &mut hashes).unwrap();++ assert_eq!(+ hashes[0],+ spark_compatible_xxhash64(100i32.to_le_bytes(), 42)+ );+ assert_eq!(+ hashes[1],+ spark_compatible_xxhash64(200i32.to_le_bytes(), 42)+ );+ assert_eq!(+ hashes[2],+ spark_compatible_xxhash64(300i32.to_le_bytes(), 42)+ );+ assert_eq!(hashes[3], hashes[0]);+ assert_eq!(hashes[4], hashes[1]);+ }++ #[test]+ fn test_xxhash64_dictionary_with_nulls() {+ use arrow::array::DictionaryArray;+ use arrow::datatypes::Int32Type;++ let keys = Int32Array::from(vec![Some(0), None, Some(1), Some(0), None]);+ let values = StringArray::from(vec!["hello", "world"]);+ let dict_array =+ DictionaryArray::<Int32Type>::try_new(keys, Arc::new(values)).unwrap();+ let array_ref: ArrayRef = Arc::new(dict_array);++ let mut hashes = vec![DEFAULT_SEED; 5];+ create_xxhash64_hashes(&[array_ref], &mut hashes).unwrap();++ assert_eq!(hashes[0], spark_compatible_xxhash64("hello", 42));+ assert_eq!(hashes[2], spark_compatible_xxhash64("world", 42));+ assert_eq!(hashes[3], spark_compatible_xxhash64("hello", 42));+ assert_eq!(hashes[1], DEFAULT_SEED);+ assert_eq!(hashes[4], DEFAULT_SEED);+ }++ #[test]+ fn test_xxhash64_dictionary_non_first_column() {+ use arrow::array::DictionaryArray;+ use arrow::datatypes::Int32Type;++ let dict_array: DictionaryArray<Int32Type> =+ vec!["hello", "world", "abc"].into_iter().collect();+ let array_ref: ArrayRef = Arc::new(dict_array);++ let mut hashes = vec![123u64, 456u64, 789u64];+ create_xxhash64_hashes_dictionary::<Int32Type>(&array_ref, &mut hashes, false)+ .unwrap();++ assert_eq!(hashes[0], spark_compatible_xxhash64("hello", 123));+ assert_eq!(hashes[1], spark_compatible_xxhash64("world", 456));+ assert_eq!(hashes[2], spark_compatible_xxhash64("abc", 789));+ }++ #[test]+ fn test_xxhash64_fixed_size_binary() {+ let array = FixedSizeBinaryArray::from(vec![+ Some(&[0x01, 0x02, 0x03, 0x04][..]),+ Some(&[0x05, 0x06, 0x07, 0x08][..]),+ None,+ Some(&[0x00, 0x00, 0x00, 0x00][..]),+ ]);+ let array_ref: ArrayRef = Arc::new(array);++ let mut hashes = vec![DEFAULT_SEED; 4];+ create_xxhash64_hashes(&[array_ref], &mut hashes).unwrap();++ assert_eq!(+ hashes[0],+ spark_compatible_xxhash64([0x01, 0x02, 0x03, 0x04], 42)+ );+ assert_eq!(+ hashes[1],+ spark_compatible_xxhash64([0x05, 0x06, 0x07, 0x08], 42)+ );+ assert_eq!(hashes[2], DEFAULT_SEED);+ assert_eq!(+ hashes[3],+ spark_compatible_xxhash64([0x00, 0x00, 0x00, 0x00], 42)+ );+ }++ #[test]+ fn test_xxhash64_struct() {+ use arrow::array::StructArray;+ use arrow::datatypes::Field;++ let int_array = Int32Array::from(vec![1, 2, 3]);+ let str_array = StringArray::from(vec!["a", "b", "c"]);+ let struct_array = StructArray::from(vec![+ (+ Arc::new(Field::new("a", DataType::Int32, false)),+ Arc::new(int_array) as ArrayRef,+ ),+ (+ Arc::new(Field::new("b", DataType::Utf8, false)),+ Arc::new(str_array) as ArrayRef,+ ),+ ]);+ let array_ref: ArrayRef = Arc::new(struct_array);++ let mut hashes = vec![DEFAULT_SEED; 3];+ create_xxhash64_hashes(&[array_ref], &mut hashes).unwrap();++ for hash in &hashes {+ assert_ne!(*hash, DEFAULT_SEED);+ }+ assert_ne!(hashes[0], hashes[1]);+ assert_ne!(hashes[1], hashes[2]);+ }++ #[test]+ fn test_xxhash64_list() {+ use arrow::array::ListArray;+ use arrow::buffer::OffsetBuffer;+ use arrow::datatypes::Field;++ let values = Int32Array::from(vec![1, 2, 3, 4, 5, 6]);+ let offsets = OffsetBuffer::new(vec![0i32, 2, 3, 6].into());+ let list_array = ListArray::new(+ Arc::new(Field::new_list_field(DataType::Int32, false)),+ offsets,+ Arc::new(values),+ None,+ );+ let array_ref: ArrayRef = Arc::new(list_array);++ let mut hashes = vec![DEFAULT_SEED; 3];+ create_xxhash64_hashes(&[array_ref], &mut hashes).unwrap();++ for hash in &hashes {+ assert_ne!(*hash, DEFAULT_SEED);+ }+ assert_ne!(hashes[0], hashes[1]);+ }+}diff --git a/datafusion/sqllogictest/test_files/spark/hash/xxhash64.slt b/datafusion/sqllogictest/test_files/spark/hash/xxhash64.sltnew file mode 100644index 0000000000..cd7d6e4947--- /dev/null+++ b/datafusion/sqllogictest/test_files/spark/hash/xxhash64.slt@@ -0,0 +1,197 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at++# http://www.apache.org/licenses/LICENSE-2.0++# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++# Tests for Spark-compatible xxhash64+# Reference: https://spark.apache.org/docs/latest/api/sql/index.html#xxhash64++# Basic integer tests (Int32 - hashed as 4 bytes)+query I+SELECT xxhash64(1);+----+-7001672635703045582++query I+SELECT xxhash64(0);+----+-5252525462095825812++query I+SELECT xxhash64(-1);+----+3858142552250413010++# Boundary values+query I+SELECT xxhash64(arrow_cast(2147483647, 'Int32'));+----+1508894993788531228++query I+SELECT xxhash64(arrow_cast(-2147483648, 'Int32'));+----+2073849959933241805++query I+SELECT xxhash64(arrow_cast(9223372036854775807, 'Int64'));+----+-3246596055638297850++query I+SELECT xxhash64(arrow_cast(-9223372036854775808, 'Int64'));+----+-8619748838626508300++# String tests+query I+SELECT xxhash64('hello');+----+-4367754540140381902++query I+SELECT xxhash64('');+----+-7444071767201028348++query I+SELECT xxhash64('abc');+----+1423657621850124518++# Emoji and CJK strings+query I+SELECT xxhash64('😁');+----+-6337236088984028203++query I+SELECT xxhash64('天地');+----+-235771157374669727++# NULL handling - nulls don't change the hash (keeps seed value 42)+query I+SELECT xxhash64(NULL);+----+42++# Multiple column hashing+query I+SELECT xxhash64(1, 2);+----+-8198029865082835910++query I+SELECT xxhash64('a', 'b', 'c');+----+-8775012835737190202++query I+SELECT xxhash64(1, 'hello');+----+-7340167327466467369++# Int64 tests+query I+SELECT xxhash64(arrow_cast(1, 'Int64'));+----+-7001672635703045582++query I+SELECT xxhash64(arrow_cast(0, 'Int64'));+----+-5252525462095825812++# Float tests - note: -0.0 and 0.0 hash to the same value in Spark+query I+SELECT xxhash64(1.0);+----+-2162451265447482029++query I+SELECT xxhash64(0.0);+----+-5252525462095825812++query I+SELECT xxhash64(-0.0);+----+-5252525462095825812++# Float edge cases+query I+SELECT xxhash64(-1.0);+----+-8296228499959607521++query I+SELECT xxhash64(arrow_cast(1.0E20, 'Float64'));+----+-1089066926670605926++# Boolean tests+query I+SELECT xxhash64(true);+----+-6698625589789238999++query I+SELECT xxhash64(false);+----+3614696996920510707++# Binary tests+query I+SELECT xxhash64(arrow_cast('hello', 'Binary'));+----+-4367754540140381902++# Date tests+query I+SELECT xxhash64(arrow_cast('2023-01-15', 'Date32'));+----+290591839883576215++# Struct hashing+query I+SELECT xxhash64(named_struct('a', 1, 'b', 2));+----+-8198029865082835910++# Array/List hashing - Spark chains hashes across list elements:+# hash(elem_n, hash(elem_n-1, ... hash(elem_0, seed) ...))+query I+SELECT xxhash64(arrow_cast(make_array(1, 2, 3), 'List(Int32)'));+----+8592097078962733837++# Test with table data+statement ok+CREATE TABLE xxhash_test AS SELECT * FROM (VALUES+ (1, 'a'),+ (2, 'b'),+ (3, 'c')+) AS t(id, name);++query II+SELECT id, xxhash64(id, name) FROM xxhash_test ORDER BY id;+----+1 -8527884376001292090+2 -7687314219880041104+3 -7773406846575579340++statement ok+DROP TABLE xxhash_test;