Skip to content

Apache DataFusion SQL Query Engine

feat: add Spark-compatible xxhash64 function (#21967)

Andy Grove6b134ddaf565
diff --git a/Cargo.lock b/Cargo.lock
index 79c6054726..b378253455 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2550,6 +2550,7 @@ dependencies = [
"serde_json",
"sha1 0.11.0",
"sha2",
+ "twox-hash",
"url",
]
@@ -6446,6 +6447,9 @@ name = "twox-hash"
version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
+dependencies = [
+ "rand 0.9.4",
+]
[[package]]
name = "typenum"
diff --git a/datafusion/spark/Cargo.toml b/datafusion/spark/Cargo.toml
index b03a732107..14f9396d76 100644
--- a/datafusion/spark/Cargo.toml
+++ b/datafusion/spark/Cargo.toml
@@ -64,6 +64,7 @@ rand = { workspace = true }
serde_json = { workspace = true }
sha1 = "0.11"
sha2 = { workspace = true }
+twox-hash = "2.1"
url = { workspace = true }
[dev-dependencies]
diff --git a/datafusion/spark/src/function/hash/mod.rs b/datafusion/spark/src/function/hash/mod.rs
index 5860596ac7..351f5d2d50 100644
--- a/datafusion/spark/src/function/hash/mod.rs
+++ b/datafusion/spark/src/function/hash/mod.rs
@@ -18,6 +18,8 @@
pub mod crc32;
pub mod sha1;
pub mod sha2;
+pub(crate) mod utils;
+pub mod xxhash64;
use datafusion_expr::ScalarUDF;
use datafusion_functions::make_udf_function;
@@ -26,16 +28,18 @@ use std::sync::Arc;
make_udf_function!(crc32::SparkCrc32, crc32);
make_udf_function!(sha1::SparkSha1, sha1);
make_udf_function!(sha2::SparkSha2, sha2);
+make_udf_function!(xxhash64::SparkXxhash64, xxhash64);
pub mod expr_fn {
use datafusion_functions::export_functions;
export_functions!(
(crc32, "crc32(expr) - Returns a cyclic redundancy check value of the expr as a bigint.", arg1),
(sha1, "sha1(expr) - Returns a SHA-1 hash value of the expr as a hex string.", arg1),
- (sha2, "sha2(expr, bitLength) - Returns a checksum of SHA-2 family as a hex string of expr. SHA-224, SHA-256, SHA-384, and SHA-512 are supported. Bit length of 0 is equivalent to 256.", arg1 arg2)
+ (sha2, "sha2(expr, bitLength) - Returns a checksum of SHA-2 family as a hex string of expr. SHA-224, SHA-256, SHA-384, and SHA-512 are supported. Bit length of 0 is equivalent to 256.", arg1 arg2),
+ (xxhash64, "xxhash64(expr1, expr2, ...) - Returns a 64-bit hash value of the arguments using xxHash.", args)
);
}
pub fn functions() -> Vec<Arc<ScalarUDF>> {
- vec![crc32(), sha1(), sha2()]
+ vec![crc32(), sha1(), sha2(), xxhash64()]
}
diff --git a/datafusion/spark/src/function/hash/utils.rs b/datafusion/spark/src/function/hash/utils.rs
new file mode 100644
index 0000000000..e7918d33ec
--- /dev/null
+++ b/datafusion/spark/src/function/hash/utils.rs
@@ -0,0 +1,1005 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Shared helpers for Spark-compatible hash functions.
+//!
+//! Generic helpers ([`hash_primitive_values`], [`hash_bytes_values`]) cover
+//! the cases where a function-based abstraction is clean; less uniform paths
+//! (booleans, decimals, list/map dispatch) remain as macros.
+//!
+//! Ported from Apache DataFusion Comet:
+//! <https://github.com/apache/datafusion-comet/blob/main/native/spark-expr/src/hash_funcs/utils.rs>
+
+use arrow::array::{Array, PrimitiveArray};
+use arrow::datatypes::ArrowPrimitiveType;
+use datafusion_common::Result;
+
+/// Hash the values of a primitive array, calling `transform` to convert each
+/// value into a byte slice before feeding it to `hash_method`.
+#[inline]
+pub(crate) fn hash_primitive_values<P, F, B, H>(
+ array: &PrimitiveArray<P>,
+ hashes: &mut [u64],
+ transform: F,
+ hash_method: H,
+) where
+ P: ArrowPrimitiveType,
+ F: Fn(P::Native) -> B,
+ B: AsRef<[u8]>,
+ H: Fn(B, u64) -> u64,
+{
+ let values = array.values();
+ if array.null_count() == 0 {
+ // Fast path: no nulls, skip null checks
+ for (i, hash) in hashes.iter_mut().enumerate() {
+ *hash = hash_method(transform(values[i]), *hash);
+ }
+ } else {
+ // Slow path: check each row for null
+ for (i, hash) in hashes.iter_mut().enumerate() {
+ if !array.is_null(i) {
+ *hash = hash_method(transform(values[i]), *hash);
+ }
+ }
+ }
+}
+
+/// Hash an array by calling `value_at(array, i)` to produce a byte slice for
+/// each non-null row. Used for byte-slice arrays (Utf8/Binary/FixedSizeBinary
+/// and their Large/View variants) and for arrays whose values need a small
+/// transform (e.g. booleans widened to `i32`, decimals to `i128` bytes).
+#[inline]
+pub(crate) fn hash_bytes_values<A, B, H>(
+ array: &A,
+ hashes: &mut [u64],
+ value_at: impl Fn(&A, usize) -> B,
+ hash_method: H,
+) where
+ A: Array,
+ B: AsRef<[u8]>,
+ H: Fn(B, u64) -> u64,
+{
+ if array.null_count() == 0 {
+ for (i, hash) in hashes.iter_mut().enumerate() {
+ *hash = hash_method(value_at(array, i), *hash);
+ }
+ } else {
+ for (i, hash) in hashes.iter_mut().enumerate() {
+ if !array.is_null(i) {
+ *hash = hash_method(value_at(array, i), *hash);
+ }
+ }
+ }
+}
+
+/// Fallible variant of [`hash_bytes_values`]: `value_at` may return an error
+/// (used by the small-decimal path, where values may not fit in `i64`).
+#[inline]
+pub(crate) fn try_hash_bytes_values<A, B, H>(
+ array: &A,
+ hashes: &mut [u64],
+ value_at: impl Fn(&A, usize) -> Result<B>,
+ hash_method: H,
+) -> Result<()>
+where
+ A: Array,
+ B: AsRef<[u8]>,
+ H: Fn(B, u64) -> u64,
+{
+ if array.null_count() == 0 {
+ for (i, hash) in hashes.iter_mut().enumerate() {
+ *hash = hash_method(value_at(array, i)?, *hash);
+ }
+ } else {
+ for (i, hash) in hashes.iter_mut().enumerate() {
+ if !array.is_null(i) {
+ *hash = hash_method(value_at(array, i)?, *hash);
+ }
+ }
+ }
+ Ok(())
+}
+
+/// Downcast `$column` to `$array_type`, panicking with a useful error if the
+/// type does not match.
+#[macro_export]
+#[doc(hidden)]
+macro_rules! downcast_array {
+ ($column:ident, $array_type:ident) => {
+ $column
+ .as_any()
+ .downcast_ref::<$array_type>()
+ .unwrap_or_else(|| {
+ panic!(
+ "Failed to downcast column to {}. Actual data type: {:?}.",
+ stringify!($array_type),
+ $column.data_type()
+ )
+ })
+ };
+}
+
+/// Hash a byte-slice-accessor array (strings, binary, fixed-size binary, ...)
+/// where `value(i)` returns the item directly.
+///
+/// Kept as a macro because `a.value(i)` returns a borrow whose lifetime
+/// depends on the array, which Rust's closure inference does not propagate
+/// through a `Fn(&A, usize) -> B` bound with a free type parameter `B`.
+#[macro_export]
+#[doc(hidden)]
+macro_rules! hash_array {
+ ($array_type:ident, $column:ident, $hashes:ident, $hash_method:ident) => {
+ let array = $crate::downcast_array!($column, $array_type);
+ if array.null_count() == 0 {
+ for i in 0..$hashes.len() {
+ $hashes[i] = $hash_method(&array.value(i), $hashes[i]);
+ }
+ } else {
+ for i in 0..$hashes.len() {
+ if !array.is_null(i) {
+ $hashes[i] = $hash_method(&array.value(i), $hashes[i]);
+ }
+ }
+ }
+ };
+}
+
+/// Hash a `BooleanArray`, widening each value to `$hash_input_type` first.
+#[macro_export]
+#[doc(hidden)]
+macro_rules! hash_array_boolean {
+ ($array_type:ident, $column:ident, $hash_input_type:ident, $hashes:ident, $hash_method:ident) => {
+ let array = $crate::downcast_array!($column, $array_type);
+ $crate::function::hash::utils::hash_bytes_values(
+ array,
+ $hashes,
+ |a, i| $hash_input_type::from(a.value(i)).to_le_bytes(),
+ $hash_method,
+ );
+ };
+}
+
+/// Hash a primitive array, widening each native value to `$ty` before hashing.
+#[macro_export]
+#[doc(hidden)]
+macro_rules! hash_array_primitive {
+ ($array_type:ident, $column:ident, $ty:ident, $hashes:ident, $hash_method:ident) => {
+ let array = $crate::downcast_array!($column, $array_type);
+ $crate::function::hash::utils::hash_primitive_values(
+ array,
+ $hashes,
+ |v| (v as $ty).to_le_bytes(),
+ $hash_method,
+ );
+ };
+}
+
+/// Hash a floating-point primitive array, normalizing `-0.0` to `0.0` per Spark.
+#[macro_export]
+#[doc(hidden)]
+macro_rules! hash_array_primitive_float {
+ ($array_type:ident, $column:ident, $ty:ident, $ty2:ident, $hashes:ident, $hash_method:ident) => {
+ let array = $crate::downcast_array!($column, $array_type);
+ $crate::function::hash::utils::hash_primitive_values(
+ array,
+ $hashes,
+ |v| {
+ // Spark uses 0 as hash for -0.0, see `Murmur3Hash` expression.
+ if v == 0.0 && v.is_sign_negative() {
+ (0 as $ty2).to_le_bytes()
+ } else {
+ (v as $ty).to_le_bytes()
+ }
+ },
+ $hash_method,
+ );
+ };
+}
+
+/// Hash a small (precision <= 18) decimal array by reducing each value to `i64`.
+/// Errors via `?` if a value does not fit in `i64`.
+#[macro_export]
+#[doc(hidden)]
+macro_rules! hash_array_small_decimal {
+ ($array_type:ident, $column:ident, $hashes:ident, $hash_method:ident) => {
+ let array = $crate::downcast_array!($column, $array_type);
+ $crate::function::hash::utils::try_hash_bytes_values(
+ array,
+ $hashes,
+ |a, i| {
+ i64::try_from(a.value(i))
+ .map(|v| v.to_le_bytes())
+ .map_err(|e| DataFusionError::Execution(e.to_string()))
+ },
+ $hash_method,
+ )?;
+ };
+}
+
+/// Hash a wide decimal array using the raw 128-bit byte representation.
+#[macro_export]
+#[doc(hidden)]
+macro_rules! hash_array_decimal {
+ ($array_type:ident, $column:ident, $hashes:ident, $hash_method:ident) => {
+ let array = $crate::downcast_array!($column, $array_type);
+ $crate::function::hash::utils::hash_bytes_values(
+ array,
+ $hashes,
+ |a, i| a.value(i).to_le_bytes(),
+ $hash_method,
+ );
+ };
+}
+
+/// Hash a list array with primitive elements by directly accessing the underlying buffer.
+/// This avoids the overhead of slicing and recursive calls for common cases.
+/// Supports both variable-length lists (with offsets) and fixed-size lists.
+#[macro_export]
+#[doc(hidden)]
+macro_rules! hash_list_primitive {
+ // Variable-length list variant (List/LargeList)
+ (offsets: $offsets:expr, $list_array:ident, $elem_array:ident, $hashes:ident, $hash_method:ident, $value_transform:expr) => {
+ if $list_array.null_count() == 0 && $elem_array.null_count() == 0 {
+ for (row_idx, hash) in $hashes.iter_mut().enumerate() {
+ let start = $offsets[row_idx] as usize;
+ let end = $offsets[row_idx + 1] as usize;
+ for elem_idx in start..end {
+ let value = $elem_array.value(elem_idx);
+ *hash = $hash_method($value_transform(value), *hash);
+ }
+ }
+ } else {
+ for (row_idx, hash) in $hashes.iter_mut().enumerate() {
+ if !$list_array.is_null(row_idx) {
+ let start = $offsets[row_idx] as usize;
+ let end = $offsets[row_idx + 1] as usize;
+ for elem_idx in start..end {
+ if !$elem_array.is_null(elem_idx) {
+ let value = $elem_array.value(elem_idx);
+ *hash = $hash_method($value_transform(value), *hash);
+ }
+ }
+ }
+ }
+ }
+ };
+ // Fixed-size list variant
+ (fixed_size: $list_size:expr, $list_array:ident, $elem_array:ident, $hashes:ident, $hash_method:ident, $value_transform:expr) => {
+ if $list_array.null_count() == 0 && $elem_array.null_count() == 0 {
+ for (row_idx, hash) in $hashes.iter_mut().enumerate() {
+ let start = row_idx * $list_size;
+ for elem_idx in 0..$list_size {
+ let value = $elem_array.value(start + elem_idx);
+ *hash = $hash_method($value_transform(value), *hash);
+ }
+ }
+ } else {
+ for (row_idx, hash) in $hashes.iter_mut().enumerate() {
+ if !$list_array.is_null(row_idx) {
+ let start = row_idx * $list_size;
+ for elem_idx in 0..$list_size {
+ if !$elem_array.is_null(start + elem_idx) {
+ let value = $elem_array.value(start + elem_idx);
+ *hash = $hash_method($value_transform(value), *hash);
+ }
+ }
+ }
+ }
+ }
+ };
+}
+
+/// Hash a list array by recursively hashing each element.
+/// For each row, we hash all elements in the list.
+/// Spark hashes arrays by recursively hashing each element, where each
+/// element's hash is computed using the previous element's hash as the seed.
+/// This creates a chain: hash(elem_n, hash(elem_n-1, ... hash(elem_0, seed)...))
+/// Dispatches hash operations for List/LargeList/FixedSizeList arrays with primitive element types.
+/// This macro eliminates duplication by handling the type-to-array mapping for all supported primitives.
+#[macro_export]
+#[doc(hidden)]
+macro_rules! hash_list_with_primitive_elements {
+ // Variant for List/LargeList with offsets
+ (offsets: $list_array_type:ident, $list_array:ident, $values:ident, $offsets:ident, $field:expr, $hashes_buffer:ident, $hash_method:ident, $recursive_hash_method:ident, $fallback_offset_type:ty, $col:ident) => {
+ match $field.data_type() {
+ DataType::Int8 => {
+ let elem_array = $values.as_any().downcast_ref::<Int8Array>().unwrap();
+ $crate::hash_list_primitive!(offsets: $offsets, $list_array, elem_array, $hashes_buffer, $hash_method, |v: i8| (v as i32).to_le_bytes());
+ }
+ DataType::Int16 => {
+ let elem_array = $values.as_any().downcast_ref::<Int16Array>().unwrap();
+ $crate::hash_list_primitive!(offsets: $offsets, $list_array, elem_array, $hashes_buffer, $hash_method, |v: i16| (v as i32).to_le_bytes());
+ }
+ DataType::Int32 => {
+ let elem_array = $values.as_any().downcast_ref::<Int32Array>().unwrap();
+ $crate::hash_list_primitive!(offsets: $offsets, $list_array, elem_array, $hashes_buffer, $hash_method, |v: i32| v.to_le_bytes());
+ }
+ DataType::Int64 => {
+ let elem_array = $values.as_any().downcast_ref::<Int64Array>().unwrap();
+ $crate::hash_list_primitive!(offsets: $offsets, $list_array, elem_array, $hashes_buffer, $hash_method, |v: i64| v.to_le_bytes());
+ }
+ DataType::Float32 => {
+ let elem_array = $values.as_any().downcast_ref::<Float32Array>().unwrap();
+ $crate::hash_list_primitive!(offsets: $offsets, $list_array, elem_array, $hashes_buffer, $hash_method,
+ |v: f32| if v == 0.0 && v.is_sign_negative() { (0_i32).to_le_bytes() } else { v.to_le_bytes() });
+ }
+ DataType::Float64 => {
+ let elem_array = $values.as_any().downcast_ref::<Float64Array>().unwrap();
+ $crate::hash_list_primitive!(offsets: $offsets, $list_array, elem_array, $hashes_buffer, $hash_method,
+ |v: f64| if v == 0.0 && v.is_sign_negative() { (0_i64).to_le_bytes() } else { v.to_le_bytes() });
+ }
+ DataType::Boolean => {
+ let elem_array = $values.as_any().downcast_ref::<BooleanArray>().unwrap();
+ $crate::hash_list_primitive!(offsets: $offsets, $list_array, elem_array, $hashes_buffer, $hash_method, |v: bool| (i32::from(v)).to_le_bytes());
+ }
+ DataType::Utf8 => {
+ let elem_array = $values.as_any().downcast_ref::<StringArray>().unwrap();
+ if $list_array.null_count() == 0 && elem_array.null_count() == 0 {
+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {
+ let start = $offsets[row_idx] as usize;
+ let end = $offsets[row_idx + 1] as usize;
+ for elem_idx in start..end {
+ *hash = $hash_method(elem_array.value(elem_idx), *hash);
+ }
+ }
+ } else {
+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {
+ if !$list_array.is_null(row_idx) {
+ let start = $offsets[row_idx] as usize;
+ let end = $offsets[row_idx + 1] as usize;
+ for elem_idx in start..end {
+ if !elem_array.is_null(elem_idx) {
+ *hash = $hash_method(elem_array.value(elem_idx), *hash);
+ }
+ }
+ }
+ }
+ }
+ }
+ DataType::Binary => {
+ let elem_array = $values.as_any().downcast_ref::<BinaryArray>().unwrap();
+ if $list_array.null_count() == 0 && elem_array.null_count() == 0 {
+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {
+ let start = $offsets[row_idx] as usize;
+ let end = $offsets[row_idx + 1] as usize;
+ for elem_idx in start..end {
+ *hash = $hash_method(elem_array.value(elem_idx), *hash);
+ }
+ }
+ } else {
+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {
+ if !$list_array.is_null(row_idx) {
+ let start = $offsets[row_idx] as usize;
+ let end = $offsets[row_idx + 1] as usize;
+ for elem_idx in start..end {
+ if !elem_array.is_null(elem_idx) {
+ *hash = $hash_method(elem_array.value(elem_idx), *hash);
+ }
+ }
+ }
+ }
+ }
+ }
+ DataType::Date32 => {
+ let elem_array = $values.as_any().downcast_ref::<Date32Array>().unwrap();
+ $crate::hash_list_primitive!(offsets: $offsets, $list_array, elem_array, $hashes_buffer, $hash_method, |v: i32| v.to_le_bytes());
+ }
+ DataType::Timestamp(TimeUnit::Microsecond, _) => {
+ let elem_array = $values.as_any().downcast_ref::<TimestampMicrosecondArray>().unwrap();
+ $crate::hash_list_primitive!(offsets: $offsets, $list_array, elem_array, $hashes_buffer, $hash_method, |v: i64| v.to_le_bytes());
+ }
+ _ => {
+ // Fall back to recursive approach for complex element types
+ $crate::hash_list_array!($list_array_type, $fallback_offset_type, $col, $hashes_buffer, $recursive_hash_method);
+ }
+ }
+ };
+ // Variant for FixedSizeList with fixed size
+ (fixed_size: $list_array:ident, $values:ident, $list_size:ident, $field:expr, $hashes_buffer:ident, $hash_method:ident, $recursive_hash_method:ident) => {
+ match $field.data_type() {
+ DataType::Int8 => {
+ let elem_array = $values.as_any().downcast_ref::<Int8Array>().unwrap();
+ $crate::hash_list_primitive!(fixed_size: $list_size, $list_array, elem_array, $hashes_buffer, $hash_method, |v: i8| (v as i32).to_le_bytes());
+ }
+ DataType::Int16 => {
+ let elem_array = $values.as_any().downcast_ref::<Int16Array>().unwrap();
+ $crate::hash_list_primitive!(fixed_size: $list_size, $list_array, elem_array, $hashes_buffer, $hash_method, |v: i16| (v as i32).to_le_bytes());
+ }
+ DataType::Int32 => {
+ let elem_array = $values.as_any().downcast_ref::<Int32Array>().unwrap();
+ $crate::hash_list_primitive!(fixed_size: $list_size, $list_array, elem_array, $hashes_buffer, $hash_method, |v: i32| v.to_le_bytes());
+ }
+ DataType::Int64 => {
+ let elem_array = $values.as_any().downcast_ref::<Int64Array>().unwrap();
+ $crate::hash_list_primitive!(fixed_size: $list_size, $list_array, elem_array, $hashes_buffer, $hash_method, |v: i64| v.to_le_bytes());
+ }
+ DataType::Float32 => {
+ let elem_array = $values.as_any().downcast_ref::<Float32Array>().unwrap();
+ $crate::hash_list_primitive!(fixed_size: $list_size, $list_array, elem_array, $hashes_buffer, $hash_method,
+ |v: f32| if v == 0.0 && v.is_sign_negative() { (0_i32).to_le_bytes() } else { v.to_le_bytes() });
+ }
+ DataType::Float64 => {
+ let elem_array = $values.as_any().downcast_ref::<Float64Array>().unwrap();
+ $crate::hash_list_primitive!(fixed_size: $list_size, $list_array, elem_array, $hashes_buffer, $hash_method,
+ |v: f64| if v == 0.0 && v.is_sign_negative() { (0_i64).to_le_bytes() } else { v.to_le_bytes() });
+ }
+ DataType::Boolean => {
+ let elem_array = $values.as_any().downcast_ref::<BooleanArray>().unwrap();
+ $crate::hash_list_primitive!(fixed_size: $list_size, $list_array, elem_array, $hashes_buffer, $hash_method, |v: bool| (i32::from(v)).to_le_bytes());
+ }
+ DataType::Utf8 => {
+ let elem_array = $values.as_any().downcast_ref::<StringArray>().unwrap();
+ if $list_array.null_count() == 0 && elem_array.null_count() == 0 {
+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {
+ let start = row_idx * $list_size;
+ for elem_idx in 0..$list_size {
+ *hash = $hash_method(elem_array.value(start + elem_idx), *hash);
+ }
+ }
+ } else {
+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {
+ if !$list_array.is_null(row_idx) {
+ let start = row_idx * $list_size;
+ for elem_idx in 0..$list_size {
+ if !elem_array.is_null(start + elem_idx) {
+ *hash = $hash_method(elem_array.value(start + elem_idx), *hash);
+ }
+ }
+ }
+ }
+ }
+ }
+ DataType::Binary => {
+ let elem_array = $values.as_any().downcast_ref::<BinaryArray>().unwrap();
+ if $list_array.null_count() == 0 && elem_array.null_count() == 0 {
+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {
+ let start = row_idx * $list_size;
+ for elem_idx in 0..$list_size {
+ *hash = $hash_method(elem_array.value(start + elem_idx), *hash);
+ }
+ }
+ } else {
+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {
+ if !$list_array.is_null(row_idx) {
+ let start = row_idx * $list_size;
+ for elem_idx in 0..$list_size {
+ if !elem_array.is_null(start + elem_idx) {
+ *hash = $hash_method(elem_array.value(start + elem_idx), *hash);
+ }
+ }
+ }
+ }
+ }
+ }
+ DataType::Date32 => {
+ let elem_array = $values.as_any().downcast_ref::<Date32Array>().unwrap();
+ $crate::hash_list_primitive!(fixed_size: $list_size, $list_array, elem_array, $hashes_buffer, $hash_method, |v: i32| v.to_le_bytes());
+ }
+ DataType::Timestamp(TimeUnit::Microsecond, _) => {
+ let elem_array = $values.as_any().downcast_ref::<TimestampMicrosecondArray>().unwrap();
+ $crate::hash_list_primitive!(fixed_size: $list_size, $list_array, elem_array, $hashes_buffer, $hash_method, |v: i64| v.to_le_bytes());
+ }
+ _ => {
+ // Fall back to recursive approach for complex element types
+ if $list_array.null_count() == 0 {
+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {
+ let start = row_idx * $list_size;
+ for elem_idx in 0..$list_size {
+ let elem_array = $values.slice(start + elem_idx, 1);
+ let mut single_hash = [*hash];
+ $recursive_hash_method(&[elem_array], &mut single_hash)?;
+ *hash = single_hash[0];
+ }
+ }
+ } else {
+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {
+ if !$list_array.is_null(row_idx) {
+ let start = row_idx * $list_size;
+ for elem_idx in 0..$list_size {
+ let elem_array = $values.slice(start + elem_idx, 1);
+ let mut single_hash = [*hash];
+ $recursive_hash_method(&[elem_array], &mut single_hash)?;
+ *hash = single_hash[0];
+ }
+ }
+ }
+ }
+ }
+ }
+ };
+}
+
+/// Hash a map array by hashing its key/value entries in order.
+///
+/// Specializes for common key/value type combinations so the hot path stays
+/// monomorphic; falls back to recursive hashing for arbitrary nested types.
+#[macro_export]
+#[doc(hidden)]
+macro_rules! hash_map_array {
+ // Specialized variant: typed key and value arrays, byte transforms.
+ ($map_array:ident, $key_array:ident, $value_array:ident, $offsets:ident, $hashes_buffer:ident, $hash_method:ident, $key_transform:expr, $value_transform:expr) => {
+ if $map_array.null_count() == 0
+ && $key_array.null_count() == 0
+ && $value_array.null_count() == 0
+ {
+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {
+ let start = $offsets[row_idx] as usize;
+ let end = $offsets[row_idx + 1] as usize;
+ for entry_idx in start..end {
+ *hash =
+ $hash_method($key_transform($key_array.value(entry_idx)), *hash);
+ *hash = $hash_method(
+ $value_transform($value_array.value(entry_idx)),
+ *hash,
+ );
+ }
+ }
+ } else {
+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {
+ if !$map_array.is_null(row_idx) {
+ let start = $offsets[row_idx] as usize;
+ let end = $offsets[row_idx + 1] as usize;
+ for entry_idx in start..end {
+ if !$key_array.is_null(entry_idx) {
+ *hash = $hash_method(
+ $key_transform($key_array.value(entry_idx)),
+ *hash,
+ );
+ }
+ if !$value_array.is_null(entry_idx) {
+ *hash = $hash_method(
+ $value_transform($value_array.value(entry_idx)),
+ *hash,
+ );
+ }
+ }
+ }
+ }
+ }
+ };
+ // Fallback variant: recursively hash each key/value entry.
+ (recursive: $map_array:ident, $keys:ident, $values:ident, $offsets:ident, $hashes_buffer:ident, $recursive_hash_method:ident) => {
+ if $map_array.null_count() == 0 {
+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {
+ let start = $offsets[row_idx] as usize;
+ let end = $offsets[row_idx + 1] as usize;
+ for entry_idx in start..end {
+ let key_array = $keys.slice(entry_idx, 1);
+ let mut single_hash = [*hash];
+ $recursive_hash_method(&[key_array], &mut single_hash)?;
+ *hash = single_hash[0];
+
+ let value_array = $values.slice(entry_idx, 1);
+ single_hash = [*hash];
+ $recursive_hash_method(&[value_array], &mut single_hash)?;
+ *hash = single_hash[0];
+ }
+ }
+ } else {
+ for (row_idx, hash) in $hashes_buffer.iter_mut().enumerate() {
+ if !$map_array.is_null(row_idx) {
+ let start = $offsets[row_idx] as usize;
+ let end = $offsets[row_idx + 1] as usize;
+ for entry_idx in start..end {
+ let key_array = $keys.slice(entry_idx, 1);
+ let mut single_hash = [*hash];
+ $recursive_hash_method(&[key_array], &mut single_hash)?;
+ *hash = single_hash[0];
+
+ let value_array = $values.slice(entry_idx, 1);
+ single_hash = [*hash];
+ $recursive_hash_method(&[value_array], &mut single_hash)?;
+ *hash = single_hash[0];
+ }
+ }
+ }
+ }
+ };
+}
+
+/// Dispatch over a `MapArray`'s key/value types, calling [`hash_map_array!`]
+/// with the right specialization (or the recursive fallback).
+#[macro_export]
+#[doc(hidden)]
+macro_rules! hash_map_with_typed_entries {
+ ($col:ident, $field:expr, $hashes_buffer:ident, $hash_method:ident, $recursive_hash_method:ident) => {
+ let map_array = $col.as_any().downcast_ref::<MapArray>().unwrap();
+ let keys = map_array.keys();
+ let values = map_array.values();
+ let offsets = map_array.offsets();
+
+ if let DataType::Struct(fields) = $field.data_type() {
+ let key_type = &fields[0].data_type();
+ let value_type = &fields[1].data_type();
+
+ match (key_type, value_type) {
+ (DataType::Utf8, DataType::Int32) => {
+ let key_array = keys.as_any().downcast_ref::<StringArray>().unwrap();
+ let value_array = values.as_any().downcast_ref::<Int32Array>().unwrap();
+ $crate::hash_map_array!(
+ map_array, key_array, value_array, offsets, $hashes_buffer,
+ $hash_method, |v| v, |v: i32| v.to_le_bytes()
+ );
+ }
+ (DataType::Int32, DataType::Utf8) => {
+ let key_array = keys.as_any().downcast_ref::<Int32Array>().unwrap();
+ let value_array = values.as_any().downcast_ref::<StringArray>().unwrap();
+ $crate::hash_map_array!(
+ map_array, key_array, value_array, offsets, $hashes_buffer,
+ $hash_method, |v: i32| v.to_le_bytes(), |v| v
+ );
+ }
+ (DataType::Utf8, DataType::Utf8) => {
+ let key_array = keys.as_any().downcast_ref::<StringArray>().unwrap();
+ let value_array = values.as_any().downcast_ref::<StringArray>().unwrap();
+ $crate::hash_map_array!(
+ map_array, key_array, value_array, offsets, $hashes_buffer,
+ $hash_method, |v| v, |v| v
+ );
+ }
+ (DataType::Int32, DataType::Int32) => {
+ let key_array = keys.as_any().downcast_ref::<Int32Array>().unwrap();
+ let value_array = values.as_any().downcast_ref::<Int32Array>().unwrap();
+ $crate::hash_map_array!(
+ map_array, key_array, value_array, offsets, $hashes_buffer,
+ $hash_method, |v: i32| v.to_le_bytes(), |v: i32| v.to_le_bytes()
+ );
+ }
+ _ => {
+ $crate::hash_map_array!(
+ recursive: map_array, keys, values, offsets,
+ $hashes_buffer, $recursive_hash_method
+ );
+ }
+ }
+ } else {
+ return Err(DataFusionError::Internal(format!(
+ "Map field type must be a struct, got: {}",
+ $field.data_type()
+ )));
+ }
+ };
+}
+
+#[macro_export]
+#[doc(hidden)]
+macro_rules! hash_list_array {
+ ($array_type:ident, $offset_type:ty, $column: ident, $hashes: ident, $recursive_hash_method: ident) => {
+ let list_array = $column
+ .as_any()
+ .downcast_ref::<$array_type>()
+ .unwrap_or_else(|| {
+ panic!(
+ "Failed to downcast column to {}. Actual data type: {:?}.",
+ stringify!($array_type),
+ $column.data_type()
+ )
+ });
+
+ let values = list_array.values();
+ let offsets = list_array.offsets();
+
+ if list_array.null_count() == 0 {
+ // Fast path: no nulls, skip null checks
+ for (row_idx, hash) in $hashes.iter_mut().enumerate() {
+ let start = offsets[row_idx] as usize;
+ let end = offsets[row_idx + 1] as usize;
+ let len = end - start;
+ // Hash each element in sequence, chaining the hash values
+ for elem_idx in 0..len {
+ let elem_array = values.slice(start + elem_idx, 1);
+ let mut single_hash = [*hash];
+ $recursive_hash_method(&[elem_array], &mut single_hash)?;
+ *hash = single_hash[0];
+ }
+ }
+ } else {
+ // Slow path: array has nulls, check each row
+ for (row_idx, hash) in $hashes.iter_mut().enumerate() {
+ if !list_array.is_null(row_idx) {
+ let start = offsets[row_idx] as usize;
+ let end = offsets[row_idx + 1] as usize;
+ let len = end - start;
+ // Hash each element in sequence, chaining the hash values
+ for elem_idx in 0..len {
+ let elem_array = values.slice(start + elem_idx, 1);
+ let mut single_hash = [*hash];
+ $recursive_hash_method(&[elem_array], &mut single_hash)?;
+ *hash = single_hash[0];
+ }
+ }
+ }
+ }
+ };
+}
+
+/// Creates hash values for every row, based on the values in the
+/// columns.
+///
+/// The number of rows to hash is determined by `hashes_buffer.len()`.
+/// `hashes_buffer` should be pre-sized appropriately
+///
+/// `hash_method` is the hash function to use.
+/// `create_dictionary_hash_method` is the function to create hashes for dictionary arrays input.
+/// `recursive_hash_method` is the function to call for recursive hashing of complex types.
+#[macro_export]
+#[doc(hidden)]
+macro_rules! create_hashes_internal {
+ ($arrays: ident, $hashes_buffer: ident, $hash_method: ident, $create_dictionary_hash_method: ident, $recursive_hash_method: ident) => {
+ use arrow::array::{types::*, *};
+ use arrow::datatypes::{DataType, TimeUnit};
+ use datafusion_common::DataFusionError;
+
+ for (i, col) in $arrays.iter().enumerate() {
+ let first_col = i == 0;
+ match col.data_type() {
+ DataType::Boolean => {
+ $crate::hash_array_boolean!(
+ BooleanArray,
+ col,
+ i32,
+ $hashes_buffer,
+ $hash_method
+ );
+ }
+ DataType::Int8 => {
+ $crate::hash_array_primitive!(
+ Int8Array,
+ col,
+ i32,
+ $hashes_buffer,
+ $hash_method
+ );
+ }
+ DataType::Int16 => {
+ $crate::hash_array_primitive!(
+ Int16Array,
+ col,
+ i32,
+ $hashes_buffer,
+ $hash_method
+ );
+ }
+ DataType::Int32 => {
+ $crate::hash_array_primitive!(
+ Int32Array,
+ col,
+ i32,
+ $hashes_buffer,
+ $hash_method
+ );
+ }
+ DataType::Int64 => {
+ $crate::hash_array_primitive!(
+ Int64Array,
+ col,
+ i64,
+ $hashes_buffer,
+ $hash_method
+ );
+ }
+ DataType::Float32 => {
+ $crate::hash_array_primitive_float!(
+ Float32Array,
+ col,
+ f32,
+ i32,
+ $hashes_buffer,
+ $hash_method
+ );
+ }
+ DataType::Float64 => {
+ $crate::hash_array_primitive_float!(
+ Float64Array,
+ col,
+ f64,
+ i64,
+ $hashes_buffer,
+ $hash_method
+ );
+ }
+ DataType::Timestamp(TimeUnit::Second, _) => {
+ $crate::hash_array_primitive!(
+ TimestampSecondArray,
+ col,
+ i64,
+ $hashes_buffer,
+ $hash_method
+ );
+ }
+ DataType::Timestamp(TimeUnit::Millisecond, _) => {
+ $crate::hash_array_primitive!(
+ TimestampMillisecondArray,
+ col,
+ i64,
+ $hashes_buffer,
+ $hash_method
+ );
+ }
+ DataType::Timestamp(TimeUnit::Microsecond, _) => {
+ $crate::hash_array_primitive!(
+ TimestampMicrosecondArray,
+ col,
+ i64,
+ $hashes_buffer,
+ $hash_method
+ );
+ }
+ DataType::Timestamp(TimeUnit::Nanosecond, _) => {
+ $crate::hash_array_primitive!(
+ TimestampNanosecondArray,
+ col,
+ i64,
+ $hashes_buffer,
+ $hash_method
+ );
+ }
+ DataType::Date32 => {
+ $crate::hash_array_primitive!(
+ Date32Array,
+ col,
+ i32,
+ $hashes_buffer,
+ $hash_method
+ );
+ }
+ DataType::Date64 => {
+ $crate::hash_array_primitive!(
+ Date64Array,
+ col,
+ i64,
+ $hashes_buffer,
+ $hash_method
+ );
+ }
+ DataType::Utf8 => {
+ $crate::hash_array!(StringArray, col, $hashes_buffer, $hash_method);
+ }
+ DataType::LargeUtf8 => {
+ $crate::hash_array!(LargeStringArray, col, $hashes_buffer, $hash_method);
+ }
+ DataType::Utf8View => {
+ $crate::hash_array!(StringViewArray, col, $hashes_buffer, $hash_method);
+ }
+ DataType::Binary => {
+ $crate::hash_array!(BinaryArray, col, $hashes_buffer, $hash_method);
+ }
+ DataType::LargeBinary => {
+ $crate::hash_array!(LargeBinaryArray, col, $hashes_buffer, $hash_method);
+ }
+ DataType::BinaryView => {
+ $crate::hash_array!(BinaryViewArray, col, $hashes_buffer, $hash_method);
+ }
+ DataType::FixedSizeBinary(_) => {
+ $crate::hash_array!(FixedSizeBinaryArray, col, $hashes_buffer, $hash_method);
+ }
+ DataType::Null => {
+ // Nulls don't update the hash
+ }
+ // Apache Spark: if it's a small decimal, i.e. precision <= 18, turn it into long and hash it.
+ // Else, turn it into bytes and hash it.
+ DataType::Decimal128(precision, _) if *precision <= 18 => {
+ $crate::hash_array_small_decimal!(Decimal128Array, col, $hashes_buffer, $hash_method);
+ }
+ DataType::Decimal128(_, _) => {
+ $crate::hash_array_decimal!(Decimal128Array, col, $hashes_buffer, $hash_method);
+ }
+ DataType::Dictionary(index_type, _) => match **index_type {
+ DataType::Int8 => {
+ $create_dictionary_hash_method::<Int8Type>(col, $hashes_buffer, first_col)?;
+ }
+ DataType::Int16 => {
+ $create_dictionary_hash_method::<Int16Type>(
+ col,
+ $hashes_buffer,
+ first_col,
+ )?;
+ }
+ DataType::Int32 => {
+ $create_dictionary_hash_method::<Int32Type>(
+ col,
+ $hashes_buffer,
+ first_col,
+ )?;
+ }
+ DataType::Int64 => {
+ $create_dictionary_hash_method::<Int64Type>(
+ col,
+ $hashes_buffer,
+ first_col,
+ )?;
+ }
+ DataType::UInt8 => {
+ $create_dictionary_hash_method::<UInt8Type>(
+ col,
+ $hashes_buffer,
+ first_col,
+ )?;
+ }
+ DataType::UInt16 => {
+ $create_dictionary_hash_method::<UInt16Type>(
+ col,
+ $hashes_buffer,
+ first_col,
+ )?;
+ }
+ DataType::UInt32 => {
+ $create_dictionary_hash_method::<UInt32Type>(
+ col,
+ $hashes_buffer,
+ first_col,
+ )?;
+ }
+ DataType::UInt64 => {
+ $create_dictionary_hash_method::<UInt64Type>(
+ col,
+ $hashes_buffer,
+ first_col,
+ )?;
+ }
+ _ => {
+ return Err(DataFusionError::Internal(format!(
+ "Unsupported dictionary type in hasher hashing: {}",
+ col.data_type(),
+ )))
+ }
+ },
+ DataType::List(field) => {
+ let list_array = col.as_any().downcast_ref::<ListArray>().unwrap();
+ let values = list_array.values();
+ let offsets = list_array.offsets();
+
+ $crate::hash_list_with_primitive_elements!(offsets: ListArray, list_array, values, offsets, field, $hashes_buffer, $hash_method, $recursive_hash_method, i32, col);
+ }
+ DataType::LargeList(field) => {
+ let list_array = col.as_any().downcast_ref::<LargeListArray>().unwrap();
+ let values = list_array.values();
+ let offsets = list_array.offsets();
+
+ $crate::hash_list_with_primitive_elements!(offsets: LargeListArray, list_array, values, offsets, field, $hashes_buffer, $hash_method, $recursive_hash_method, i64, col);
+ }
+ DataType::FixedSizeList(field, size) => {
+ let list_array = col.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
+ let values = list_array.values();
+ let list_size = *size as usize;
+
+ $crate::hash_list_with_primitive_elements!(fixed_size: list_array, values, list_size, field, $hashes_buffer, $hash_method, $recursive_hash_method);
+ }
+ DataType::Struct(_) => {
+ let struct_array = col.as_any().downcast_ref::<StructArray>().unwrap();
+ // Hash each field of the struct - Spark hashes all fields recursively
+ let columns: Vec<ArrayRef> = struct_array.columns().to_vec();
+ if !columns.is_empty() {
+ $recursive_hash_method(&columns, $hashes_buffer)?;
+ }
+ }
+ DataType::Map(field, _) => {
+ $crate::hash_map_with_typed_entries!(
+ col, field, $hashes_buffer, $hash_method, $recursive_hash_method
+ );
+ }
+ _ => {
+ // This is internal because we should have caught this before.
+ return Err(DataFusionError::Internal(format!(
+ "Unsupported data type in hasher: {}",
+ col.data_type()
+ )));
+ }
+ }
+ }
+ };
+}
diff --git a/datafusion/spark/src/function/hash/xxhash64.rs b/datafusion/spark/src/function/hash/xxhash64.rs
new file mode 100644
index 0000000000..5dca47bcb8
--- /dev/null
+++ b/datafusion/spark/src/function/hash/xxhash64.rs
@@ -0,0 +1,445 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use arrow::array::{
+ Array, ArrayRef, DictionaryArray, Int64Array, types::ArrowDictionaryKeyType,
+};
+use arrow::buffer::{Buffer, ScalarBuffer};
+use arrow::compute::take;
+use arrow::datatypes::{ArrowNativeType, DataType, Field, FieldRef};
+use datafusion_common::{Result, ScalarValue, internal_err};
+use datafusion_expr::{
+ ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ Volatility,
+};
+use twox_hash::XxHash64;
+
+use crate::create_hashes_internal;
+
+const DEFAULT_SEED: u64 = 42;
+
+/// Spark-compatible xxhash64 function.
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#xxhash64>
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkXxhash64 {
+ signature: Signature,
+}
+
+impl Default for SparkXxhash64 {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkXxhash64 {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::variadic_any(Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkXxhash64 {
+ fn name(&self) -> &str {
+ "xxhash64"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ internal_err!("return_field_from_args should be used instead")
+ }
+
+ fn return_field_from_args(&self, _args: ReturnFieldArgs) -> Result<FieldRef> {
+ // Spark's HashExpression overrides nullable to false: NULL inputs are
+ // skipped and the seed is used, so the result is never null.
+ Ok(Arc::new(Field::new(self.name(), DataType::Int64, false)))
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
+ let num_rows = args.number_rows;
+ let mut hashes: Vec<u64> = vec![DEFAULT_SEED; num_rows];
+
+ let arrays = ColumnarValue::values_to_arrays(&args.args)?;
+ create_xxhash64_hashes(&arrays, &mut hashes)?;
+
+ if num_rows == 1 {
+ Ok(ColumnarValue::Scalar(ScalarValue::Int64(Some(
+ hashes[0] as i64,
+ ))))
+ } else {
+ // Reinterpret Vec<u64> as ScalarBuffer<i64> without copying — both
+ // types have identical layout, and `as i64` is a bitcast.
+ let buffer = ScalarBuffer::<i64>::from(Buffer::from_vec(hashes));
+ Ok(ColumnarValue::Array(Arc::new(Int64Array::new(
+ buffer, None,
+ ))))
+ }
+ }
+}
+
+#[inline]
+fn spark_compatible_xxhash64<T: AsRef<[u8]>>(data: T, seed: u64) -> u64 {
+ XxHash64::oneshot(seed, data.as_ref())
+}
+
+/// Hash the values in a dictionary array using xxhash64.
+fn create_xxhash64_hashes_dictionary<K: ArrowDictionaryKeyType>(
+ array: &ArrayRef,
+ hashes_buffer: &mut [u64],
+ first_col: bool,
+) -> Result<()> {
+ let dict_array = array.as_any().downcast_ref::<DictionaryArray<K>>().unwrap();
+ if !first_col {
+ let unpacked = take(dict_array.values().as_ref(), dict_array.keys(), None)?;
+ create_xxhash64_hashes(&[unpacked], hashes_buffer)?;
+ } else {
+ // Hash each dictionary value once, then look up by key. This avoids
+ // redundant hashing of large dictionary entries (e.g. long strings).
+ let dict_values = Arc::clone(dict_array.values());
+ let mut dict_hashes = vec![DEFAULT_SEED; dict_values.len()];
+ create_xxhash64_hashes(&[dict_values], &mut dict_hashes)?;
+
+ for (hash, key) in hashes_buffer.iter_mut().zip(dict_array.keys().iter()) {
+ if let Some(key) = key {
+ *hash = dict_hashes[key.as_usize()]
+ }
+ // No update for Null keys, consistent with other types.
+ }
+ }
+ Ok(())
+}
+
+/// Create xxhash64 hash values for every row, based on the values in the columns.
+///
+/// The number of rows to hash is determined by `hashes_buffer.len()`.
+/// `hashes_buffer` should be pre-sized appropriately and seeded with the
+/// initial hash value (Spark uses `42`).
+fn create_xxhash64_hashes(arrays: &[ArrayRef], hashes_buffer: &mut [u64]) -> Result<()> {
+ create_hashes_internal!(
+ arrays,
+ hashes_buffer,
+ spark_compatible_xxhash64,
+ create_xxhash64_hashes_dictionary,
+ create_xxhash64_hashes
+ );
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use arrow::array::{FixedSizeBinaryArray, Int32Array, StringArray};
+
+ #[test]
+ fn test_xxhash64_nullability() -> Result<()> {
+ let func = SparkXxhash64::new();
+
+ // Spark's xxhash64 is never null (NULL args are skipped, seed is returned),
+ // so the output field is non-nullable regardless of input nullability.
+ let nullable: FieldRef = Arc::new(Field::new("a", DataType::Int32, true));
+ let non_nullable: FieldRef = Arc::new(Field::new("b", DataType::Int32, false));
+
+ let out = func.return_field_from_args(ReturnFieldArgs {
+ arg_fields: &[Arc::clone(&nullable), Arc::clone(&non_nullable)],
+ scalar_arguments: &[None, None],
+ })?;
+ assert!(!out.is_nullable());
+ assert_eq!(out.data_type(), &DataType::Int64);
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_xxhash64_i32() {
+ let seed = 42u64;
+ assert_eq!(
+ spark_compatible_xxhash64(1i32.to_le_bytes(), seed),
+ 0xa309b38455455929
+ );
+ assert_eq!(
+ spark_compatible_xxhash64(0i32.to_le_bytes(), seed),
+ 0x3229fbc4681e48f3
+ );
+ assert_eq!(
+ spark_compatible_xxhash64((-1i32).to_le_bytes(), seed),
+ 0x1bfdda8861c06e45
+ );
+ }
+
+ #[test]
+ fn test_xxhash64_i32_boundary() {
+ let seed = 42u64;
+ let h = spark_compatible_xxhash64(i32::MAX.to_le_bytes(), seed);
+ assert_ne!(h, seed);
+ let h = spark_compatible_xxhash64(i32::MIN.to_le_bytes(), seed);
+ assert_ne!(h, seed);
+ }
+
+ #[test]
+ fn test_xxhash64_i8() {
+ let seed = 42u64;
+ // i8 is widened to i32 before hashing
+ assert_eq!(
+ spark_compatible_xxhash64((1i8 as i32).to_le_bytes(), seed),
+ spark_compatible_xxhash64(1i32.to_le_bytes(), seed),
+ );
+ }
+
+ #[test]
+ fn test_xxhash64_i64() {
+ let seed = 42u64;
+ assert_eq!(
+ spark_compatible_xxhash64(1i64.to_le_bytes(), seed),
+ 0x9ed50fd59358d232
+ );
+ assert_eq!(
+ spark_compatible_xxhash64(0i64.to_le_bytes(), seed),
+ 0xb71b47ebda15746c
+ );
+ assert_eq!(
+ spark_compatible_xxhash64((-1i64).to_le_bytes(), seed),
+ 0x358ae035bfb46fd2
+ );
+ }
+
+ #[test]
+ fn test_xxhash64_i64_boundary() {
+ let seed = 42u64;
+ let h = spark_compatible_xxhash64(i64::MAX.to_le_bytes(), seed);
+ assert_ne!(h, seed);
+ let h = spark_compatible_xxhash64(i64::MIN.to_le_bytes(), seed);
+ assert_ne!(h, seed);
+ }
+
+ /// Spark normalizes `-0.0` to `0.0` before hashing, so both produce
+ /// the same hash. Exercise the dispatch through `create_xxhash64_hashes`
+ /// to cover the `hash_array_primitive_float!` normalization path.
+ #[test]
+ fn test_xxhash64_negative_zero_f32() {
+ use arrow::array::Float32Array;
+ let array: ArrayRef = Arc::new(Float32Array::from(vec![0.0f32, -0.0f32]));
+ let mut hashes = vec![DEFAULT_SEED; 2];
+ create_xxhash64_hashes(&[array], &mut hashes).unwrap();
+ assert_eq!(hashes[0], hashes[1]);
+ assert_eq!(hashes[0], spark_compatible_xxhash64(0i32.to_le_bytes(), 42));
+ }
+
+ #[test]
+ fn test_xxhash64_negative_zero_f64() {
+ use arrow::array::Float64Array;
+ let array: ArrayRef = Arc::new(Float64Array::from(vec![0.0f64, -0.0f64]));
+ let mut hashes = vec![DEFAULT_SEED; 2];
+ create_xxhash64_hashes(&[array], &mut hashes).unwrap();
+ assert_eq!(hashes[0], hashes[1]);
+ assert_eq!(hashes[0], spark_compatible_xxhash64(0i64.to_le_bytes(), 42));
+ }
+
+ #[test]
+ fn test_xxhash64_string() {
+ let seed = 42u64;
+ assert_eq!(spark_compatible_xxhash64("hello", seed), 0xc3629e6318d53932);
+ assert_eq!(spark_compatible_xxhash64("", seed), 0x98b1582b0977e704);
+ assert_eq!(spark_compatible_xxhash64("abc", seed), 0x13c1d910702770e6);
+ }
+
+ #[test]
+ fn test_xxhash64_string_emoji_cjk() {
+ let seed = 42u64;
+ let h1 = spark_compatible_xxhash64("😁", seed);
+ assert_ne!(h1, seed);
+ let h2 = spark_compatible_xxhash64("天地", seed);
+ assert_ne!(h2, seed);
+ assert_ne!(h1, h2);
+ }
+
+ #[test]
+ fn test_xxhash64_dictionary_string() {
+ use arrow::array::DictionaryArray;
+ use arrow::datatypes::Int32Type;
+
+ let dict_array: DictionaryArray<Int32Type> =
+ vec!["hello", "world", "abc", "hello", "world"]
+ .into_iter()
+ .collect();
+ let array_ref: ArrayRef = Arc::new(dict_array);
+
+ let mut hashes = vec![DEFAULT_SEED; 5];
+ create_xxhash64_hashes(&[array_ref], &mut hashes).unwrap();
+
+ assert_eq!(hashes[0], spark_compatible_xxhash64("hello", 42));
+ assert_eq!(hashes[1], spark_compatible_xxhash64("world", 42));
+ assert_eq!(hashes[2], spark_compatible_xxhash64("abc", 42));
+ assert_eq!(hashes[3], hashes[0]);
+ assert_eq!(hashes[4], hashes[1]);
+ }
+
+ #[test]
+ fn test_xxhash64_dictionary_int() {
+ use arrow::array::DictionaryArray;
+ use arrow::datatypes::Int32Type;
+
+ let keys = Int32Array::from(vec![0, 1, 2, 0, 1]);
+ let values = Int32Array::from(vec![100, 200, 300]);
+ let dict_array =
+ DictionaryArray::<Int32Type>::try_new(keys, Arc::new(values)).unwrap();
+ let array_ref: ArrayRef = Arc::new(dict_array);
+
+ let mut hashes = vec![DEFAULT_SEED; 5];
+ create_xxhash64_hashes(&[array_ref], &mut hashes).unwrap();
+
+ assert_eq!(
+ hashes[0],
+ spark_compatible_xxhash64(100i32.to_le_bytes(), 42)
+ );
+ assert_eq!(
+ hashes[1],
+ spark_compatible_xxhash64(200i32.to_le_bytes(), 42)
+ );
+ assert_eq!(
+ hashes[2],
+ spark_compatible_xxhash64(300i32.to_le_bytes(), 42)
+ );
+ assert_eq!(hashes[3], hashes[0]);
+ assert_eq!(hashes[4], hashes[1]);
+ }
+
+ #[test]
+ fn test_xxhash64_dictionary_with_nulls() {
+ use arrow::array::DictionaryArray;
+ use arrow::datatypes::Int32Type;
+
+ let keys = Int32Array::from(vec![Some(0), None, Some(1), Some(0), None]);
+ let values = StringArray::from(vec!["hello", "world"]);
+ let dict_array =
+ DictionaryArray::<Int32Type>::try_new(keys, Arc::new(values)).unwrap();
+ let array_ref: ArrayRef = Arc::new(dict_array);
+
+ let mut hashes = vec![DEFAULT_SEED; 5];
+ create_xxhash64_hashes(&[array_ref], &mut hashes).unwrap();
+
+ assert_eq!(hashes[0], spark_compatible_xxhash64("hello", 42));
+ assert_eq!(hashes[2], spark_compatible_xxhash64("world", 42));
+ assert_eq!(hashes[3], spark_compatible_xxhash64("hello", 42));
+ assert_eq!(hashes[1], DEFAULT_SEED);
+ assert_eq!(hashes[4], DEFAULT_SEED);
+ }
+
+ #[test]
+ fn test_xxhash64_dictionary_non_first_column() {
+ use arrow::array::DictionaryArray;
+ use arrow::datatypes::Int32Type;
+
+ let dict_array: DictionaryArray<Int32Type> =
+ vec!["hello", "world", "abc"].into_iter().collect();
+ let array_ref: ArrayRef = Arc::new(dict_array);
+
+ let mut hashes = vec![123u64, 456u64, 789u64];
+ create_xxhash64_hashes_dictionary::<Int32Type>(&array_ref, &mut hashes, false)
+ .unwrap();
+
+ assert_eq!(hashes[0], spark_compatible_xxhash64("hello", 123));
+ assert_eq!(hashes[1], spark_compatible_xxhash64("world", 456));
+ assert_eq!(hashes[2], spark_compatible_xxhash64("abc", 789));
+ }
+
+ #[test]
+ fn test_xxhash64_fixed_size_binary() {
+ let array = FixedSizeBinaryArray::from(vec![
+ Some(&[0x01, 0x02, 0x03, 0x04][..]),
+ Some(&[0x05, 0x06, 0x07, 0x08][..]),
+ None,
+ Some(&[0x00, 0x00, 0x00, 0x00][..]),
+ ]);
+ let array_ref: ArrayRef = Arc::new(array);
+
+ let mut hashes = vec![DEFAULT_SEED; 4];
+ create_xxhash64_hashes(&[array_ref], &mut hashes).unwrap();
+
+ assert_eq!(
+ hashes[0],
+ spark_compatible_xxhash64([0x01, 0x02, 0x03, 0x04], 42)
+ );
+ assert_eq!(
+ hashes[1],
+ spark_compatible_xxhash64([0x05, 0x06, 0x07, 0x08], 42)
+ );
+ assert_eq!(hashes[2], DEFAULT_SEED);
+ assert_eq!(
+ hashes[3],
+ spark_compatible_xxhash64([0x00, 0x00, 0x00, 0x00], 42)
+ );
+ }
+
+ #[test]
+ fn test_xxhash64_struct() {
+ use arrow::array::StructArray;
+ use arrow::datatypes::Field;
+
+ let int_array = Int32Array::from(vec![1, 2, 3]);
+ let str_array = StringArray::from(vec!["a", "b", "c"]);
+ let struct_array = StructArray::from(vec![
+ (
+ Arc::new(Field::new("a", DataType::Int32, false)),
+ Arc::new(int_array) as ArrayRef,
+ ),
+ (
+ Arc::new(Field::new("b", DataType::Utf8, false)),
+ Arc::new(str_array) as ArrayRef,
+ ),
+ ]);
+ let array_ref: ArrayRef = Arc::new(struct_array);
+
+ let mut hashes = vec![DEFAULT_SEED; 3];
+ create_xxhash64_hashes(&[array_ref], &mut hashes).unwrap();
+
+ for hash in &hashes {
+ assert_ne!(*hash, DEFAULT_SEED);
+ }
+ assert_ne!(hashes[0], hashes[1]);
+ assert_ne!(hashes[1], hashes[2]);
+ }
+
+ #[test]
+ fn test_xxhash64_list() {
+ use arrow::array::ListArray;
+ use arrow::buffer::OffsetBuffer;
+ use arrow::datatypes::Field;
+
+ let values = Int32Array::from(vec![1, 2, 3, 4, 5, 6]);
+ let offsets = OffsetBuffer::new(vec![0i32, 2, 3, 6].into());
+ let list_array = ListArray::new(
+ Arc::new(Field::new_list_field(DataType::Int32, false)),
+ offsets,
+ Arc::new(values),
+ None,
+ );
+ let array_ref: ArrayRef = Arc::new(list_array);
+
+ let mut hashes = vec![DEFAULT_SEED; 3];
+ create_xxhash64_hashes(&[array_ref], &mut hashes).unwrap();
+
+ for hash in &hashes {
+ assert_ne!(*hash, DEFAULT_SEED);
+ }
+ assert_ne!(hashes[0], hashes[1]);
+ }
+}
diff --git a/datafusion/sqllogictest/test_files/spark/hash/xxhash64.slt b/datafusion/sqllogictest/test_files/spark/hash/xxhash64.slt
new file mode 100644
index 0000000000..cd7d6e4947
--- /dev/null
+++ b/datafusion/sqllogictest/test_files/spark/hash/xxhash64.slt
@@ -0,0 +1,197 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Tests for Spark-compatible xxhash64
+# Reference: https://spark.apache.org/docs/latest/api/sql/index.html#xxhash64
+
+# Basic integer tests (Int32 - hashed as 4 bytes)
+query I
+SELECT xxhash64(1);
+----
+-7001672635703045582
+
+query I
+SELECT xxhash64(0);
+----
+-5252525462095825812
+
+query I
+SELECT xxhash64(-1);
+----
+3858142552250413010
+
+# Boundary values
+query I
+SELECT xxhash64(arrow_cast(2147483647, 'Int32'));
+----
+1508894993788531228
+
+query I
+SELECT xxhash64(arrow_cast(-2147483648, 'Int32'));
+----
+2073849959933241805
+
+query I
+SELECT xxhash64(arrow_cast(9223372036854775807, 'Int64'));
+----
+-3246596055638297850
+
+query I
+SELECT xxhash64(arrow_cast(-9223372036854775808, 'Int64'));
+----
+-8619748838626508300
+
+# String tests
+query I
+SELECT xxhash64('hello');
+----
+-4367754540140381902
+
+query I
+SELECT xxhash64('');
+----
+-7444071767201028348
+
+query I
+SELECT xxhash64('abc');
+----
+1423657621850124518
+
+# Emoji and CJK strings
+query I
+SELECT xxhash64('😁');
+----
+-6337236088984028203
+
+query I
+SELECT xxhash64('天地');
+----
+-235771157374669727
+
+# NULL handling - nulls don't change the hash (keeps seed value 42)
+query I
+SELECT xxhash64(NULL);
+----
+42
+
+# Multiple column hashing
+query I
+SELECT xxhash64(1, 2);
+----
+-8198029865082835910
+
+query I
+SELECT xxhash64('a', 'b', 'c');
+----
+-8775012835737190202
+
+query I
+SELECT xxhash64(1, 'hello');
+----
+-7340167327466467369
+
+# Int64 tests
+query I
+SELECT xxhash64(arrow_cast(1, 'Int64'));
+----
+-7001672635703045582
+
+query I
+SELECT xxhash64(arrow_cast(0, 'Int64'));
+----
+-5252525462095825812
+
+# Float tests - note: -0.0 and 0.0 hash to the same value in Spark
+query I
+SELECT xxhash64(1.0);
+----
+-2162451265447482029
+
+query I
+SELECT xxhash64(0.0);
+----
+-5252525462095825812
+
+query I
+SELECT xxhash64(-0.0);
+----
+-5252525462095825812
+
+# Float edge cases
+query I
+SELECT xxhash64(-1.0);
+----
+-8296228499959607521
+
+query I
+SELECT xxhash64(arrow_cast(1.0E20, 'Float64'));
+----
+-1089066926670605926
+
+# Boolean tests
+query I
+SELECT xxhash64(true);
+----
+-6698625589789238999
+
+query I
+SELECT xxhash64(false);
+----
+3614696996920510707
+
+# Binary tests
+query I
+SELECT xxhash64(arrow_cast('hello', 'Binary'));
+----
+-4367754540140381902
+
+# Date tests
+query I
+SELECT xxhash64(arrow_cast('2023-01-15', 'Date32'));
+----
+290591839883576215
+
+# Struct hashing
+query I
+SELECT xxhash64(named_struct('a', 1, 'b', 2));
+----
+-8198029865082835910
+
+# Array/List hashing - Spark chains hashes across list elements:
+# hash(elem_n, hash(elem_n-1, ... hash(elem_0, seed) ...))
+query I
+SELECT xxhash64(arrow_cast(make_array(1, 2, 3), 'List(Int32)'));
+----
+8592097078962733837
+
+# Test with table data
+statement ok
+CREATE TABLE xxhash_test AS SELECT * FROM (VALUES
+ (1, 'a'),
+ (2, 'b'),
+ (3, 'c')
+) AS t(id, name);
+
+query II
+SELECT id, xxhash64(id, name) FROM xxhash_test ORDER BY id;
+----
+1 -8527884376001292090
+2 -7687314219880041104
+3 -7773406846575579340
+
+statement ok
+DROP TABLE xxhash_test;