Skip to content

Apache DataFusion SQL Query Engine

fix: keep the running hash for nested dictionaries in Spark xxhash64 (#25813)

Varun03204c0c8412
diff --git a/datafusion/spark/src/function/hash/utils.rs b/datafusion/spark/src/function/hash/utils.rs
index 68ee9e8337..7b6f520ea4 100644
--- a/datafusion/spark/src/function/hash/utils.rs
+++ b/datafusion/spark/src/function/hash/utils.rs
@@ -898,7 +898,14 @@ macro_rules! create_hashes_internal {
DataType::Decimal128(_, _) => {
$crate::hash_array_decimal!(Decimal128Array, col, $hashes_buffer, $hash_method);
}
- DataType::Dictionary(index_type, _) => match **index_type {
+ DataType::Dictionary(index_type, _) => {
+ // Only use the fast path if every row starts from the same hash
+ let first_col = first_col
+ && match $hashes_buffer.first() {
+ None => true,
+ Some(first) => $hashes_buffer.iter().all(|h| h == first),
+ };
+ match **index_type {
DataType::Int8 => {
$create_dictionary_hash_method::<Int8Type>(col, $hashes_buffer, first_col)?;
}
@@ -957,7 +964,8 @@ macro_rules! create_hashes_internal {
col.data_type(),
)))
}
- },
+ }
+ }
DataType::List(field) => {
let list_array = col.as_any().downcast_ref::<ListArray>().unwrap();
let values = list_array.values();
diff --git a/datafusion/spark/src/function/hash/xxhash64.rs b/datafusion/spark/src/function/hash/xxhash64.rs
index d1d62d6e4a..c15e793cda 100644
--- a/datafusion/spark/src/function/hash/xxhash64.rs
+++ b/datafusion/spark/src/function/hash/xxhash64.rs
@@ -105,17 +105,19 @@ fn spark_compatible_xxhash64<T: AsRef<[u8]>>(data: T, seed: u64) -> u64 {
fn create_xxhash64_hashes_dictionary<K: ArrowDictionaryKeyType>(
array: &ArrayRef,
hashes_buffer: &mut [u64],
- first_col: bool,
+ seeds_are_pristine: bool,
) -> Result<()> {
let dict_array = array.as_any().downcast_ref::<DictionaryArray<K>>().unwrap();
- if !first_col {
+ if !seeds_are_pristine {
let unpacked = take(dict_array.values().as_ref(), dict_array.keys(), None)?;
create_xxhash64_hashes(&[unpacked], hashes_buffer)?;
} else {
// Hash each dictionary value once, then look up by key. This avoids
// redundant hashing of large dictionary entries (e.g. long strings).
let dict_values = Arc::clone(dict_array.values());
- let mut dict_hashes = vec![DEFAULT_SEED; dict_values.len()];
+ // Every row has the same running hash here, so start from it instead of 42
+ let seed = hashes_buffer.first().copied().unwrap_or(DEFAULT_SEED);
+ let mut dict_hashes = vec![seed; dict_values.len()];
create_xxhash64_hashes(&[dict_values], &mut dict_hashes)?;
for (hash, key) in hashes_buffer.iter_mut().zip(dict_array.keys().iter()) {
@@ -343,6 +345,61 @@ mod tests {
assert_eq!(hashes[4], DEFAULT_SEED);
}
+ /// A dictionary inside a list keeps the running hash instead of restarting from 42
+ #[test]
+ fn test_dictionary_element_in_list_matches_decoded() {
+ use arrow::array::ListArray;
+ use arrow::buffer::OffsetBuffer;
+ use arrow::datatypes::Int8Type;
+
+ let values: ArrayRef = Arc::new(Int32Array::from(vec![10, 20]));
+ let keys = arrow::array::Int8Array::from(vec![0i8, 1]);
+ let dict: ArrayRef =
+ Arc::new(DictionaryArray::<Int8Type>::try_new(keys, values).unwrap());
+ let decoded: ArrayRef = Arc::new(Int32Array::from(vec![10, 20]));
+
+ let as_list = |elems: ArrayRef| -> ArrayRef {
+ Arc::new(ListArray::new(
+ Arc::new(Field::new("item", elems.data_type().clone(), true)),
+ OffsetBuffer::new(vec![0i32, 2].into()),
+ elems,
+ None,
+ ))
+ };
+
+ let mut from_dict = vec![DEFAULT_SEED; 1];
+ create_xxhash64_hashes(&[as_list(dict)], &mut from_dict).unwrap();
+ let mut from_decoded = vec![DEFAULT_SEED; 1];
+ create_xxhash64_hashes(&[as_list(decoded)], &mut from_decoded).unwrap();
+ assert_eq!(from_dict, from_decoded);
+ }
+
+ /// Rows with different running hashes skip the fast path
+ #[test]
+ fn test_dictionary_with_nonuniform_seeds_matches_decoded() {
+ use arrow::datatypes::Int8Type;
+
+ let values: ArrayRef = Arc::new(Int32Array::from(vec![Some(10), Some(20), None]));
+ let keys =
+ arrow::array::Int8Array::from(vec![Some(0), Some(1), Some(2), None, Some(0)]);
+ let dict: ArrayRef =
+ Arc::new(DictionaryArray::<Int8Type>::try_new(keys, values).unwrap());
+ let decoded: ArrayRef = Arc::new(Int32Array::from(vec![
+ Some(10),
+ Some(20),
+ None,
+ None,
+ Some(10),
+ ]));
+
+ let seeds: Vec<u64> = vec![7, 38, 69, 100, 131];
+ let mut from_dict = seeds.clone();
+ create_xxhash64_hashes(&[dict], &mut from_dict).unwrap();
+ let mut from_decoded = seeds;
+ create_xxhash64_hashes(&[decoded], &mut from_decoded).unwrap();
+ assert_eq!(from_dict, from_decoded);
+ }
+
#[test]
fn test_xxhash64_dictionary_non_first_column() {
use arrow::array::DictionaryArray;