Skip to content

Apache DataFusion SQL Query Engine

perf: cache dictionary arc pointer (#24418)

RIchard Baah66a901f68ddf
diff --git a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/dictionary.rs b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/dictionary.rs
index 4dd488380a..8ec8c808b5 100644
--- a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/dictionary.rs
+++ b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/dictionary.rs
@@ -57,6 +57,9 @@ pub struct DictionaryGroupValuesColumn<K: ArrowDictionaryKeyType + Send + Sync>
val_to_inner: Vec<usize>,
/// Reusable hash buffer for the dictionary values array.
val_hashes: Vec<u64>,
+ /// The last `dict.values()` Arc hashed in `append_val`. When the incoming
+ /// values array is `ptr_eq` to this, `val_hashes` can be reused directly.
+ cached_values: Option<ArrayRef>,
_phantom: PhantomData<K>,
}
@@ -73,6 +76,7 @@ impl<K: ArrowDictionaryKeyType + Send + Sync> DictionaryGroupValuesColumn<K> {
random_state: AGGREGATION_HASH_SEED,
val_to_inner: Vec::default(),
val_hashes: Vec::default(),
+ cached_values: None,
_phantom: PhantomData,
}
}
@@ -159,6 +163,7 @@ impl<K: ArrowDictionaryKeyType + Send + Sync> DictionaryGroupValuesColumn<K> {
}
fn hash_values(&mut self, values: &ArrayRef) {
+ self.cached_values = None;
self.val_hashes.clear();
self.val_hashes.resize(values.len(), 0);
create_hashes(
@@ -296,9 +301,25 @@ impl<K: ArrowDictionaryKeyType + Send + Sync> GroupColumn
}
Some(val_idx) => {
let dict_values = dict.values();
- let single = dict_values.slice(val_idx, 1);
- self.hash_values(&single);
- self.find_or_insert_value(dict_values, val_idx, self.val_hashes[0])?
+ // check if the dictionary values array we are hashing was already seen.
+ // if its arc was already stored we dont need to rehash the entire array again
+ // if its new hash the entire array and store an arc ptr for future use
+ let cache_hit = self
+ .cached_values
+ .as_ref()
+ .is_some_and(|c| Arc::ptr_eq(c, dict_values));
+ if !cache_hit {
+ self.val_hashes.clear();
+ self.val_hashes.resize(dict_values.len(), 0);
+ create_hashes(
+ std::slice::from_ref(dict_values),
+ &self.random_state,
+ &mut self.val_hashes,
+ )
+ .unwrap();
+ self.cached_values = Some(Arc::clone(dict_values));
+ }
+ self.find_or_insert_value(dict_values, val_idx, self.val_hashes[val_idx])?
}
};
self.group_to_inner.push(inner_slot);