Apache DataFusion SQL Query Engine
perf: reuse cached dictionary value slots in vectorized_equal_to (#25652)
Yu-Chuan Hung622354ff2f40
diff --git a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/dictionary.rs b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/dictionary.rsindex 45a7c15bfd..b9547079aa 100644--- a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/dictionary.rs+++ b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/dictionary.rs@@ -232,6 +232,42 @@ impl<K: ArrowDictionaryKeyType + Send + Sync> DictionaryGroupValuesColumn<K> {}}+ /// Compares the value at `val_idx` against `inner[lhs_slot]` directly,+ /// memorizing `lhs_slot` in `val_to_inner` when they are equal.+ ///+ /// Each distinct value occupies exactly one slot in `inner`, so an equal+ /// comparison proves `lhs_slot` is the slot for `val_idx` without probing+ /// `value_dedup` or hashing. An unequal result is deliberately not+ /// memoized: it says which slot the value is *not* in, not which one it+ /// is. That is what keeps `val_to_inner` fill-only and never stale.+ ///+ /// Callers must have run [`Self::sync_value_cache`] for `dict_values`.+ ///+ /// Only called on a `val_to_inner` miss. It takes the fields it needs+ /// rather than `&mut self`, so the caller's hot loop can keep its slices+ /// of `val_to_inner` and `group_to_inner` in registers across the call.+ #[cold]+ #[inline(never)]+ fn equal_to_uncached(+ val_to_inner: &mut [usize],+ inner: &dyn GroupColumn,+ null_array: &ArrayRef,+ dict_values: &ArrayRef,+ val_idx: usize,+ lhs_slot: usize,+ ) -> bool {+ let equal = if dict_values.is_null(val_idx) {+ inner.equal_to(lhs_slot, null_array, 0)+ } else {+ inner.equal_to(lhs_slot, dict_values, val_idx)+ };++ if equal {+ val_to_inner[val_idx] = lhs_slot;+ }+ equal+ }+fn find_or_insert_null(&mut self) -> Result<usize> {if let Some(slot) = self.null_inner_slot {return Ok(slot);@@ -242,38 +278,13 @@ impl<K: ArrowDictionaryKeyType + Send + Sync> DictionaryGroupValuesColumn<K> {Ok(slot)}- fn build_lookup_table(- &self,- dict_values: &ArrayRef,- val_hashes: &[u64],- ) -> Vec<usize> {- let num_distinct = dict_values.len();- let mut table = vec![usize::MAX; num_distinct + 1];- let inner = &*self.inner;- for val_idx in 0..num_distinct {- if dict_values.is_null(val_idx) {- table[val_idx] = self.null_inner_slot.unwrap_or(usize::MAX);- } else {- let hash = val_hashes[val_idx];- if let Some(&(_, slot)) =- self.value_dedup.find(hash, |&(entry_hash, slot)| {- entry_hash == hash && inner.equal_to(slot, dict_values, val_idx)- })- {- table[val_idx] = slot;- }- }- }- table[num_distinct] = self.null_inner_slot.unwrap_or(usize::MAX);- table- }-- /// Per-row fallback for `vectorized_equal_to` used when the number of rows- /// to check is smaller than the dictionary cardinality, making the O(D)- /// lookup-table build more expensive than direct value comparison.+ /// Per-row fallback for `vectorized_equal_to` used when the values array is+ /// not cached and the number of rows to check is smaller than the+ /// dictionary cardinality, making the O(D) hashing in `sync_value_cache`+ /// more expensive than direct value comparison.////// `#[cold]` + `#[inline(never)]` keeps this code out of the hot- /// lookup-table loops in `vectorized_equal_to` so LLVM can pipeline them.+ /// `val_to_inner` loops in `vectorized_equal_to` so LLVM can pipeline them.#[cold]#[inline(never)]fn equal_to_per_row(@@ -348,12 +359,12 @@ impl<K: ArrowDictionaryKeyType + Send + Sync> GroupColumnlet dict = array.as_dictionary::<K>();let dict_keys = dict.keys();let dict_values = dict.values();- let num_distinct = dict_values.len();- // The fallback is in a separate #[cold] function so its code does not- // appear inline here and cannot prevent LLVM from pipelining / unrolling- // the hot lookup-table loops below.- if rhs_rows.len() < num_distinct {+ let cached = self+ .cached_values+ .as_ref()+ .is_some_and(|c| Arc::ptr_eq(c, dict_values));+ if !cached && rhs_rows.len() < dict_values.len() {self.equal_to_per_row(lhs_rows,dict_values,@@ -363,44 +374,64 @@ impl<K: ArrowDictionaryKeyType + Send + Sync> GroupColumn);return;}+ self.sync_value_cache(dict_values);- let mut val_hashes = vec![0u64; dict_values.len()];- create_hashes(- std::slice::from_ref(dict_values),- &self.random_state,- &mut val_hashes,- )- .unwrap();- let lookup = self.build_lookup_table(dict_values, &val_hashes);-+ let raw_keys = dict_keys.values();let group_to_inner = self.group_to_inner.as_slice();-+ let val_to_inner = self.val_to_inner.as_mut_slice();+ let inner = &*self.inner;+ let null_array = &self.null_array;+ let null_inner_slot = self.null_inner_slot;if dict_keys.null_count() == 0 {// No null keys : skip the get_bit guard: we only ever write false,// so overwriting an already-false bit is a no-op.- let raw_keys = dict_keys.values();for (idx, (&lhs_row, &rhs_row)) inlhs_rows.iter().zip(rhs_rows.iter()).enumerate(){- let rhs_slot = lookup[raw_keys[rhs_row].as_usize()];- if rhs_slot == usize::MAX || group_to_inner[lhs_row] != rhs_slot {+ let val_idx = raw_keys[rhs_row].as_usize();+ let lhs_slot = group_to_inner[lhs_row];+ let equal = match val_to_inner[val_idx] {+ usize::MAX => Self::equal_to_uncached(+ val_to_inner,+ inner,+ null_array,+ dict_values,+ val_idx,+ lhs_slot,+ ),+ slot => slot == lhs_slot,+ };+ if !equal {equal_to_results.set_bit(idx, false);}}} else {let null_buf = dict_keys.nulls().unwrap();- let raw_keys = dict_keys.values();for (idx, (&lhs_row, &rhs_row)) inlhs_rows.iter().zip(rhs_rows.iter()).enumerate(){if equal_to_results.get_bit(idx) {- let val_idx = if null_buf.is_null(rhs_row) {- num_distinct+ // A null key is not a position in the values array, so it+ // resolves straight to the null slot instead of going+ // through `val_to_inner`.+ let lhs_slot = group_to_inner[lhs_row];+ let equal = if null_buf.is_null(rhs_row) {+ null_inner_slot == Some(lhs_slot)} else {- raw_keys[rhs_row].as_usize()+ let val_idx = raw_keys[rhs_row].as_usize();+ match val_to_inner[val_idx] {+ usize::MAX => Self::equal_to_uncached(+ val_to_inner,+ inner,+ null_array,+ dict_values,+ val_idx,+ lhs_slot,+ ),+ slot => slot == lhs_slot,+ }};- let rhs_slot = lookup[val_idx];- if rhs_slot == usize::MAX || group_to_inner[lhs_row] != rhs_slot {+ if !equal {equal_to_results.set_bit(idx, false);}}@@ -1060,4 +1091,101 @@ mod tests {assert_eq!(col.inner.len(), 2, "'a' must not be duplicated");assert_eq!(col.len(), 3);}++ /// On a cache miss `vectorized_equal_to` compares against the group's own+ /// slot: an equal result proves the slot and is cached, an unequal one+ /// only rules that slot out and must leave the entry unresolved.+ #[test]+ fn vectorized_equal_to_caches_only_equal_results() {+ let mut col = utf8_col();+ col.vectorized_append(+ &i32_dict(&[Some(0), Some(1)], &[Some("a"), Some("b")]),+ &[0, 1],+ )+ .unwrap();+ let (group_a, group_b) = (0, 1);+ let slot_a = col.group_to_inner[group_a];+ let slot_b = col.group_to_inner[group_b];++ // A new values array in the opposite order, so `val_idx` != slot.+ // Two rows for two values keeps the per-row fallback out of the way.+ let (idx_b, idx_a) = (0, 1);+ let values: ArrayRef = Arc::new(StringArray::from(vec![Some("b"), Some("a")]));+ let a_then_b =+ dict_with_values(&[Some(idx_a as i32), Some(idx_b as i32)], &values);++ // "a" vs group a is equal, "b" vs group a is not.+ let mut buf = all_true(2);+ col.vectorized_equal_to(&[group_a, group_a], &a_then_b, &[0, 1], &mut buf);+ assert_eq!(bool_vec(&buf), vec![true, false]);+ assert_eq!(col.val_to_inner[idx_a], slot_a);+ assert_eq!(+ col.val_to_inner[idx_b],+ usize::MAX,+ "unequal result was cached"+ );++ // Later, "b" vs group b fills the entry that was left unresolved.+ let only_b = dict_with_values(&[Some(idx_b as i32)], &values);+ let mut buf = all_true(1);+ col.vectorized_equal_to(&[group_b], &only_b, &[0], &mut buf);+ assert_eq!(bool_vec(&buf), vec![true]);+ assert_eq!(col.val_to_inner[idx_b], slot_b);+ }++ /// A non-null key pointing at a null value and a null key must both+ /// compare equal to the null group, and only the former goes through+ /// `val_to_inner`.+ #[test]+ fn vectorized_equal_to_matches_null_value_and_null_key_to_null_group() {+ let mut col = utf8_col();+ col.vectorized_append(&i32_dict(&[None, Some(0)], &[Some("a")]), &[0, 1])+ .unwrap();+ let (null_group, group_a) = (0, 1);+ let null_slot = col.null_inner_slot.unwrap();++ let idx_null = 0;+ let values: ArrayRef = Arc::new(StringArray::from(vec![None, Some("a")]));+ let batch = dict_with_values(+ &[Some(idx_null as i32), Some(idx_null as i32), None],+ &values,+ );++ // null value vs null group, null value vs group a, null key vs null group.+ let mut buf = all_true(3);+ col.vectorized_equal_to(+ &[null_group, group_a, null_group],+ &batch,+ &[0, 1, 2],+ &mut buf,+ );+ assert_eq!(bool_vec(&buf), vec![true, false, true]);+ assert_eq!(col.val_to_inner[idx_null], null_slot);+ }++ /// Fewer rows than values in an uncached values array takes the per-row+ /// fallback, which must not rebuild the cache for that array.+ #[test]+ fn vectorized_equal_to_small_batch_of_new_values_skips_cache_rebuild() {+ let old_values: ArrayRef = Arc::new(StringArray::from(vec![Some("a")]));+ let mut col = utf8_col();+ col.vectorized_append(&dict_with_values(&[Some(0)], &old_values), &[0])+ .unwrap();+ let group_a = 0;++ // One row against three values in an array the cache has not seen.+ let new_values: ArrayRef =+ Arc::new(StringArray::from(vec![Some("x"), Some("a"), Some("y")]));+ let idx_a = 1;+ let only_a = dict_with_values(&[Some(idx_a)], &new_values);+ let mut buf = all_true(1);+ col.vectorized_equal_to(&[group_a], &only_a, &[0], &mut buf);+ assert_eq!(bool_vec(&buf), vec![true]);+ assert!(+ col.cached_values+ .as_ref()+ .is_some_and(|c| Arc::ptr_eq(c, &old_values)),+ "fallback must leave the cache on the previous values array"+ );+ }}