Apache DataFusion SQL Query Engine
perf: hash only the selected values of union arrays (#25793)
Abhishek2566bc1052b3
diff --git a/datafusion/common/benches/with_hashes.rs b/datafusion/common/benches/with_hashes.rsindex 0e9c53c896..ea75b32bdd 100644--- a/datafusion/common/benches/with_hashes.rs+++ b/datafusion/common/benches/with_hashes.rs@@ -94,6 +94,24 @@ fn criterion_benchmark(c: &mut Criterion) {array: sparse_union_array(BATCH_SIZE),supports_nulls: false,},+ BenchData {+ name: "sparse_union (2 types)",+ array: sparse_union_of(+ (0..2)+ .map(|_| primitive_array::<Int64Type>(BATCH_SIZE))+ .collect(),+ ),+ supports_nulls: false,+ },+ BenchData {+ name: "sparse_union (utf8)",+ array: sparse_union_of(+ (0..5)+ .map(|_| pool.string_array::<i32>(BATCH_SIZE))+ .collect(),+ ),+ supports_nulls: false,+ },BenchData {name: "dense_union",array: dense_union_array(BATCH_SIZE),@@ -354,6 +372,24 @@ fn sliced_array_benchmark(c: &mut Criterion) {},);}++ // Sliced Dense UnionArray+ {+ let full_array = dense_union_array(total_rows);+ let sliced: ArrayRef = Arc::new(+ full_array+ .as_any()+ .downcast_ref::<UnionArray>()+ .unwrap()+ .slice(slice_offset, slice_len),+ );+ c.bench_function(+ &format!("dense_union_sliced: 1/{ratio} of {total_rows} rows"),+ |b| {+ do_hash_test_with_len(b, std::slice::from_ref(&sliced), slice_len);+ },+ );+ }}}@@ -430,23 +466,25 @@ fn map_array(num_rows: usize) -> ArrayRef {}fn sparse_union_array(num_rows: usize) -> ArrayRef {+ sparse_union_of(+ (0..5)+ .map(|_| primitive_array::<Int64Type>(num_rows))+ .collect(),+ )+}++/// Sparse union whose rows each select a random child+fn sparse_union_of(children: Vec<ArrayRef>) -> ArrayRef {let mut rng = make_rng();- let num_types = 5;+ let num_types = children.len() as i32;- let type_ids: Vec<i8> = (0..num_rows)+ let type_ids: Vec<i8> = (0..children[0].len()).map(|_| rng.random_range(0..num_types) as i8).collect();- let (fields, children): (Vec<_>, Vec<_>) = (0..num_types)- .map(|i| {- (- (- i as i8,- Arc::new(Field::new(format!("f{i}"), DataType::Int64, true)),- ),- primitive_array::<Int64Type>(num_rows),- )- })- .unzip();+ let fields = children.iter().enumerate().map(|(i, child)| {+ let field = Field::new(format!("f{i}"), child.data_type().clone(), true);+ (i as i8, Arc::new(field))+ });Arc::new(UnionArray::try_new(diff --git a/datafusion/common/src/hash_utils.rs b/datafusion/common/src/hash_utils.rsindex 8da165e8fb..a51278b455 100644--- a/datafusion/common/src/hash_utils.rs+++ b/datafusion/common/src/hash_utils.rs@@ -20,15 +20,15 @@use arrow::array::types::{IntervalDayTime, IntervalMonthDayNano};use arrow::array::*;#[cfg(not(feature = "force_hash_collisions"))]-use arrow::compute::take;+use arrow::buffer::ScalarBuffer;+#[cfg(not(feature = "force_hash_collisions"))]+use arrow::compute::{nullif, take};use arrow::datatypes::*;#[cfg(not(feature = "force_hash_collisions"))]use arrow::{downcast_dictionary_array, downcast_primitive_array};use foldhash::fast::FixedState;#[cfg(not(feature = "force_hash_collisions"))]use itertools::Itertools;-#[cfg(not(feature = "force_hash_collisions"))]-use std::collections::HashMap;use std::hash::{BuildHasher, Hash, Hasher};/// [`RandomState`] is optimized for speed and suitable for hash tables and@@ -826,67 +826,61 @@ fn hash_union_array(};if array.is_dense() {- // Dense union: children only contain values of their type, so they're already compact.- // Use the default hashing approach which is efficient for dense unions.- hash_union_array_default(array, union_fields, child_hashing, hashes_buffer)+ hash_dense_union_array(array, union_fields, child_hashing, hashes_buffer)} else {- // Sparse union: each child has the same length as the union array.- // Optimization: only hash the elements that are actually referenced by type_ids,- // instead of hashing all K*N elements (where K = num types, N = array length).hash_sparse_union_array(array, union_fields, child_hashing, hashes_buffer)}}-/// Default hashing for union arrays - hashes all elements of each child array fully.-///-/// This approach works for both dense and sparse union arrays:-/// - Dense unions: children are compact (each child only contains values of that type)-/// - Sparse unions: children have the same length as the union array-///-/// For sparse unions with 3+ types, the optimized take/scatter approach in-/// `hash_sparse_union_array` is more efficient, but for 1-2 types or dense unions,-/// this simpler approach is preferred.+/// Slicing a dense union does not slice its children, so when their total length+/// exceeds the union's, only the span its offsets reference is hashed.#[cfg(not(feature = "force_hash_collisions"))]-fn hash_union_array_default(+fn hash_dense_union_array(array: &UnionArray,union_fields: &UnionFields,child_hashing: &impl ChildHashing,hashes_buffer: &mut [u64],) -> Result<()> {- let mut child_hashes: HashMap<i8, Vec<u64>> =- HashMap::with_capacity(union_fields.len());-- // Hash each child array fully- for (type_id, _field) in union_fields.iter() {- let child = array.child(type_id);- let mut child_hash_buffer = vec![0; child.len()];- child_hashing.create_hashes([child], &mut child_hash_buffer)?;-- child_hashes.insert(type_id, child_hash_buffer);+ let type_ids = array.type_ids();+ let offsets = array.offsets().expect("dense union has offsets");++ // `[start, end)` of each child to hash, indexed by type id+ let mut spans = [(0, 0); 128];+ for (type_id, _) in union_fields.iter() {+ spans[type_id as usize].1 = array.child(type_id).len();+ }+ if spans.iter().map(|(_, end)| end).sum::<usize>() > array.len() {+ spans = [(usize::MAX, 0); 128];+ for (&type_id, &offset) in type_ids.iter().zip(offsets.iter()) {+ let (start, end) = &mut spans[type_id as usize];+ *start = (*start).min(offset as usize);+ *end = (*end).max(offset as usize + 1);+ }}- // Combine hashes for each row using the appropriate child offset- // For dense unions: value_offset points to the actual position in the child- // For sparse unions: value_offset equals the row index- #[expect(clippy::needless_range_loop)]- for i in 0..array.len() {- let type_id = array.type_id(i);- let child_offset = array.value_offset(i);+ let mut child_hashes = vec![vec![]; spans.len()];+ for (type_id, _) in union_fields.iter() {+ let (start, end) = spans[type_id as usize];+ if start < end {+ let child = array.child(type_id).slice(start, end - start);+ let hashes = &mut child_hashes[type_id as usize];+ hashes.resize(end - start, 0);+ child_hashing.create_hashes([child], hashes)?;+ }+ }- let child_hash = child_hashes.get(&type_id).expect("invalid type_id");- hashes_buffer[i] = combine_hashes(hashes_buffer[i], child_hash[child_offset]);+ for ((hash, &type_id), &offset) in hashes_buffer.iter_mut().zip(type_ids).zip(offsets)+ {+ let start = spans[type_id as usize].0;+ let child_hash = child_hashes[type_id as usize][offset as usize - start];+ *hash = combine_hashes(*hash, child_hash);}Ok(())}-/// Hash a sparse union array.-/// Sparse unions have child arrays with the same length as the union array.-/// For 3+ types, we optimize by only hashing the N elements that are actually used-/// (via take/scatter), instead of hashing all K*N elements.-///-/// For 1-2 types, the overhead of take/scatter outweighs the benefit, so we use-/// the default approach of hashing all children (same as dense unions).+/// Each row of a sparse union uses only one child, so each child is hashed+/// only at the rows that select it.#[cfg(not(feature = "force_hash_collisions"))]fn hash_sparse_union_array(array: &UnionArray,@@ -894,55 +888,84 @@ fn hash_sparse_union_array(child_hashing: &impl ChildHashing,hashes_buffer: &mut [u64],) -> Result<()> {- use std::collections::HashMap;-- // For 1-2 types, the take/scatter overhead isn't worth it.- // Fall back to the default approach (same as dense union).- if union_fields.len() <= 2 {- return hash_union_array_default(- array,- union_fields,- child_hashing,- hashes_buffer,- );- }-let type_ids = array.type_ids();+ let len = array.len();- // Group indices by type_id- let mut indices_by_type: HashMap<i8, Vec<u32>> = HashMap::new();- for (i, &type_id) in type_ids.iter().enumerate() {- indices_by_type.entry(type_id).or_default().push(i as u32);- }-- // For each type, extract only the needed elements, hash them, and scatter back- for (type_id, _field) in union_fields.iter() {- if let Some(indices) = indices_by_type.get(&type_id) {- if indices.is_empty() {+ // Rows the child hashing skips keep a hash of 0+ let mut child_hashes = vec![0; len];+ if let Some(&type_id) = type_ids.first()+ && type_ids.iter().all(|&t| t == type_id)+ {+ child_hashing.create_hashes([array.child(type_id)], &mut child_hashes)?;+ } else {+ let (starts, rows) = group_rows_by_type_id(type_ids);+ for (type_id, _) in union_fields.iter() {+ let (start, end) = (starts[type_id as usize], starts[type_id as usize + 1]);+ if start == end {continue;}let child = array.child(type_id);- let indices_array = UInt32Array::from(indices.clone());-- // Extract only the elements we need using take()- let filtered = take(child.as_ref(), &indices_array, None)?;-- // Hash the filtered array- let mut filtered_hashes = vec![0u64; filtered.len()];- child_hashing.create_hashes([&filtered], &mut filtered_hashes)?;-- // Scatter hashes back to correct positions- for (hash, &idx) in filtered_hashes.iter().zip(indices.iter()) {- hashes_buffer[idx as usize] =- combine_hashes(hashes_buffer[idx as usize], *hash);+ let variable_width = matches!(+ child.data_type(),+ DataType::Utf8+ | DataType::LargeUtf8+ | DataType::Binary+ | DataType::LargeBinary+ );+ // Masking the other rows as null (skipped by hashing) avoids copying+ // variable-width values but costs O(len) per child, so `take` wins for+ // fixed-width values and for children selecting few rows+ if variable_width && (end - start) * 32 >= len {+ let mut other_rows = BooleanBufferBuilder::new(len);+ other_rows.append_n(len, true);+ for &row in &rows[start..end] {+ other_rows.set_bit(row as usize, false);+ }+ let masked =+ nullif(child, &BooleanArray::new(other_rows.finish(), None))?;+ child_hashing.create_hashes([masked], &mut child_hashes)?;+ } else {+ let indices = UInt32Array::new(rows.slice(start, end - start), None);+ let mut taken_hashes = vec![0; indices.len()];+ child_hashing+ .create_hashes([take(child, &indices, None)?], &mut taken_hashes)?;+ for (&row, hash) in indices.values().iter().zip(taken_hashes) {+ child_hashes[row as usize] = hash;+ }}}}+ for (hash, child_hash) in hashes_buffer.iter_mut().zip(child_hashes) {+ *hash = combine_hashes(*hash, child_hash);+ }+Ok(())}+/// Counting sort of row indices by type id: the rows of `type_id` are+/// `rows[starts[type_id]..starts[type_id + 1]]`+#[cfg(not(feature = "force_hash_collisions"))]+fn group_rows_by_type_id(type_ids: &[i8]) -> ([usize; 129], ScalarBuffer<u32>) {+ let mut starts = [0; 129];+ for &type_id in type_ids {+ starts[type_id as usize + 1] += 1;+ }+ let mut total = 0;+ for start in &mut starts {+ total += *start;+ *start = total;+ }+ let mut next = starts;+ let mut rows = vec![0; type_ids.len()];+ for (row, &type_id) in type_ids.iter().enumerate() {+ rows[next[type_id as usize]] = row as u32;+ next[type_id as usize] += 1;+ }+ (starts, rows.into())+}+#[cfg(not(feature = "force_hash_collisions"))]fn hash_fixed_list_array(array: &FixedSizeListArray,@@ -2406,6 +2429,102 @@ mod tests {assert_eq!(hashes[0], hashes[4]);}+ #[test]+ #[cfg(not(feature = "force_hash_collisions"))]+ fn create_hashes_for_union_arrays_match_selected_values() {+ // In sparse unions both string children are masked, the others use `take`+ let children = |lens: [usize; 5]| -> Vec<ArrayRef> {+ let ints = |len| {+ Int32Array::from_iter((0..len as i32).map(|v| (v % 3 != 1).then_some(v)))+ };+ let views = (0..lens[0]).map(|v| format!("longer than inline {}", v % 2));+ let strings = (0..lens[1]).map(|v| (v % 3 != 1).then(|| format!("s{v}")));+ let large = (0..lens[4]).map(|v| (v % 2 == 0).then(|| format!("l{v}")));+ vec![+ Arc::new(StringViewArray::from_iter_values(views)),+ Arc::new(StringArray::from_iter(strings)),+ Arc::new(StructArray::from(vec![(+ Arc::new(Field::new("i", DataType::Int32, true)),+ Arc::new(ints(lens[2])) as ArrayRef,+ )])),+ Arc::new(NullArray::new(lens[3])),+ Arc::new(LargeStringArray::from_iter(large)),+ ]+ };+ let fields: UnionFields = [0, 3, 5, 7, 9]+ .into_iter()+ .zip(children([0; 5]))+ .map(|(id, c)| (id, Arc::new(Field::new("", c.data_type().clone(), true))))+ .collect();+ let type_ids = ScalarBuffer::from(vec![0, 3, 9, 0, 0, 0, 5, 7, 3, 9]);+ let offsets = ScalarBuffer::from(vec![0, 0, 0, 1, 2, 3, 0, 0, 1, 1]);+ let sparse = UnionArray::try_new(+ fields.clone(),+ type_ids.clone(),+ None,+ children([10; 5]),+ )+ .unwrap();+ // Utf8 selecting too few rows to mask is hashed through `take` instead+ let few_strings = (0..40).map(|i| if i == 20 { 3 } else { [0, 5, 7][i % 3] });+ let sparse_few_strings = UnionArray::try_new(+ fields.clone(),+ few_strings.collect(),+ None,+ children([40; 5]),+ )+ .unwrap();+ let dense = UnionArray::try_new(+ fields,+ type_ids,+ Some(offsets),+ children([4, 2, 1, 1, 2]),+ )+ .unwrap();++ for union in [+ sparse.clone(),+ sparse.slice(1, 6),+ sparse.slice(3, 3), // every row selects the same child+ sparse_few_strings,+ dense.clone(),+ dense.slice(2, 4),+ ] {+ assert_union_hashes_match_selected_values(&union, |arrays, hashes| {+ let random_state = RandomState::with_seed(0);+ create_hashes(arrays.iter().copied(), &random_state, hashes).unwrap();+ });+ assert_union_hashes_match_selected_values(&union, |arrays, hashes| {+ let hasher = BuildHasherDefault::<TestHasher>::default();+ create_hashes_with_hasher(arrays.iter().copied(), &hasher, hashes)+ .unwrap();+ });+ }+ }++ /// Asserts each union row contributes the hash of its selected value on its own+ #[cfg(not(feature = "force_hash_collisions"))]+ fn assert_union_hashes_match_selected_values(+ union: &UnionArray,+ create_hashes: impl Fn(&[&dyn Array], &mut [u64]),+ ) {+ let prev = Int64Array::from_iter_values(0..union.len() as i64);+ let mut prev_hashes = vec![0; union.len()];+ create_hashes(&[&prev], &mut prev_hashes);++ let mut hashes = vec![0; union.len()];+ create_hashes(&[&prev, union], &mut hashes);+ for (i, hash) in hashes.into_iter().enumerate() {+ let value = union+ .child(union.type_id(i))+ .slice(union.value_offset(i), 1);+ let mut value_hash = [0];+ create_hashes(&[&value], &mut value_hash);+ let expected = combine_hashes(prev_hashes[i], value_hash[0]);+ assert_eq!(hash, expected, "row {i} of {union:?}");+ }+ }+#[test]#[cfg(not(feature = "force_hash_collisions"))]fn create_hashes_for_sliced_run_array() -> Result<()> {