Apache DataFusion SQL Query Engine
perf: route probe rows to ArrayMap build partitions in the merged hash join filter (#27091)
Adrian Garcia Badaraccoced13f8b8446
diff --git a/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rs b/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rsindex ad52f23a52..023369e05c 100644--- a/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rs+++ b/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rs@@ -7,7 +7,8 @@use std::{fmt::Display, hash::Hash, sync::Arc};use arrow::{- array::{ArrayRef, BooleanArray, BooleanBufferBuilder, UInt64Array},+ array::{ArrayRef, BooleanArray, BooleanBufferBuilder, UInt32Array, UInt64Array},+ compute::take_arrays,datatypes::{DataType, Schema},record_batch::RecordBatch,};@@ -280,10 +281,11 @@ pub struct HashTableLookupExpr {enum Lookup {/// One map holding every build key, probed directly with no routing.Single(Arc<ReservedMap>),- /// One hash table per build partition of a `Partitioned` join, `None` where- /// the partition is empty. A key is only ever built into the partition its- /// key routes to, so each probe row is looked up in that partition's table- /// alone: no `CASE` over per-partition filters, and no merged table.+ /// One map (a hash table or an array map) per build partition of a+ /// `Partitioned` join, `None` where the partition is empty. A key is only+ /// ever built into the partition its key routes to, so each probe row is+ /// looked up in that partition's map alone: no `CASE` over per-partition+ /// filters, and no merged map.Routed {router: PartitionRouter,maps: Arc<[Option<Arc<ReservedMap>>]>,@@ -352,8 +354,7 @@ impl HashTableLookupExpr {/// for pruning only, or `None`////// A single non-empty map is probed directly without routing, so it may be- /// any [`Map`] and `router` is ignored. Otherwise `router` is required and- /// every map must be a [`Map::HashMap`].+ /// any [`Map`] and `router` is ignored. Otherwise `router` is required.////// # Public Only for Internal Use:/// `datafusion-proto` tests require this constructor, but it is not part of@@ -399,20 +400,10 @@ impl HashTableLookupExpr {"a lookup over anything but one non-empty map needs a partition router");}- (Some(router), _) => {- if maps.iter().flatten().any(|map| {- let map: &Map = map;- !matches!(map, Map::HashMap(_))- }) {- return internal_err!(- "routed lookups only support hash table partitions"- );- }- Lookup::Routed {- router,- maps: maps.into(),- }- }+ (Some(router), _) => Lookup::Routed {+ router,+ maps: maps.into(),+ },};Ok(Self {on_columns,@@ -428,7 +419,7 @@ impl HashTableLookupExpr {self.pruning_bitmap.as_ref()}- /// Membership of each row of `keys` in the hash table of its routed partition.+ /// Membership of each row of `keys` in the map of its routed partition.fn evaluate_routed(&self,keys: &[ArrayRef],@@ -464,17 +455,20 @@ impl HashTableLookupExpr {for (partition, map) in maps.iter().enumerate() {// An empty partition holds no key, so its rows stay unmatched.let Some(map) = map else { continue };- let map: &Map = map;- let Map::HashMap(map) = map else {- return internal_err!(- "routed lookups only support hash table partitions"- );- };let run = starts[partition]..starts[partition + 1];if run.is_empty() {continue;}- let hits = map.contain_hashes(&sorted_hashes[run.clone()]);+ let map: &Map = map;+ let hits = match map {+ Map::HashMap(map) => map.contain_hashes(&sorted_hashes[run.clone()]),+ // An array map is keyed by value, not hash, so it probes+ // the run's own key values.+ Map::ArrayMap(map) => {+ let indices = UInt32Array::from(rows[run.clone()].to_vec());+ map.contain_keys(&take_arrays(keys, &indices, None)?)?+ }+ };for hit in hits.values().set_indices() {matched.set_bit(rows[run.start + hit] as usize, true);}@@ -668,7 +662,9 @@ fn evaluate_columns(#[cfg(test)]mod tests {use super::*;+ use crate::joins::array_map::ArrayMap;use crate::joins::join_hash_map::JoinHashMapU32;+ use arrow::array::AsArray;use datafusion_common::ScalarValue;use datafusion_physical_expr::expressions::Column;use std::collections::hash_map::DefaultHasher;@@ -1140,7 +1136,7 @@ mod tests {)]));let batch = RecordBatch::try_new(schema, vec![Arc::clone(probe)])?;let matched = expr.evaluate(&batch)?.into_array(batch.num_rows())?;- Ok(arrow::array::AsArray::as_boolean(&matched).clone())+ Ok(matched.as_boolean().clone())}/// A hash table of `keys`, hashed with `random_state` as the build side@@ -1154,6 +1150,17 @@ mod tests {Ok(Arc::new(Map::HashMap(Box::new(map))))}+ /// An array map of `keys`, as a dense build side gets.+ fn array_map(keys: &ArrayRef) -> Result<Arc<Map>> {+ let values = keys.as_primitive::<arrow::datatypes::Int32Type>();+ let key = |value: Option<i32>| {+ ArrayMap::key_to_u64(&ScalarValue::Int32(value)).expect("Int32 key")+ };+ let min = key(arrow::compute::min(values));+ let max = key(arrow::compute::max(values));+ Ok(Arc::new(Map::ArrayMap(ArrayMap::try_new(keys, min, max)?)))+ }+/// One randomly generated scenario, see [`routed_lookup_fuzz`].fn run_routed_lookup_scenario(seed: u64) -> Result<()> {use crate::repartition::{BatchPartitioner, REPARTITION_RANDOM_STATE};@@ -1195,7 +1202,16 @@ mod tests {.partition(RecordBatch::try_new(schema, vec![Arc::clone(&build)])?,|partition, batch| {- maps[partition] = Some(hash_table(batch.column(0), &random_state)?);+ // Half the partitions build an array map, as dense keys would. Under+ // forced collisions a hash table matches every key, so an array+ // map would disagree with the single-table reference below.+ let dense =+ cfg!(not(feature = "force_hash_collisions")) && rng.random_bool(0.5);+ maps[partition] = Some(if dense {+ array_map(batch.column(0))?+ } else {+ hash_table(batch.column(0), &random_state)?+ });Ok(())},)?;diff --git a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rsindex 395e8c92f8..f083566acf 100644--- a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs+++ b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs@@ -722,15 +722,7 @@ impl SharedBuildAccumulator {}) => {real_partition_ids.push(partition_id);keys_have_null |= partition_keys_have_null;- let mergeable = match &pushdown {- PushdownStrategy::Map(map, _) => {- let map: &Map = map;- matches!(map, Map::HashMap(_))- }- PushdownStrategy::InList(_) => true,- PushdownStrategy::Empty => false,- };- members = members.filter(|_| mergeable).map(|mut members| {+ members = members.map(|mut members| {members.push((partition_id, pushdown.clone(), bounds.clone()));members});@@ -865,8 +857,8 @@ impl SharedBuildAccumulator {.update(self.preserve_probe_nulls(filter_expr, keys_have_null)?)}- /// `members` holds each non-empty partition's id, strategy (an IN list or a- /// hash table) and bounds, out of `num_partitions` partitions.+ /// `members` holds each non-empty partition's id, strategy (an IN list, a+ /// hash table or an array map) and bounds, out of `num_partitions` partitions.fn merged_membership_filter(&self,members: &[(usize, PushdownStrategy, PartitionBounds)],@@ -973,8 +965,7 @@ impl SharedBuildAccumulator {/// Container pruning reads one bitmap, so merge the partitions' key ranges:/// a hash table's bitmap, an IN list's keys, or else a partition's whole- /// bounds. `None` when no partition narrows its bounds, as the bounds- /// predicate already prunes that much.+ /// bounds. `None` when nothing can be excluded.fn merged_pruning_bitmap(&self,members: &[(usize, PushdownStrategy, PartitionBounds)],@@ -999,12 +990,6 @@ impl SharedBuildAccumulator {}),}).collect::<Option<Vec<_>>>()?;- if parts- .iter()- .all(|part| matches!(part, KeyRangePart::Range { .. }))- {- return None;- }KeyRangeBitmap::union(&merged.min, &merged.max, &parts).map(Arc::new)}@@ -1496,6 +1481,104 @@ mod tests {assert_eq!(matched.as_boolean(), &BooleanArray::from(expected.to_vec()));}+ /// An array map holding `keys`, as a dense build side gets.+ fn array_map_of(keys: &[i32]) -> PushdownStrategy {+ use crate::joins::array_map::ArrayMap;++ let bound = |key: Option<&i32>| {+ ArrayMap::key_to_u64(&ScalarValue::Int32(key.copied())).unwrap()+ };+ let min = bound(keys.iter().min());+ let max = bound(keys.iter().max());+ let keys: ArrayRef = Arc::new(Int32Array::from(keys.to_vec()));+ let map = ArrayMap::try_new(&keys, min, max).unwrap();+ let map = Arc::new(Map::ArrayMap(map));+ PushdownStrategy::Map(Arc::new(ReservedMap::unreserved(map)), None)+ }++ /// Array map partitions merge into one lookup with hash table partitions.+ #[test]+ #[cfg(not(feature = "force_hash_collisions"))]+ fn partitioned_array_map_and_hash_map_partitions_merge() {+ let acc = make_partitioned_expr_accumulator_for_test(4);+ let build: Vec<i32> = (0..40).step_by(2).collect();++ acc.build_filter(FinalizeInput::Partitioned(routed_partitions(+ &acc,+ 4,+ &build,+ |partition, keys| {+ if partition % 2 == 0 {+ array_map_of(keys)+ } else {+ hash_map_of(keys)+ }+ },+ )))+ .unwrap();++ assert!(+ current_expr(&acc).downcast_ref::<CaseExpr>().is_none(),+ "expected a merged lookup, got: {}",+ current_expr(&acc)+ );+ let probe: Vec<i32> = (0..40).collect();+ let expected: Vec<bool> = probe.iter().map(|key| key % 2 == 0).collect();+ assert_matches(&acc, &probe, &expected);+ }++ /// Array map partitions merge under Range routing too.+ #[test]+ #[cfg(not(feature = "force_hash_collisions"))]+ fn partitioned_range_array_map_partitions_merge() -> Result<()> {+ let mut acc = make_partitioned_expr_accumulator_for_test(4);+ acc.probe_range_partitioning = Some(RangePartitioning::try_new(+ [PhysicalSortExpr::new(+ Arc::clone(&acc.on_right[0]),+ Default::default(),+ )]+ .into(),+ vec![+ SplitPoint::new(vec![ScalarValue::Int32(Some(10))]),+ SplitPoint::new(vec![ScalarValue::Int32(Some(20))]),+ SplitPoint::new(vec![ScalarValue::Int32(Some(30))]),+ ],+ )?);++ acc.build_filter(FinalizeInput::Partitioned(vec![+ reported(array_map_of(&[7, 8, 9]), bounds(7, 9)),+ reported(PushdownStrategy::Empty, no_bounds()),+ reported(array_map_of(&[27, 28, 29]), bounds(27, 29)),+ reported(array_map_of(&[37, 38, 39]), bounds(37, 39)),+ ]))?;++ assert!(current_expr(&acc).downcast_ref::<CaseExpr>().is_none());+ let probe: Vec<i32> = (0..50).collect();+ let expected: Vec<bool> = probe+ .iter()+ .map(|key| matches!(key, 7..=9 | 27..=29 | 37..=39))+ .collect();+ assert_matches(&acc, &probe, &expected);+ Ok(())+ }++ /// Partitions without a bitmap still leave the gaps between them prunable.+ #[test]+ fn disjoint_array_map_partitions_keep_gap_pruning() {+ let acc = make_partitioned_expr_accumulator_for_test(2);+ let members = [+ (0, array_map_of(&[7, 8, 9]), bounds(7, 9)),+ (1, array_map_of(&[37, 38, 39]), bounds(37, 39)),+ ];++ let bitmap = acc+ .merged_pruning_bitmap(&members, Some(&bounds(7, 39)))+ .expect("the gap between the partitions can be pruned");++ assert!(bitmap.may_contain_range(7, 9));+ assert!(!bitmap.may_contain_range(10, 36));+ }+#[test]fn partitioned_one_real_partition_with_rest_empty_skips_case() {let acc = make_partitioned_expr_accumulator_for_test(3);