Apache DataFusion SQL Query Engine
feat(perf): compute routing and lookup hashes once (#26249)
WeblWable77a370b9fe8
diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.ymlindex a240f2c3e2..f632b9f2ef 100644--- a/.github/workflows/extended.yml+++ b/.github/workflows/extended.yml@@ -84,8 +84,8 @@ jobs:df -h- name: Run tests (excluding doctests)run: cargo xtask ci step test extended- - name: Run spill pool fuzzer with more iterations- run: cargo xtask ci step test spill-pool-fuzz+ - name: Run fuzzers with more iterations+ run: cargo xtask ci step test fuzz- parallel:- name: Verify Working Directory Cleanrun: git diff --exit-codediff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown.rsindex eeba0dc220..6b4f47227c 100644--- a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs@@ -1152,7 +1152,7 @@ async fn test_hashjoin_dynamic_filter_pushdown_partitioned() {- RepartitionExec: partitioning=Hash([a@0, b@1], 12), input_partitions=1- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true- RepartitionExec: partitioning=Hash([a@0, b@1], 12), input_partitions=1- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, e], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ CASE hash_repartition % 12 WHEN 5 THEN a@0 >= ab AND a@0 <= ab AND b@1 >= bb AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:ab,c1:bb}]) WHEN 8 THEN a@0 >= aa AND a@0 <= aa AND b@1 >= ba AND b@1 <= ba AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}]) ELSE false END ]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, e], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:ab,c1:bb}, {c0:aa,c1:ba}]) ]");@@ -1368,7 +1368,7 @@ async fn test_hashjoin_dynamic_filter_pushdown_range_partitioned() {- RepartitionExec: partitioning=Range([a@0 ASC, b@1 ASC], [(aa, bb)], 2), input_partitions=1- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true- RepartitionExec: partitioning=Range([a@0 ASC, b@1 ASC], [(aa, bb)], 2), input_partitions=1- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, e], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ CASE range_partition WHEN 0 THEN a@0 >= aa AND a@0 <= aa AND b@1 >= ba AND b@1 <= ba AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}]) WHEN 1 THEN a@0 >= ab AND a@0 <= ab AND b@1 >= bb AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:ab,c1:bb}]) ELSE false END ]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, e], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ]");diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex 8273a99438..20a7def8f3 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -1688,6 +1688,7 @@ impl ExecutionPlan for HashJoinExec {repartition_random_state,self.null_equality,not_in,+ &context.session_config().options().optimizer,))})))})@@ -9897,19 +9898,8 @@ mod tests {dynamic_filter.wait_complete().await;let filter = dynamic_filter.current()?.to_string();- // The aggregated build-side NULL must widen the routed filter once, at the- // top: `c2 IS NULL OR CASE hash(c2) % 4 WHEN ... END`.- //- // Under `force_hash_collisions` every key hashes to the same value, so the- // repartition routes all build rows to a single partition and- // `build_partitioned_filter` collapses to that partition's filter with no- // routing `CASE`. The NULL widening this test guards is asserted either way.- if cfg!(not(feature = "force_hash_collisions")) {- assert!(- filter.contains("CASE"),- "expected a routed partitioned filter, got: {filter}"- );- }+ // The aggregated build-side NULL must widen the whole merged filter once,+ // at the top: `c2 IS NULL OR <membership over every partition>`.assert!(filter.starts_with("c2@0 IS NULL OR"),"expected the IS NULL disjunct to wrap the whole routed filter, got: {filter}"diff --git a/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rs b/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rsindex b580687e15..66c7c75f61 100644--- a/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rs+++ b/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rs@@ -7,21 +7,20 @@use std::{fmt::Display, hash::Hash, sync::Arc};use arrow::{- array::{ArrayRef, UInt64Array},+ array::{ArrayRef, BooleanArray, BooleanBufferBuilder, UInt64Array},datatypes::{DataType, Schema},record_batch::RecordBatch,};-use datafusion_common::Result;use datafusion_common::hash_utils::RandomState;use datafusion_common::hash_utils::{create_hashes, with_hashes};-#[cfg(feature = "proto")]-use datafusion_common::internal_err;+use datafusion_common::{Result, internal_err};use datafusion_expr::ColumnarValue;use datafusion_expr_common::dyn_eq::DynHash;use datafusion_physical_expr_common::physical_expr::{PhysicalExpr, PhysicalExprRef};use crate::joins::Map;use crate::joins::key_range_bitmap::KeyRangeBitmap;+use crate::repartition::RangeExpr;/// RandomState wrapper that preserves the seed used to create it.///@@ -261,29 +260,100 @@ impl HashExpr {/// Physical expression that checks join keys in a [`Map`] (hash table or array map).///-/// Returns a [`BooleanArray`](arrow::array::BooleanArray) indicating if join keys (from `on_columns`) exist in the map.+/// Returns a [`BooleanArray`] indicating if join keys (from `on_columns`) exist in the map.// TODO: rename to MapLookupExprpub struct HashTableLookupExpr {/// Columns in the ON clause used to compute the join key for lookupson_columns: Vec<PhysicalExprRef>,/// Random state for hashing (with seeds preserved for serialization)random_state: SeededRandomState,- /// Map to check against (hash table or array map)- map: Arc<Map>,+ /// Map(s) to check against+ lookup: Lookup,/// Description for displaydescription: String,/// Shared pruning-only view of the build side's key rangepruning_bitmap: Option<Arc<KeyRangeBitmap>>,}++/// The build side a [`HashTableLookupExpr`] checks probe keys against.+#[derive(Clone)]+enum Lookup {+ /// One map holding every build key, probed directly with no routing.+ Single(Arc<Map>),+ /// One hash table per build partition of a `Partitioned` join, `None` where+ /// the partition is empty. A key is only ever built into the partition its+ /// key routes to, so each probe row is looked up in that partition's table+ /// alone: no `CASE` over per-partition filters, and no merged table.+ Routed {+ router: PartitionRouter,+ maps: Arc<[Option<Arc<Map>>]>,+ },+}++/// Assigns probe rows to the build partition their key was repartitioned to.+///+/// # Public Only for Internal Use:+/// Public only because [`HashTableLookupExpr::new`] takes it; not part of the+/// supported public API.+#[doc(hidden)]+#[derive(Clone)]+pub enum PartitionRouter {+ /// `hash(keys) % partitions`, as hash repartitioning routes rows.+ Hash(SeededRandomState),+ /// The probe side's range partitioning.+ Range(Arc<RangeExpr>),+}++impl PartitionRouter {+ /// Writes the partition of each row of `keys` to `partition_ids`, which+ /// holds one slot per row.+ fn route(+ &self,+ keys: &[ArrayRef],+ partitions: usize,+ partition_ids: &mut Vec<u64>,+ ) -> Result<()> {+ match self {+ PartitionRouter::Hash(random_state) => {+ partition_ids.resize(keys[0].len(), 0);+ create_hashes(keys, random_state.random_state(), partition_ids)?;+ let partitions = partitions as u64;+ for id in partition_ids.iter_mut() {+ *id %= partitions;+ }+ Ok(())+ }+ PartitionRouter::Range(range) => {+ range.route_partition_ids(keys, partition_ids)+ }+ }+ }++ fn seed(&self) -> Option<u64> {+ match self {+ PartitionRouter::Hash(random_state) => Some(random_state.seed()),+ PartitionRouter::Range(_) => None,+ }+ }+}+impl HashTableLookupExpr {- /// Create a new HashTableLookupExpr+ /// Create a new HashTableLookupExpr that looks each probe row up in the+ /// map of the build partition `router` assigns it to.////// # Arguments/// * `on_columns` - Columns in the ON clause used to compute the join key/// * `random_state` - SeededRandomState for hashing- /// * `map` - Map to check membership (hash table or array map)+ /// * `router` - Routes each probe row to the partition its key was+ /// repartitioned to; `None` when `maps` is a single map+ /// * `maps` - One entry per build partition, `None` for an empty partition/// * `description` - Description for debugging- /// * `pruning_bitmap` - key-range summary for pruning only, or `None`+ /// * `pruning_bitmap` - key-range summary covering every partition's keys+ /// for pruning only, or `None`+ ///+ /// A single non-empty map is probed directly without routing, so it may be+ /// any [`Map`] and `router` is ignored. Otherwise `router` is required and+ /// every map must be a [`Map::HashMap`].////// # Public Only for Internal Use:/// `datafusion-proto` tests require this constructor, but it is not part of@@ -292,23 +362,102 @@ impl HashTableLookupExpr {pub fn new(on_columns: Vec<PhysicalExprRef>,random_state: SeededRandomState,- map: Arc<Map>,+ router: Option<PartitionRouter>,+ maps: Vec<Option<Arc<Map>>>,description: String,pruning_bitmap: Option<Arc<KeyRangeBitmap>>,- ) -> Self {- Self {+ ) -> Result<Self> {+ let lookup = match (router, maps.as_slice()) {+ // Every row routes to the only partition, so skip routing.+ (_, [Some(map)]) => Lookup::Single(Arc::clone(map)),+ (None, _) => {+ return internal_err!(+ "a lookup over anything but one non-empty map needs a partition router"+ );+ }+ (Some(router), _) => {+ if maps+ .iter()+ .flatten()+ .any(|map| !matches!(map.as_ref(), Map::HashMap(_)))+ {+ return internal_err!(+ "routed lookups only support hash table partitions"+ );+ }+ Lookup::Routed {+ router,+ maps: maps.into(),+ }+ }+ };+ Ok(Self {on_columns,random_state,- map,+ lookup,description,pruning_bitmap,- }+ })}/// Which parts of this build side's key range hold keys, for container pruning.pub fn pruning_bitmap(&self) -> Option<&Arc<KeyRangeBitmap>> {self.pruning_bitmap.as_ref()}++ /// Membership of each row of `keys` in the hash table of its routed partition.+ fn evaluate_routed(+ &self,+ keys: &[ArrayRef],+ router: &PartitionRouter,+ maps: &[Option<Arc<Map>>],+ ) -> Result<BooleanArray> {+ let num_rows = keys[0].len();+ let mut partition_ids = Vec::with_capacity(num_rows);+ router.route(keys, maps.len(), &mut partition_ids)?;++ with_hashes(keys, self.random_state.random_state(), |hashes| {+ // Counting sort of the rows by partition, so each table is probed+ // once per batch over a contiguous run of its rows' hashes.+ let mut starts = vec![0usize; maps.len() + 1];+ for &id in &partition_ids {+ starts[id as usize + 1] += 1;+ }+ for i in 0..maps.len() {+ starts[i + 1] += starts[i];+ }+ let mut next = starts.clone();+ let mut rows = vec![0u32; num_rows];+ let mut sorted_hashes = vec![0u64; num_rows];+ for (row, (&id, &hash)) in partition_ids.iter().zip(hashes).enumerate() {+ let slot = &mut next[id as usize];+ rows[*slot] = row as u32;+ sorted_hashes[*slot] = hash;+ *slot += 1;+ }++ let mut matched = BooleanBufferBuilder::new(num_rows);+ matched.append_n(num_rows, false);+ for (partition, map) in maps.iter().enumerate() {+ // An empty partition holds no key, so its rows stay unmatched.+ let Some(map) = map else { continue };+ let Map::HashMap(map) = map.as_ref() else {+ return internal_err!(+ "routed lookups only support hash table partitions"+ );+ };+ let run = starts[partition]..starts[partition + 1];+ if run.is_empty() {+ continue;+ }+ let hits = map.contain_hashes(&sorted_hashes[run.clone()]);+ for hit in hits.values().set_indices() {+ matched.set_bit(rows[run.start + hit] as usize, true);+ }+ }+ Ok(BooleanArray::new(matched.finish(), None))+ })+ }}impl std::fmt::Debug for HashTableLookupExpr {fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {@@ -319,7 +468,21 @@ impl std::fmt::Debug for HashTableLookupExpr {.collect::<Vec<_>>().join(", ");let seed = self.random_state.seed();- write!(f, "{}({cols}, [{seed}])", self.description)+ match &self.lookup {+ Lookup::Single(_) => write!(f, "{}({cols}, [{seed}])", self.description),+ Lookup::Routed { router, maps } => {+ let router = match router.seed() {+ Some(routing_seed) => format!("hash [{routing_seed}]"),+ None => "range".to_string(),+ };+ write!(+ f,+ "{}({cols}, [{seed}], routed by {router} over {} partitions)",+ self.description,+ maps.len()+ )+ }+ }}}@@ -329,7 +492,7 @@ impl Hash for HashTableLookupExpr {self.description.hash(state);self.random_state.seed().hash(state);// Hash immutable map identity, consistently with pointer equality below.- Arc::as_ptr(&self.map).hash(state);+ self.lookup.identity().hash(state);}}@@ -340,12 +503,23 @@ impl PartialEq for HashTableLookupExpr {self.on_columns == other.on_columns&& self.description == other.description&& self.random_state.seed() == other.random_state.seed()- && Arc::ptr_eq(&self.map, &other.map)+ && self.lookup.identity() == other.lookup.identity()}}impl Eq for HashTableLookupExpr {}+impl Lookup {+ /// Identity of the immutable build-side maps. A routed lookup's router is+ /// created together with its maps, so the maps identify it as well.+ fn identity(&self) -> *const () {+ match self {+ Lookup::Single(map) => Arc::as_ptr(map).cast(),+ Lookup::Routed { maps, .. } => Arc::as_ptr(maps).cast(),+ }+ }+}+impl Display for HashTableLookupExpr {fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {write!(f, "{}", self.description)@@ -364,7 +538,7 @@ impl PhysicalExpr for HashTableLookupExpr {Ok(Arc::new(HashTableLookupExpr {on_columns: children,random_state: self.random_state.clone(),- map: Arc::clone(&self.map),+ lookup: self.lookup.clone(),description: self.description.clone(),pruning_bitmap: self.pruning_bitmap.clone(),}))@@ -382,15 +556,21 @@ impl PhysicalExpr for HashTableLookupExpr {// Evaluate columnslet join_keys = evaluate_columns(&self.on_columns, batch)?;- match self.map.as_ref() {- Map::HashMap(map) => {- with_hashes(&join_keys, self.random_state.random_state(), |hashes| {- let array = map.contain_hashes(hashes);+ match &self.lookup {+ Lookup::Single(map) => match map.as_ref() {+ Map::HashMap(map) => {+ with_hashes(&join_keys, self.random_state.random_state(), |hashes| {+ let array = map.contain_hashes(hashes);+ Ok(ColumnarValue::Array(Arc::new(array)))+ })+ }+ Map::ArrayMap(map) => {+ let array = map.contain_keys(&join_keys)?;Ok(ColumnarValue::Array(Arc::new(array)))- })- }- Map::ArrayMap(map) => {- let array = map.contain_keys(&join_keys)?;+ }+ },+ Lookup::Routed { router, maps } => {+ let array = self.evaluate_routed(&join_keys, router, maps)?;Ok(ColumnarValue::Array(Arc::new(array)))}}@@ -411,7 +591,7 @@ impl PhysicalExpr for HashTableLookupExpr {// with lit(true), see the comment belowon_columns: _,random_state: _,- map: _,+ lookup: _,description: _,pruning_bitmap: _,} = self;@@ -474,7 +654,7 @@ mod tests {/// The pruning bitmap must survive `with_new_children`, which a scan calls/// once per file.#[test]- fn test_pruning_bitmap_shared_with_derived_children() {+ fn test_pruning_bitmap_shared_with_derived_children() -> Result<()> {let keys: ArrayRef = Arc::new(arrow::array::Int64Array::from(vec![0, 1_000_000]));let bitmap = Arc::new(KeyRangeBitmap::try_new(@@ -483,22 +663,28 @@ mod tests {&ScalarValue::Int64(Some(1_000_000)),2,)- .expect("two keys a million apart leave prunable gaps"),+ .ok_or_else(|| {+ datafusion_common::internal_datafusion_err!(+ "two keys a million apart leave prunable gaps"+ )+ })?,);let expr = Arc::new(HashTableLookupExpr::new(vec![Arc::new(Column::new("a", 0))],SeededRandomState::with_seed(1),- Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(10)))),+ None,+ vec![Some(Arc::new(Map::HashMap(Box::new(+ JoinHashMapU32::with_capacity(10),+ ))))],"hash_lookup".to_string(),Some(Arc::clone(&bitmap)),- ));+ )?);- let derived = expr- .with_new_children(vec![Arc::new(Column::new("a", 7))])- .unwrap();+ let derived = expr.with_new_children(vec![Arc::new(Column::new("a", 7))])?;let derived = derived.downcast_ref::<HashTableLookupExpr>().unwrap();assert_eq!(derived.children()[0].to_string(), "a@7");assert!(Arc::ptr_eq(&bitmap, derived.pruning_bitmap().unwrap()));+ Ok(())}#[test]@@ -769,7 +955,7 @@ mod tests {}#[test]- fn test_hash_table_lookup_expr_eq_same() {+ fn test_hash_table_lookup_expr_eq_same() -> Result<()> {let col_a: PhysicalExprRef = Arc::new(Column::new("a", 0));let hash_map =Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(10))));@@ -777,24 +963,27 @@ mod tests {let expr1 = HashTableLookupExpr::new(vec![Arc::clone(&col_a)],SeededRandomState::with_seed(1),- Arc::clone(&hash_map),+ None,+ vec![Some(Arc::clone(&hash_map))],"lookup".to_string(),None,- );+ )?;let expr2 = HashTableLookupExpr::new(vec![Arc::clone(&col_a)],SeededRandomState::with_seed(1),- Arc::clone(&hash_map),+ None,+ vec![Some(Arc::clone(&hash_map))],"lookup".to_string(),None,- );+ )?;assert_eq!(expr1, expr2);+ Ok(())}#[test]- fn test_hash_table_lookup_expr_eq_different_columns() {+ fn test_hash_table_lookup_expr_eq_different_columns() -> Result<()> {let col_a: PhysicalExprRef = Arc::new(Column::new("a", 0));let col_b: PhysicalExprRef = Arc::new(Column::new("b", 1));@@ -804,24 +993,27 @@ mod tests {let expr1 = HashTableLookupExpr::new(vec![Arc::clone(&col_a)],SeededRandomState::with_seed(1),- Arc::clone(&hash_map),+ None,+ vec![Some(Arc::clone(&hash_map))],"lookup".to_string(),None,- );+ )?;let expr2 = HashTableLookupExpr::new(vec![Arc::clone(&col_b)],SeededRandomState::with_seed(1),- Arc::clone(&hash_map),+ None,+ vec![Some(Arc::clone(&hash_map))],"lookup".to_string(),None,- );+ )?;assert_ne!(expr1, expr2);+ Ok(())}#[test]- fn test_hash_table_lookup_expr_eq_different_description() {+ fn test_hash_table_lookup_expr_eq_different_description() -> Result<()> {let col_a: PhysicalExprRef = Arc::new(Column::new("a", 0));let hash_map =Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(10))));@@ -829,24 +1021,27 @@ mod tests {let expr1 = HashTableLookupExpr::new(vec![Arc::clone(&col_a)],SeededRandomState::with_seed(1),- Arc::clone(&hash_map),+ None,+ vec![Some(Arc::clone(&hash_map))],"lookup_one".to_string(),None,- );+ )?;let expr2 = HashTableLookupExpr::new(vec![Arc::clone(&col_a)],SeededRandomState::with_seed(1),- Arc::clone(&hash_map),+ None,+ vec![Some(Arc::clone(&hash_map))],"lookup_two".to_string(),None,- );+ )?;assert_ne!(expr1, expr2);+ Ok(())}#[test]- fn test_hash_table_lookup_expr_eq_different_hash_map() {+ fn test_hash_table_lookup_expr_eq_different_hash_map() -> Result<()> {let col_a: PhysicalExprRef = Arc::new(Column::new("a", 0));// Two different Arc pointers (even with same content) should not be equal@@ -857,25 +1052,28 @@ mod tests {let expr1 = HashTableLookupExpr::new(vec![Arc::clone(&col_a)],SeededRandomState::with_seed(1),- hash_map1,+ None,+ vec![Some(hash_map1)],"lookup".to_string(),None,- );+ )?;let expr2 = HashTableLookupExpr::new(vec![Arc::clone(&col_a)],SeededRandomState::with_seed(1),- hash_map2,+ None,+ vec![Some(hash_map2)],"lookup".to_string(),None,- );+ )?;// Different Arc pointers means not equal (uses Arc::ptr_eq)assert_ne!(expr1, expr2);+ Ok(())}#[test]- fn test_hash_table_lookup_expr_hash_consistency() {+ fn test_hash_table_lookup_expr_hash_consistency() -> Result<()> {let col_a: PhysicalExprRef = Arc::new(Column::new("a", 0));let hash_map =Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(10))));@@ -883,21 +1081,142 @@ mod tests {let expr1 = HashTableLookupExpr::new(vec![Arc::clone(&col_a)],SeededRandomState::with_seed(1),- Arc::clone(&hash_map),+ None,+ vec![Some(Arc::clone(&hash_map))],"lookup".to_string(),None,- );+ )?;let expr2 = HashTableLookupExpr::new(vec![Arc::clone(&col_a)],SeededRandomState::with_seed(1),- Arc::clone(&hash_map),+ None,+ vec![Some(Arc::clone(&hash_map))],"lookup".to_string(),None,- );+ )?;// Equal expressions should have equal hashesassert_eq!(expr1, expr2);assert_eq!(compute_hash(&expr1), compute_hash(&expr2));+ Ok(())+ }++ /// Probes `probe` against `expr`, one row per key of column `a`.+ fn lookup(expr: &HashTableLookupExpr, probe: &ArrayRef) -> Result<BooleanArray> {+ let schema = Arc::new(Schema::new(vec![arrow::datatypes::Field::new(+ "a",+ DataType::Int32,+ true,+ )]));+ let batch = RecordBatch::try_new(schema, vec![Arc::clone(probe)])?;+ let matched = expr.evaluate(&batch)?.into_array(batch.num_rows())?;+ Ok(arrow::array::AsArray::as_boolean(&matched).clone())+ }++ /// A hash table of `keys`, hashed with `random_state` as the build side+ /// hashes them.+ fn hash_table(keys: &ArrayRef, random_state: &SeededRandomState) -> Result<Arc<Map>> {+ use crate::joins::join_hash_map::JoinHashMapType;+ let mut hashes = vec![0; keys.len()];+ create_hashes([keys], random_state.random_state(), &mut hashes)?;+ let mut map = JoinHashMapU32::with_capacity(keys.len());+ map.update_from_iter(Box::new(hashes.iter().enumerate()), 0);+ Ok(Arc::new(Map::HashMap(Box::new(map))))+ }++ /// One randomly generated scenario, see [`routed_lookup_fuzz`].+ fn run_routed_lookup_scenario(seed: u64) -> Result<()> {+ use crate::repartition::{BatchPartitioner, REPARTITION_RANDOM_STATE};+ use rand::rngs::StdRng;+ use rand::{Rng, SeedableRng};++ let mut rng = StdRng::seed_from_u64(seed);+ let num_partitions = rng.random_range(1..8usize);+ // A key domain near the row counts, so probes both hit and miss.+ let build: Vec<i32> = (0..rng.random_range(0..128))+ .map(|_| rng.random_range(-64..64))+ .collect();+ let probe: Vec<Option<i32>> = (0..rng.random_range(0..128))+ .map(|_| rng.random_bool(0.9).then(|| rng.random_range(-80..80)))+ .collect();+ let context = format!(+ "seed {seed}: {num_partitions} partitions, {} build rows, {} probe rows",+ build.len(),+ probe.len()+ );++ let column: PhysicalExprRef = Arc::new(Column::new("a", 0));+ let schema = Arc::new(Schema::new(vec![arrow::datatypes::Field::new(+ "a",+ DataType::Int32,+ false,+ )]));+ let build: ArrayRef = Arc::new(arrow::array::Int32Array::from(build));+ let probe: ArrayRef = Arc::new(arrow::array::Int32Array::from(probe));+ let random_state = SeededRandomState::with_seed(1);++ // Split the build side exactly as hash repartitioning does.+ let mut maps = vec![None; num_partitions];+ BatchPartitioner::new_hash_partitioner(+ vec![Arc::clone(&column)],+ num_partitions,+ crate::metrics::Time::new(),+ )?+ .partition(+ RecordBatch::try_new(schema, vec![Arc::clone(&build)])?,+ |partition, batch| {+ maps[partition] = Some(hash_table(batch.column(0), &random_state)?);+ Ok(())+ },+ )?;+ let routed = HashTableLookupExpr::new(+ vec![Arc::clone(&column)],+ random_state.clone(),+ Some(PartitionRouter::Hash(REPARTITION_RANDOM_STATE)),+ maps,+ "routed".to_string(),+ None,+ )?;+ // Reference: the single-table lookup over the whole build side.+ let single = HashTableLookupExpr::new(+ vec![column],+ random_state.clone(),+ None,+ vec![Some(hash_table(&build, &random_state)?)],+ "single".to_string(),+ None,+ )?;++ assert_eq!(+ lookup(&routed, &probe)?,+ lookup(&single, &probe)?,+ "{context}"+ );+ Ok(())+ }++ /// Routing each probe row to its build partition's table matches one table+ /// over the whole build side, when the build side is split the way hash+ /// repartitioning splits it.+ ///+ /// `DATAFUSION_ROUTED_LOOKUP_FUZZ_ITERATIONS` and+ /// `DATAFUSION_ROUTED_LOOKUP_FUZZ_SEED` select how many scenarios run and+ /// from which seed; a failure names its seed so it can be replayed.+ #[test]+ fn routed_lookup_fuzz() -> Result<()> {+ let env_u64 = |name: &str, default: u64| {+ std::env::var(name)+ .ok()+ .and_then(|v| v.parse().ok())+ .unwrap_or(default)+ };+ let first_seed = env_u64("DATAFUSION_ROUTED_LOOKUP_FUZZ_SEED", 0);+ let iterations = env_u64("DATAFUSION_ROUTED_LOOKUP_FUZZ_ITERATIONS", 256);+ // `saturating_add` so a replay seed near `u64::MAX` cannot overflow.+ for seed in first_seed..first_seed.saturating_add(iterations) {+ run_routed_lookup_scenario(seed)?;+ }+ Ok(())}}diff --git a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rsindex c3c37a193e..321ca4b10d 100644--- a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs+++ b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs@@ -17,16 +17,19 @@ use crate::joins::hash_join::exec::HASH_JOIN_SEED;use crate::joins::hash_join::inlist_builder::build_struct_fields;use crate::joins::hash_join::not_in::NotIn;use crate::joins::hash_join::partitioned_hash_eval::{- HashExpr, HashTableLookupExpr, SeededRandomState,+ HashExpr, HashTableLookupExpr, PartitionRouter, SeededRandomState,};-use crate::joins::key_range_bitmap::KeyRangeBitmap;+use crate::joins::join_hash_map::{JoinHashMapType, JoinHashMapU32};+use crate::joins::key_range_bitmap::{KeyRangeBitmap, KeyRangePart};use crate::repartition::RangeExpr;-use arrow::array::ArrayRef;+use arrow::array::{ArrayRef, AsArray};+use arrow::compute::concat;use arrow::datatypes::{DataType, Field, Schema};-use datafusion_common::config::ConfigOptions;+use datafusion_common::config::{ConfigOptions, OptimizerOptions};+use datafusion_common::hash_utils::create_hashes;use datafusion_common::{DataFusionError, NullEquality, Result, ScalarValue, SharedResult,- assert_or_internal_err,+ assert_or_internal_err, internal_err,};use datafusion_expr::Operator;use datafusion_functions::core::r#struct as struct_func;@@ -46,7 +49,7 @@ use tokio::sync::Notify;pub(crate) struct ColumnBounds {/// The minimum value observed for this columnpub(crate) min: ScalarValue,- /// The maximum value observed for this column+ /// The maximum value observed for this columnpub(crate) max: ScalarValue,}@@ -73,6 +76,37 @@ impl PartitionBounds {pub(crate) fn get_column_bounds(&self, index: usize) -> Option<&ColumnBounds> {self.column_bounds.get(index)}++ /// Bounds covering both `self` and `other`, which must describe the same+ /// columns. A NULL bound means the partition holds no non-null key in that+ /// column, so it never widens the other side's bound.+ fn union(self, other: &PartitionBounds) -> PartitionBounds {+ // Every non-empty partition of a join reports one bound per join key.+ debug_assert_eq!(+ self.column_bounds.len(),+ other.column_bounds.len(),+ "partitions report bounds for different numbers of columns"+ );+ let column_bounds = self+ .column_bounds+ .into_iter()+ .zip(&other.column_bounds)+ .map(|(a, b)| {+ let min = if !b.min.is_null() && (a.min.is_null() || b.min < a.min) {+ b.min.clone()+ } else {+ a.min+ };+ let max = if !b.max.is_null() && (a.max.is_null() || b.max > a.max) {+ b.max.clone()+ } else {+ a.max+ };+ ColumnBounds::new(min, max)+ })+ .collect();+ PartitionBounds::new(column_bounds)+ }}/// Creates a membership predicate for filter pushdown.@@ -129,10 +163,11 @@ fn create_membership_predicate(Ok(Some(Arc::new(HashTableLookupExpr::new(on_right.to_vec(),random_state.clone(),- hash_map,+ None,+ vec![Some(hash_map)],"hash_lookup".to_string(),pruning_bitmap,- )) as Arc<dyn PhysicalExpr>))+ )?) as Arc<dyn PhysicalExpr>))}// Empty partition - should not create a filter for thisPushdownStrategy::Empty => Ok(None),@@ -262,6 +297,12 @@ pub(crate) struct SharedBuildAccumulator {/// Null-aware anti join (`NOT IN`). A probe-side NULL must reach the join so its/// three-valued logic can collapse the result, so the pushed filter keeps NULL rows.not_in: Option<NotIn>,+ /// Largest merged IN list, in bytes, that partitioned filters keep as an IN+ /// list: `hash_join_inlist_pushdown_max_size`, as for a single build.+ in_list_max_size: usize,+ /// Largest merged IN list, in values, that partitioned filters keep as an IN+ /// list: `hash_join_inlist_pushdown_max_distinct_values`, as for a single build.+ in_list_max_values: usize,}/// Strategy for filter pushdown (decided at collection time)@@ -374,6 +415,7 @@ impl SharedBuildAccumulator {repartition_random_state: SeededRandomState,null_equality: NullEquality,not_in: Option<NotIn>,+ optimizer: &OptimizerOptions,) -> Self {// Troubleshooting: If partition counts are incorrect, verify this logic matches// the actual execution pattern in collect_build_side()@@ -431,6 +473,8 @@ impl SharedBuildAccumulator {probe_range_partitioning,null_equality,not_in,+ in_list_max_size: optimizer.hash_join_inlist_pushdown_max_size,+ in_list_max_values: optimizer.hash_join_inlist_pushdown_max_distinct_values,}}@@ -656,6 +700,10 @@ impl SharedBuildAccumulator {let mut empty_partition_ids = Vec::new();let mut has_canceled_unknown = false;let mut keys_have_null = false;+ // Every non-empty partition's membership strategy and bounds, merged into+ // one membership filter below; `None` once a partition cannot be merged.+ let mut members = Some(Vec::with_capacity(partitions.len()));+ let mut merged_bounds: Option<PartitionBounds> = None;for (partition_id, partition) in partitions.into_iter().enumerate() {match partition {@@ -673,6 +721,22 @@ impl SharedBuildAccumulator {}) => {real_partition_ids.push(partition_id);keys_have_null |= partition_keys_have_null;+ let mergeable = match &pushdown {+ PushdownStrategy::Map(map, _) => {+ matches!(map.as_ref(), Map::HashMap(_))+ }+ PushdownStrategy::InList(_) => true,+ PushdownStrategy::Empty => false,+ };+ members = members.filter(|_| mergeable).map(|mut members| {+ members.push((partition_id, pushdown.clone(), bounds.clone()));+ members+ });+ merged_bounds = Some(match merged_bounds {+ None => bounds.clone(),+ Some(merged) => merged.union(&bounds),+ });+let membership_expr = create_membership_predicate(&self.on_right,pushdown,@@ -688,6 +752,8 @@ impl SharedBuildAccumulator {PartitionStatus::CanceledUnknown => {has_canceled_unknown = true;partition_filters.push(lit(true));+ // Rows routed here must pass, which a merged filter cannot express.+ members = None;// A canceled partition's build content is unknown, so it// may hold a NULL key.keys_have_null = true;@@ -717,6 +783,12 @@ impl SharedBuildAccumulator {// Only one build partition contains rows, so its filter covers every// possible probe match without routing.Arc::clone(&partition_filters[real_partition_ids[0]])+ } else if let Some(members) = members {+ self.merged_membership_filter(+ &members,+ merged_bounds,+ partition_filters.len(),+ )?} else {// Builds the shared sparse `CASE` for partition filter routing.// Without cancellation, omitted branches are known empty and safely fall@@ -791,6 +863,145 @@ impl SharedBuildAccumulator {.update(self.preserve_probe_nulls(filter_expr, keys_have_null)?)}+ /// `members` holds each non-empty partition's id, strategy (an IN list or a+ /// hash table) and bounds, out of `num_partitions` partitions.+ fn merged_membership_filter(+ &self,+ members: &[(usize, PushdownStrategy, PartitionBounds)],+ merged_bounds: Option<PartitionBounds>,+ num_partitions: usize,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ let lists = members+ .iter()+ .filter_map(|(_, pushdown, _)| match pushdown {+ PushdownStrategy::InList(values) => Some(values),+ _ => None,+ })+ .collect::<Vec<_>>();++ // Every partition is an IN list, and together they still fit the limits a+ // single build uses to pick an IN list: one combined list keeps the+ // statistics and bloom filter pruning an IN list allows. Past those+ // limits, a hash lookup is cheaper per row and carries a pruning bitmap.+ let combined_in_list = lists.len() == members.len() && {+ let values = lists.iter().map(|list| list.len()).sum::<usize>();+ let bytes = lists+ .iter()+ .map(|list| list.get_array_memory_size())+ .sum::<usize>();+ values <= self.in_list_max_values && bytes <= self.in_list_max_size+ };+ let membership = if combined_in_list {+ let values = concat(&lists.iter().map(|l| l.as_ref()).collect::<Vec<_>>())?;+ create_membership_predicate(+ &self.on_right,+ PushdownStrategy::InList(values),+ &HASH_JOIN_SEED,+ self.probe_schema.as_ref(),+ )?+ .expect("an IN list always has a membership predicate")+ } else {+ // Each probe row is looked up in the table of the partition its key+ // routes to, so the partitions' tables are shared as they are: no+ // merged table to build while every probe stream waits on the filter.+ let mut maps = vec![None; num_partitions];+ for (partition_id, pushdown, _) in members {+ maps[*partition_id] = Some(match pushdown {+ PushdownStrategy::Map(map, _) => Arc::clone(map),+ PushdownStrategy::InList(values) => self.in_list_hash_map(values)?,+ PushdownStrategy::Empty => {+ return internal_err!("empty partitions have no members");+ }+ });+ }+ Arc::new(HashTableLookupExpr::new(+ self.on_right.clone(),+ HASH_JOIN_SEED,+ Some(self.partition_router(num_partitions)?),+ maps,+ String::from("hash_lookup"),+ self.merged_pruning_bitmap(members, merged_bounds.as_ref()),+ )?)+ };++ let bounds = merged_bounds+ .and_then(|bounds| create_bounds_predicate(&self.on_right, &bounds));+ Ok(combine_membership_and_bounds(Some(membership), bounds)+ .expect("membership predicate is always present"))+ }++ /// How probe rows find their build partition: as the probe side was+ /// partitioned.+ fn partition_router(&self, num_partitions: usize) -> Result<PartitionRouter> {+ Ok(match &self.probe_range_partitioning {+ Some(range_partitioning) => {+ assert_or_internal_err!(+ num_partitions == range_partitioning.partition_count(),+ "Dynamic filter partition count {} does not match Range partition count {}",+ num_partitions,+ range_partitioning.partition_count()+ );+ PartitionRouter::Range(Arc::new(RangeExpr::try_new_with_schema(+ self.on_right.clone(),+ range_partitioning,+ &self.probe_schema,+ )?))+ }+ None => PartitionRouter::Hash(self.repartition_random_state.clone()),+ })+ }++ /// A hash table of an IN list partition's keys, hashed as the probe side+ /// hashes its join keys: one column per join key.+ fn in_list_hash_map(&self, values: &ArrayRef) -> Result<Arc<Map>> {+ let columns = match values.as_struct_opt() {+ Some(columns) if self.on_right.len() > 1 => columns.columns(),+ _ => std::slice::from_ref(values),+ };+ let mut hashes = vec![0; values.len()];+ create_hashes(columns, HASH_JOIN_SEED.random_state(), &mut hashes)?;+ let mut map = JoinHashMapU32::with_capacity(hashes.len());+ map.update_from_iter(Box::new(hashes.iter().enumerate()), 0);+ Ok(Arc::new(Map::HashMap(Box::new(map))))+ }++ /// Container pruning reads one bitmap, so merge the partitions' key ranges:+ /// a hash table's bitmap, an IN list's keys, or else a partition's whole+ /// bounds. `None` when no partition narrows its bounds, as the bounds+ /// predicate already prunes that much.+ fn merged_pruning_bitmap(+ &self,+ members: &[(usize, PushdownStrategy, PartitionBounds)],+ merged_bounds: Option<&PartitionBounds>,+ ) -> Option<Arc<KeyRangeBitmap>> {+ if self.on_right.len() != 1 {+ return None;+ }+ let merged = merged_bounds?.get_column_bounds(0)?;+ let parts = members+ .iter()+ .map(|(_, pushdown, bounds)| match pushdown {+ PushdownStrategy::Map(_, Some(bitmap)) => {+ Some(KeyRangePart::Bitmap(bitmap))+ }+ PushdownStrategy::InList(keys) => Some(KeyRangePart::Keys(keys.as_ref())),+ _ => bounds+ .get_column_bounds(0)+ .map(|bounds| KeyRangePart::Range {+ min: &bounds.min,+ max: &bounds.max,+ }),+ })+ .collect::<Option<Vec<_>>>()?;+ if parts+ .iter()+ .all(|part| matches!(part, KeyRangePart::Range { .. }))+ {+ return None;+ }+ KeyRangeBitmap::union(&merged.min, &merged.max, &parts).map(Arc::new)+ }+/// Keeps probe rows with a NULL key when the join semantics need them.////// The build-side predicate drops probe rows whose key is NULL. A null-aware join@@ -887,6 +1098,8 @@ pub(super) fn make_partitioned_accumulator_for_test(probe_range_partitioning: None,null_equality: NullEquality::NullEqualsNothing,not_in: None,+ in_list_max_size: 128 * 1024,+ in_list_max_values: 150,}}@@ -908,6 +1121,7 @@ mod tests {use super::*;use crate::joins::hash_join::not_in::{NotInOutput, NotInScope};+ use crate::joins::join_hash_map::JoinHashMapU32;use arrow::array::{ArrayRef, BooleanArray, Float64Array, Int32Array};use arrow::compute::SortOptions;use arrow::record_batch::RecordBatch;@@ -953,6 +1167,8 @@ mod tests {probe_range_partitioning: None,null_equality: NullEquality::NullEqualsNothing,not_in: None,+ in_list_max_size: 128 * 1024,+ in_list_max_values: 150,}}@@ -1124,6 +1340,157 @@ mod tests {assert_literal_bool(&expr, true);}+ fn hash_map() -> PushdownStrategy {+ PushdownStrategy::Map(+ Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(1)))),+ None,+ )+ }++ #[test]+ fn partitioned_hash_map_partitions_route_without_case() {+ let acc = make_partitioned_expr_accumulator_for_test(3);++ acc.build_filter(FinalizeInput::Partitioned(vec![+ reported(hash_map(), bounds(5, 10)),+ reported(PushdownStrategy::Empty, no_bounds()),+ reported(hash_map(), bounds(1, 7)),+ ]))+ .unwrap();++ let expr = current_expr(&acc);+ assert!(expr.downcast_ref::<CaseExpr>().is_none());+ assert_eq!(+ expr.to_string(),+ "probe_key@0 >= 1 AND probe_key@0 <= 10 AND hash_lookup"+ );+ }++ /// A hash table holding `keys`, hashed as the build side hashes them.+ #[cfg(not(feature = "force_hash_collisions"))]+ fn hash_map_of(keys: &[i32]) -> PushdownStrategy {+ use crate::joins::join_hash_map::JoinHashMapType;+ let keys: ArrayRef = Arc::new(Int32Array::from(keys.to_vec()));+ let mut hashes = vec![0; keys.len()];+ create_hashes([&keys], HASH_JOIN_SEED.random_state(), &mut hashes).unwrap();+ let mut map = JoinHashMapU32::with_capacity(keys.len());+ map.update_from_iter(Box::new(hashes.iter().enumerate()), 0);+ PushdownStrategy::Map(Arc::new(Map::HashMap(Box::new(map))), None)+ }++ /// Splits `keys` across `num_partitions` build partitions as hash+ /// repartitioning would, reporting each non-empty partition with the+ /// strategy `strategy` picks for it.+ #[cfg(not(feature = "force_hash_collisions"))]+ fn routed_partitions(+ acc: &SharedBuildAccumulator,+ num_partitions: usize,+ keys: &[i32],+ strategy: impl Fn(usize, &[i32]) -> PushdownStrategy,+ ) -> Vec<PartitionStatus> {+ let array: ArrayRef = Arc::new(Int32Array::from(keys.to_vec()));+ let mut hashes = vec![0; keys.len()];+ create_hashes(+ [&array],+ acc.repartition_random_state.random_state(),+ &mut hashes,+ )+ .unwrap();+ (0..num_partitions)+ .map(|partition| {+ let held: Vec<i32> = keys+ .iter()+ .zip(&hashes)+ .filter(|(_, hash)| *hash % num_partitions as u64 == partition as u64)+ .map(|(key, _)| *key)+ .collect();+ if held.is_empty() {+ reported(PushdownStrategy::Empty, no_bounds())+ } else {+ reported(strategy(partition, &held), no_bounds())+ }+ })+ .collect()+ }++ /// Partitions that chose an IN list and partitions that chose a hash table+ /// merge into one lookup without a `CASE`: a probe row passes exactly when+ /// the partition its key routes to holds the key.+ #[test]+ #[cfg(not(feature = "force_hash_collisions"))]+ fn partitioned_in_list_and_hash_map_partitions_merge() {+ let acc = make_partitioned_expr_accumulator_for_test(4);+ let build: Vec<i32> = (0..40).step_by(2).collect();++ acc.build_filter(FinalizeInput::Partitioned(routed_partitions(+ &acc,+ 4,+ &build,+ |partition, keys| {+ if partition % 2 == 0 {+ hash_map_of(keys)+ } else {+ in_list(keys)+ }+ },+ )))+ .unwrap();++ assert!(+ current_expr(&acc).downcast_ref::<CaseExpr>().is_none(),+ "expected a merged lookup, got: {}",+ current_expr(&acc)+ );+ let probe: Vec<i32> = (0..40).collect();+ let expected: Vec<bool> = probe.iter().map(|key| key % 2 == 0).collect();+ assert_matches(&acc, &probe, &expected);+ }++ /// IN list partitions whose combined list passes the IN list limits merge+ /// into a hash lookup rather than one oversized IN list, which must still+ /// accept every partition's keys.+ #[test]+ #[cfg(not(feature = "force_hash_collisions"))]+ fn partitioned_in_lists_past_limit_merge_into_hash_lookup() {+ let mut acc = make_partitioned_expr_accumulator_for_test(3);+ acc.in_list_max_values = 2;+ let build: Vec<i32> = (1..=9).collect();++ acc.build_filter(FinalizeInput::Partitioned(routed_partitions(+ &acc,+ 3,+ &build,+ |_, keys| in_list(keys),+ )))+ .unwrap();++ let expr = current_expr(&acc);+ assert!(+ expr.downcast_ref::<CaseExpr>().is_none()+ && expr.downcast_ref::<InListExpr>().is_none(),+ "expected a hash lookup, got: {expr}"+ );+ let probe: Vec<i32> = (0..20).collect();+ let expected: Vec<bool> = probe.iter().map(|key| (1..=9).contains(key)).collect();+ assert_matches(&acc, &probe, &expected);+ }++ /// Evaluates the current filter over probe keys `keys`.+ #[cfg(not(feature = "force_hash_collisions"))]+ fn assert_matches(acc: &SharedBuildAccumulator, keys: &[i32], expected: &[bool]) {+ let probe = RecordBatch::try_new(+ test_probe_schema(),+ vec![Arc::new(Int32Array::from(keys.to_vec()))],+ )+ .unwrap();+ let matched = current_expr(acc)+ .evaluate(&probe)+ .unwrap()+ .into_array(keys.len())+ .unwrap();+ assert_eq!(matched.as_boolean(), &BooleanArray::from(expected.to_vec()));+ }+#[test]fn partitioned_one_real_partition_with_rest_empty_skips_case() {let acc = make_partitioned_expr_accumulator_for_test(3);@@ -1444,6 +1811,8 @@ mod tests {scope: NotInScope::Uncorrelated,output: NotInOutput::Rows,}),+ in_list_max_size: 128 * 1024,+ in_list_max_values: 150,}}@@ -1491,9 +1860,9 @@ mod tests {acc}- /// The IS NULL disjunct must wrap the routed `CASE` as a whole. A probe NULL- /// takes exactly one route, so widening only the branch of the partition that- /// reported the NULL would still drop it on every other route.+ /// The IS NULL disjunct must wrap the merged filter as a whole. A probe NULL+ /// belongs to exactly one partition, so widening only the filter of the+ /// partition that reported the NULL would still drop it everywhere else.#[test]fn partitioned_null_keys_in_one_partition_widen_whole_routed_filter() {let acc = null_equal_partitioned_accumulator(2);@@ -1511,7 +1880,7 @@ mod tests {widened.left().downcast_ref::<IsNullExpr>().is_some(),"expected the IS NULL disjunct first, got: {expr}");- case_expr(widened.right());+ assert_in_list_column_values(widened.right(), "probe_key", 0, &[1, 2]);assert_eq!(format!("{expr}").matches("IS NULL").count(),1,@@ -1659,6 +2028,8 @@ mod tests {scope: NotInScope::Correlated { value_keys },output: NotInOutput::Mark,}),+ in_list_max_size: 128 * 1024,+ in_list_max_values: 150,};let bounds = PartitionBounds::new(diff --git a/datafusion/physical-plan/src/joins/key_range_bitmap.rs b/datafusion/physical-plan/src/joins/key_range_bitmap.rsindex 60631dd2c5..47030f5cad 100644--- a/datafusion/physical-plan/src/joins/key_range_bitmap.rs+++ b/datafusion/physical-plan/src/joins/key_range_bitmap.rs@@ -15,6 +15,19 @@ use crate::joins::array_map::ArrayMap;/// Largest bitmap, in bits (128 KiB), which bounds what one build side can hold.const MAX_BUCKETS: u64 = 1 << 20;+/// One build partition's contribution to [`KeyRangeBitmap::union`].+pub(crate) enum KeyRangePart<'a> {+ /// The partition's own bitmap.+ Bitmap(&'a KeyRangeBitmap),+ /// The partition's build keys, which must lie within its `[min, max]`.+ Keys(&'a dyn Array),+ /// The partition has no bitmap, so any key in `[min, max]` may be present.+ Range {+ min: &'a ScalarValue,+ max: &'a ScalarValue,+ },+}+/// Which parts of a build side's key range hold at least one key.////// Keys map order-preservingly onto buckets spanning `[min, max]`, one bit each,@@ -88,6 +101,104 @@ impl KeyRangeBitmap {})}+ /// One bitmap over `[min, max]` marking every bucket that any of `parts`+ /// may hold a key in. `[min, max]` must cover every part.+ ///+ /// Coarser than the parts' own bitmaps where its buckets are wider, but+ /// never clears a bucket a part may hold a key in. `None` when the key+ /// type has no `u64` ordering, or when nothing can be excluded.+ pub(crate) fn union(+ min: &ScalarValue,+ max: &ScalarValue,+ parts: &[KeyRangePart<'_>],+ ) -> Option<Self> {+ let offset = ArrayMap::key_to_u64(min)?;+ let span = ArrayMap::key_to_u64(max)?.wrapping_sub(offset);+ let buckets = span.saturating_add(1).min(MAX_BUCKETS).next_power_of_two();+ let shift = (0u32..63).find(|s| (span >> s) < buckets).unwrap_or(63);++ // One bit per bucket, in words, so a part's bitmap ORs in a word at a time.+ let mut words =+ MutableBuffer::from_len_zeroed((buckets as usize).div_ceil(64) * 8);+ let reachable = ((span >> shift) + 1) as usize;+ for part in parts {+ match part {+ KeyRangePart::Bitmap(part) => {+ let words = words.typed_data_mut::<u64>();+ // Every key below `part.offset` is outside `[min, max]`, so+ // this wrapping distance is the part's true position.+ let base = part.offset.wrapping_sub(offset);+ let last = (part.span >> part.shift) as usize;+ if part.shift == shift {+ // Same bucket width: part bucket `b` covers union bucket+ // `b + q`, and `b + q + 1` too when the part's buckets+ // start `r` keys into a union bucket. Only the last part+ // bucket, which may be narrower, can stop short of it.+ let q = (base >> shift) as usize;+ let r = base & ((1u64 << shift) - 1);+ or_shifted(words, &part.bits, last + 1, q);+ if r != 0 {+ or_shifted(words, &part.bits, last, q + 1);+ if part.bits.value(last) {+ let lo = base + ((last as u64) << shift);+ let hi = (base + part.span).min(span);+ set_bit_range(+ words,+ (lo >> shift) as usize,+ (hi >> shift) as usize,+ );+ }+ }+ } else {+ let width = (1u64 << part.shift) - 1;+ for bucket in part.bits.slice(0, last + 1).set_indices() {+ let lo = base + ((bucket as u64) << part.shift);+ let hi = (base+ + ((bucket as u64) << part.shift)+ .saturating_add(width)+ .min(part.span))+ .min(span);+ set_bit_range(+ words,+ (lo >> shift) as usize,+ (hi >> shift) as usize,+ );+ }+ }+ }+ KeyRangePart::Keys(keys) => {+ if !ArrayMap::is_supported_type(keys.data_type()) {+ return None;+ }+ set_bucket_bits(*keys, offset, shift, buckets, words.as_slice_mut());+ }+ KeyRangePart::Range { min, max } => {+ let lo = ArrayMap::key_to_u64(min)?.wrapping_sub(offset);+ let hi = ArrayMap::key_to_u64(max)?.wrapping_sub(offset).min(span);+ if lo <= hi {+ set_bit_range(+ words.typed_data_mut::<u64>(),+ (lo >> shift) as usize,+ (hi >> shift) as usize,+ );+ }+ }+ }+ // Bits are only ever added, so once every reachable bucket is set+ // the remaining parts cannot make the bitmap exclude anything.+ if all_set(words.typed_data::<u64>(), reachable) {+ return None;+ }+ }++ Some(Self {+ offset,+ span,+ shift,+ bits: BooleanBuffer::new(words.into(), 0, buckets as usize),+ })+ }+/// One Boolean per `[min[i], max[i]]` container interval: `false` proves no/// build key lies in that container, `None` where a bound is absent or the/// statistics are not integer keys.@@ -144,9 +255,20 @@ impl KeyRangeBitmap {/// One bit per bucket, set for each bucket holding a non-null key of `keys`.fn bucket_bits(keys: &dyn Array, offset: u64, shift: u32, buckets: u64) -> BooleanBuffer {let mut words = MutableBuffer::new_null(buckets as usize);+ set_bucket_bits(keys, offset, shift, buckets, words.as_slice_mut());+ BooleanBuffer::new(words.into(), 0, buckets as usize)+}++/// Sets the bit of each bucket holding a non-null key of `keys` in `bytes`.+fn set_bucket_bits(+ keys: &dyn Array,+ offset: u64,+ shift: u32,+ buckets: u64,+ bytes: &mut [u8],+) {downcast_integer_array!(keys => {- let bytes = words.as_slice_mut();let mut set = |key: u64| {let bucket = (key.wrapping_sub(offset) >> shift).min(buckets - 1);bit_util::set_bit(bytes, bucket as usize);@@ -159,7 +281,45 @@ fn bucket_bits(keys: &dyn Array, offset: u64, shift: u32, buckets: u64) -> Boole}_ => unreachable!("guarded by ArrayMap::is_supported_type"),);- BooleanBuffer::new(words.into(), 0, buckets as usize)+}++/// ORs bits `0..len` of `src` into `words`, moved `by` bits further along.+fn or_shifted(words: &mut [u64], src: &BooleanBuffer, len: usize, by: usize) {+ let (skip, bit) = (by / 64, by % 64);+ let chunks = src.inner().bit_chunks(src.offset(), len);+ for (i, chunk) in chunks.iter_padded().enumerate() {+ if chunk == 0 {+ continue;+ }+ words[skip + i] |= chunk << bit;+ if bit != 0 {+ // A set bit always lands on a bucket, so it never spills past the end.+ if let Some(next) = words.get_mut(skip + i + 1) {+ *next |= chunk >> (64 - bit);+ }+ }+ }+}++/// Sets bits `from..=to` of `words`.+fn set_bit_range(words: &mut [u64], from: usize, to: usize) {+ let (first, last) = (from / 64, to / 64);+ let head = u64::MAX << (from % 64);+ let tail = u64::MAX >> (63 - to % 64);+ if first == last {+ words[first] |= head & tail;+ } else {+ words[first] |= head;+ words[first + 1..last].fill(u64::MAX);+ words[last] |= tail;+ }+}++/// Whether bits `0..len` of `words` are all set.+fn all_set(words: &[u64], len: usize) -> bool {+ let (full, rest) = (len / 64, len % 64);+ words[..full].iter().all(|word| *word == u64::MAX)+ && (rest == 0 || words[full] | (u64::MAX << rest) == u64::MAX)}#[cfg(test)]@@ -201,6 +361,49 @@ mod tests {assert!(kept <= 220, "expected ~200 kept, got {kept}");}+ /// Partitions spanning negative and positive keys, one without a bitmap:+ /// the union must keep every container holding a key of any partition, and+ /// still prune the gaps no partition covers.+ #[test]+ fn union_never_drops_a_partition_key() {+ let clustered: Vec<i64> = (-1_000_000..-999_000).chain(0..1000).collect();+ let sparse: Vec<i64> = (0..400).map(|i| 500_000 + 1000 * i).collect();+ let dense = (-200_000i64, -190_000i64);+ let a = bitmap(&clustered).expect("not saturated");+ let b = bitmap(&sparse).expect("not saturated");+ let (min, max) = (+ ScalarValue::Int64(Some(-1_000_000)),+ ScalarValue::Int64(Some(899_000)),+ );+ let (dense_min, dense_max) = (+ ScalarValue::Int64(Some(dense.0)),+ ScalarValue::Int64(Some(dense.1)),+ );+ let union = KeyRangeBitmap::union(+ &min,+ &max,+ &[+ KeyRangePart::Bitmap(&a),+ KeyRangePart::Bitmap(&b),+ KeyRangePart::Range {+ min: &dense_min,+ max: &dense_max,+ },+ ],+ )+ .expect("not saturated");++ for key in clustered+ .iter()+ .chain(&sparse)+ .chain(&[dense.0, -195_000, dense.1])+ {+ assert!(may(&union, *key, *key), "dropped key {key}");+ }+ assert!(!may(&union, -900_000, -800_000));+ assert!(!may(&union, 100_000, 400_000));+ }+#[test]fn contiguous_keys_no_bitmap() {assert!(bitmap(&(0..5000i64).collect::<Vec<_>>()).is_none());@@ -222,4 +425,189 @@ mod tests {assert!(may(&s, -1, 1));assert!(!may(&s, 1 << 50, 1 << 51));}++ /// `(offset, span, shift, bits)`, to compare bitmaps by value.+ fn parts_of(b: &KeyRangeBitmap) -> (u64, u64, u32, &BooleanBuffer) {+ (b.offset, b.span, b.shift, &b.bits)+ }++ /// One randomly generated scenario, see [`union_fuzz`].+ fn run_union_scenario(seed: u64) {+ use rand::rngs::StdRng;+ use rand::{Rng, SeedableRng};++ let mut rng = StdRng::seed_from_u64(seed);+ // Domain widths from one key per bucket to many keys per bucket, so+ // the union's buckets are sometimes exact and sometimes coarse.+ let width = [64i64, 1 << 12, 1 << 24, 1 << 40][rng.random_range(0..4)];+ let start = rng.random_range(-(1i64 << 50)..(1i64 << 50));+ let key = |rng: &mut StdRng| start + rng.random_range(0..width);+ let keys_array =+ |keys: &[i64]| -> ArrayRef { Arc::new(Int64Array::from(keys.to_vec())) };+ let scalar = |key: i64| ScalarValue::Int64(Some(key));++ enum Part {+ Keys(ArrayRef),+ Range(ScalarValue, ScalarValue),+ Bitmap(KeyRangeBitmap),+ }+ // Each part, and every key any part may hold.+ let mut parts = vec![];+ let mut all_keys = vec![];+ let mut has_bitmap = false;+ for _ in 0..rng.random_range(1..5) {+ if rng.random_bool(1.0 / 3.0) {+ // Narrow, so every key it may hold can be listed.+ let lo = key(&mut rng);+ let hi = lo + rng.random_range(0..64);+ parts.push(Part::Range(scalar(lo), scalar(hi)));+ all_keys.extend(lo..=hi);+ continue;+ }+ let keys: Vec<i64> = (0..rng.random_range(1..50))+ .map(|_| key(&mut rng))+ .collect();+ let (lo, hi) = (*keys.iter().min().unwrap(), *keys.iter().max().unwrap());+ let bitmap = rng+ .random_bool(0.5)+ .then(|| {+ KeyRangeBitmap::try_new(+ &keys_array(&keys),+ &scalar(lo),+ &scalar(hi),+ 0,+ )+ })+ .flatten();+ parts.push(match bitmap {+ Some(bitmap) => {+ has_bitmap = true;+ Part::Bitmap(bitmap)+ }+ None => Part::Keys(keys_array(&keys)),+ });+ all_keys.extend(keys);+ }+ let min = ScalarValue::Int64(all_keys.iter().min().copied());+ let max = ScalarValue::Int64(all_keys.iter().max().copied());+ let context = format!(+ "seed {seed}: width {width}, {} parts, {} keys",+ parts.len(),+ all_keys.len()+ );++ let union = KeyRangeBitmap::union(+ &min,+ &max,+ &parts+ .iter()+ .map(|part| match part {+ Part::Keys(keys) => KeyRangePart::Keys(keys.as_ref()),+ Part::Range(min, max) => KeyRangePart::Range { min, max },+ Part::Bitmap(bitmap) => KeyRangePart::Bitmap(bitmap),+ })+ .collect::<Vec<_>>(),+ );+ // Reference: one bitmap built straight from every key any part may hold.+ let exact = KeyRangeBitmap::try_new(&keys_array(&all_keys), &min, &max, 0);++ // Never excludes a key any part may hold.+ if let Some(union) = &union {+ for key in &all_keys {+ assert!(may(union, *key, *key), "{context}: dropped key {key}");+ }+ }+ if has_bitmap {+ // A part's bitmap can only widen buckets: every bucket the exact+ // bitmap sets, the union sets too.+ match (&union, &exact) {+ (_, None) => assert!(union.is_none(), "{context}"),+ (None, Some(_)) => {}+ (Some(union), Some(exact)) => {+ assert_eq!(+ (union.offset, union.span, union.shift),+ (exact.offset, exact.span, exact.shift),+ "{context}"+ );+ for bucket in exact.bits.set_indices() {+ assert!(union.bits.value(bucket), "{context}: bucket {bucket}");+ }+ }+ }+ // Each set bucket of a part's bitmap stands for its whole key range,+ // so the union equals the one over those ranges given as `Range`s.+ let ranges: Vec<(ScalarValue, ScalarValue)> = parts+ .iter()+ .flat_map(|part| match part {+ Part::Bitmap(bitmap) => bitmap+ .bits+ .set_indices()+ .map(|bucket| {+ let lo = (bucket as u64) << bitmap.shift;+ let hi = lo+ .saturating_add((1u64 << bitmap.shift) - 1)+ .min(bitmap.span);+ let key =+ |at: u64| scalar(bitmap.offset.wrapping_add(at) as i64);+ (key(lo), key(hi))+ })+ .collect(),+ _ => vec![],+ })+ .collect();+ let expanded = KeyRangeBitmap::union(+ &min,+ &max,+ &parts+ .iter()+ .filter_map(|part| match part {+ Part::Keys(keys) => Some(KeyRangePart::Keys(keys.as_ref())),+ Part::Range(min, max) => Some(KeyRangePart::Range { min, max }),+ Part::Bitmap(_) => None,+ })+ .chain(+ ranges+ .iter()+ .map(|(min, max)| KeyRangePart::Range { min, max }),+ )+ .collect::<Vec<_>>(),+ );+ assert_eq!(+ union.as_ref().map(parts_of),+ expanded.as_ref().map(parts_of),+ "{context}: bitmap parts differ from their bucket ranges"+ );+ } else {+ // Keys and ranges mark exactly the buckets their keys fall in.+ assert_eq!(+ union.as_ref().map(parts_of),+ exact.as_ref().map(parts_of),+ "{context}"+ );+ }+ }++ /// Randomized partitions against [`KeyRangeBitmap::union`]'s contract: it+ /// never excludes a key any part may hold, equals the bitmap built from all+ /// parts' keys when no part brings its own bitmap, and only widens that+ /// bitmap's buckets when one does.+ ///+ /// `DATAFUSION_KEY_RANGE_UNION_FUZZ_ITERATIONS` and+ /// `DATAFUSION_KEY_RANGE_UNION_FUZZ_SEED` select how many scenarios run and+ /// from which seed; a failure names its seed so it can be replayed.+ #[test]+ fn union_fuzz() {+ let env_u64 = |name: &str, default: u64| {+ std::env::var(name)+ .ok()+ .and_then(|v| v.parse().ok())+ .unwrap_or(default)+ };+ let first_seed = env_u64("DATAFUSION_KEY_RANGE_UNION_FUZZ_SEED", 0);+ let iterations = env_u64("DATAFUSION_KEY_RANGE_UNION_FUZZ_ITERATIONS", 256);+ // `saturating_add` so a replay seed near `u64::MAX` cannot overflow.+ for seed in first_seed..first_seed.saturating_add(iterations) {+ run_union_scenario(seed);+ }+ }}diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rsindex 0ecea79ef4..929f52fd15 100644--- a/datafusion/physical-plan/src/repartition/mod.rs+++ b/datafusion/physical-plan/src/repartition/mod.rs@@ -783,6 +783,20 @@ impl RangeExpr {pub fn sort_options(&self) -> &[SortOptions] {self.router.sort_options()}++ /// Appends the partition of each row of `arrays`, the evaluated+ /// `on_columns`, to `partition_ids`.+ pub(crate) fn route_partition_ids(+ &self,+ arrays: &[arrow::array::ArrayRef],+ partition_ids: &mut Vec<u64>,+ ) -> Result<()> {+ if self.router.split_points().is_empty() {+ partition_ids.resize(partition_ids.len() + arrays[0].len(), 0);+ return Ok(());+ }+ self.router.route_partition_ids(arrays, partition_ids)+ }}impl Display for RangeExpr {diff --git a/datafusion/proto/tests/cases/plans/exprs.rs b/datafusion/proto/tests/cases/plans/exprs.rsindex a76d40325a..8c193283ad 100644--- a/datafusion/proto/tests/cases/plans/exprs.rs+++ b/datafusion/proto/tests/cases/plans/exprs.rs@@ -446,10 +446,11 @@ fn roundtrip_hash_table_lookup_expr_to_lit() -> Result<()> {let lookup_expr: Arc<dyn PhysicalExpr> = Arc::new(HashTableLookupExpr::new(on_columns,datafusion::physical_plan::joins::SeededRandomState::with_seed(0),- hash_map,+ None,+ vec![Some(hash_map)],"test_lookup".to_string(),None,- ));+ )?);// Create a filter with the lookup expressionlet filter = Arc::new(FilterExec::try_new(lookup_expr, input)?);diff --git a/datafusion/pruning/src/pruning_predicate.rs b/datafusion/pruning/src/pruning_predicate.rsindex aa48b75ae7..2e3c84673f 100644--- a/datafusion/pruning/src/pruning_predicate.rs+++ b/datafusion/pruning/src/pruning_predicate.rs@@ -7470,7 +7470,7 @@ mod tests {fn hash_lookup(column: &Arc<dyn PhysicalExpr>,keys: &[i64],- ) -> Arc<dyn PhysicalExpr> {+ ) -> Result<Arc<dyn PhysicalExpr>> {let array: ArrayRef = Arc::new(Int64Array::from(keys.to_vec()));let bitmap = KeyRangeBitmap::try_new(&array,@@ -7478,26 +7478,28 @@ mod tests {&ScalarValue::Int64(Some(*keys.iter().max().unwrap())),keys.len(),);- Arc::new(HashTableLookupExpr::new(+ Ok(Arc::new(HashTableLookupExpr::new(vec![Arc::clone(column)],SeededRandomState::with_seed(1),- Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(1)))),+ None,+ vec![Some(Arc::new(Map::HashMap(Box::new(+ JoinHashMapU32::with_capacity(1),+ ))))],"hash_lookup".to_string(),bitmap.map(Arc::new),- ))+ )?))}#[test]- fn test_hash_lookup_pruning_via_min_max() {+ fn test_hash_lookup_pruning_via_min_max() -> Result<()> {let schema = Arc::new(Schema::new(vec![Field::new("b", DataType::Int64, true)]));let column: Arc<dyn PhysicalExpr> = Arc::new(phys_expr::Column::new("b", 0));// Gapped keys: 10,20,30 then a far-away 100000, so the bitmap is sparse.- let lookup = hash_lookup(&column, &[10, 20, 30, 100_000]);+ let lookup = hash_lookup(&column, &[10, 20, 30, 100_000])?;let predicate = PruningPredicateBuilder::new().with_file_schema(Arc::clone(&schema))- .try_build(lookup)- .unwrap();+ .try_build(lookup)?;// An opaque lookup yields no literal guarantees, so `contained()` is never// consulted and every exclusion below comes from the min/max rewrite.@@ -7511,10 +7513,11 @@ mod tests {),);- let result = predicate.prune(&statistics).unwrap();+ let result = predicate.prune(&statistics)?;// Containers 0 and 2 sit in the gap between 30 and 100000; container 1// holds 20. The bitmap excludes the first and third.assert_eq!(result, vec![false, true, false]);+ Ok(())}#[rstest]@@ -7528,18 +7531,17 @@ mod tests {#[case] file_type: DataType,#[case] cast_type: DataType,#[case] pruned: bool,- ) {+ ) -> Result<()> {let schema =Arc::new(Schema::new(vec![Field::new("b", file_type.clone(), true)]));let column: Arc<dyn PhysicalExpr> = Arc::new(phys_expr::Column::new("b", 0));let cast_column: Arc<dyn PhysicalExpr> =Arc::new(phys_expr::CastExpr::new(column, cast_type, None));- let lookup = hash_lookup(&cast_column, &[10, 20, 30, 100_000]);+ let lookup = hash_lookup(&cast_column, &[10, 20, 30, 100_000])?;let predicate = PruningPredicateBuilder::new().with_file_schema(Arc::clone(&schema))- .try_build(lookup)- .unwrap();+ .try_build(lookup)?;let to_file_type = |values: Vec<Option<i64>>| -> ArrayRef {let values: ArrayRef = Arc::new(values.into_iter().collect::<Int64Array>());@@ -7552,7 +7554,7 @@ mod tests {.with_max(to_file_type(vec![Some(8000), Some(25), Some(60000)])),);- let result = predicate.prune(&statistics).unwrap();+ let result = predicate.prune(&statistics)?;// Containers 0 and 2 sit in the gap between 30 and 100000; container 1// holds 20. A working cast excludes the first and third; a declined one// falls back to keeping every container.@@ -7562,10 +7564,11 @@ mod tests {vec![true, true, true]};assert_eq!(result, expected);+ Ok(())}#[test]- fn test_partition_routed_hash_lookup_pruning() {+ fn test_partition_routed_hash_lookup_pruning() -> Result<()> {let schema = Arc::new(Schema::new(vec![Field::new("b", DataType::Int32, true)]));let column: Arc<dyn PhysicalExpr> = Arc::new(phys_expr::Column::new("b", 0));let literal = |value: ScalarValue| -> Arc<dyn PhysicalExpr> {@@ -7573,29 +7576,25 @@ mod tests {};// One branch per build partition, each holding its own slice of the build// side, as `build_partitioned_filter` produces them.- let case: Arc<dyn PhysicalExpr> = Arc::new(- phys_expr::CaseExpr::try_new(- Some(literal(ScalarValue::UInt64(Some(0)))),- vec![- (- literal(ScalarValue::UInt64(Some(0))),- hash_lookup(&column, &[10, 20, 30]),- ),- (- literal(ScalarValue::UInt64(Some(1))),- hash_lookup(&column, &[40, 50, 60]),- ),- ],- // Partitions with no build rows reject everything routed to them.- Some(literal(ScalarValue::Boolean(Some(false)))),- )- .unwrap(),- );+ let case: Arc<dyn PhysicalExpr> = Arc::new(phys_expr::CaseExpr::try_new(+ Some(literal(ScalarValue::UInt64(Some(0)))),+ vec![+ (+ literal(ScalarValue::UInt64(Some(0))),+ hash_lookup(&column, &[10, 20, 30])?,+ ),+ (+ literal(ScalarValue::UInt64(Some(1))),+ hash_lookup(&column, &[40, 50, 60])?,+ ),+ ],+ // Partitions with no build rows reject everything routed to them.+ Some(literal(ScalarValue::Boolean(Some(false)))),+ )?);let predicate = PruningPredicateBuilder::new().with_file_schema(Arc::clone(&schema))- .try_build(case)- .unwrap();+ .try_build(case)?;let statistics = TestStatistics::new().with("b",@@ -7605,10 +7604,11 @@ mod tests {),);- let result = predicate.prune(&statistics).unwrap();+ let result = predicate.prune(&statistics)?;// Container 1 ([15,25]) holds 20 from the first branch and container 2// ([45,55]) holds 50 from the second, so a branch may match in each. The// other two intersect neither branch, nor the `ELSE`.assert_eq!(result, vec![false, true, true, false]);+ Ok(())}}diff --git a/datafusion/sqllogictest/test_files/range_partitioning.slt b/datafusion/sqllogictest/test_files/range_partitioning.sltindex 4573f28b7f..f389e265f8 100644--- a/datafusion/sqllogictest/test_files/range_partitioning.slt+++ b/datafusion/sqllogictest/test_files/range_partitioning.slt@@ -1886,10 +1886,9 @@ SELECT range_key, SUM(value) FROM (########### TEST 48: Hash Join Dynamic Filter Pushdown on Compatible Range Inputs-# The Parquet-backed probe accepts the partition-routed dynamic filter. Matching-# Range split points keep build filter i aligned with probe partition i. The-# build has rows only in partitions 0 and 2, so the sparse runtime CASE includes-# only those branches and lets empty partitions fall through to false.+# The Parquet-backed probe accepts the dynamic filter. The build has rows only in+# partitions 0 and 2, and both chose an IN list, so they merge into one IN list+# with combined bounds instead of a per-partition CASE.##########statement ok@@ -1930,7 +1929,7 @@ JOIN range_partitioned p ON b.range_key = p.range_key;Plan with Metrics01)HashJoinExec: mode=Partitioned<slt:ignore>metrics=[output_rows=2,<slt:ignore>]02)--DataSourceExec: <slt:ignore>file_type=parquet, predicate=range_key@0 IN (SET) ([5, 20])<slt:ignore>metrics=[output_rows=2,<slt:ignore>]-03)--DataSourceExec: <slt:ignore>file_type=parquet, predicate=DynamicFilter [ CASE range_partition WHEN 0 THEN range_key@0 >= 5 AND range_key@0 <= 5 AND range_key@0 IN (SET) ([5]) WHEN 2 THEN range_key@0 >= 20 AND range_key@0 <= 20 AND range_key@0 IN (SET) ([20]) ELSE false END ]<slt:ignore>metrics=[output_rows=2,<slt:ignore>]+03)--DataSourceExec: <slt:ignore>file_type=parquet, predicate=DynamicFilter [ range_key@0 >= 5 AND range_key@0 <= 20 AND range_key@0 IN (SET) ([5, 20]) ]<slt:ignore>metrics=[output_rows=2,<slt:ignore>]query IIISELECT b.range_key, b.value, p.valuediff --git a/xtask/src/ci_steps.rs b/xtask/src/ci_steps.rsindex a23dd563af..4ece399a79 100644--- a/xtask/src/ci_steps.rs+++ b/xtask/src/ci_steps.rs@@ -15,8 +15,12 @@ type StepRunner = fn(&StepContext, &[String]) -> StepResult<CommandSpec>;const CI_COMMAND: &str = "cargo xtask ci step";const CI_SHORTCUT: &str = "cargo ci-step";const TPCH_DATA_DIR: &str = "datafusion/sqllogictest/test_files/tpch/data";-/// The `datafusion-physical-plan` unit test that the `spill-pool-fuzz` step runs.-const SPILL_POOL_FUZZ_TEST: &str = "spill::spill_pool::tests::spill_pool_scenario_fuzz";+/// The `datafusion-physical-plan` unit tests that the `fuzz` step runs.+const FUZZ_TESTS: &[&str] = &[+ "spill::spill_pool::tests::spill_pool_scenario_fuzz",+ "joins::hash_join::partitioned_hash_eval::tests::routed_lookup_fuzz",+ "joins::key_range_bitmap::tests::union_fuzz",+];/// Metadata and implementation for one CI step.///@@ -40,7 +44,7 @@ static CI_STEPS: &[StepInfo] = &[},StepInfo {command: "test",- help_usage: "test <workspace|slt-coverage|cli|doctest|ffi|microbenchmarks|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|spill-pool-fuzz|hash-collisions|sqlite> [--explain]",+ help_usage: "test <workspace|slt-coverage|cli|doctest|ffi|microbenchmarks|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|fuzz|hash-collisions|sqlite> [--explain]",help_examples: &["test cli", "test workspace"],help_description: "Run a CI test suite",runner: StepContext::run_test,@@ -459,10 +463,10 @@ impl StepContext {]).env("RUST_BACKTRACE", "1");}- "spill-pool-fuzz" => {+ "fuzz" => {cmd_builder.doc(- "Run more random spill pool fuzzer scenarios than the `workspace`\n\- step. This takes about a minute.",+ "Run more random fuzzer scenarios than the `workspace` step. This\n\+ takes about a minute for the spill pool, seconds for the others.",);cmd_builder.program("cargo")@@ -475,10 +479,12 @@ impl StepContext {"--lib","--","--exact",- SPILL_POOL_FUZZ_TEST,])+ .args(FUZZ_TESTS).env("RUST_BACKTRACE", "1")- .env("DATAFUSION_SPILL_POOL_FUZZ_ITERATIONS", "1000");+ .env("DATAFUSION_SPILL_POOL_FUZZ_ITERATIONS", "1000")+ .env("DATAFUSION_ROUTED_LOOKUP_FUZZ_ITERATIONS", "10000")+ .env("DATAFUSION_KEY_RANGE_UNION_FUZZ_ITERATIONS", "10000");}"hash-collisions" => {cmd_builder.doc(@@ -559,7 +565,7 @@ mod tests {use super::*;use std::ffi::OsStr;- const TEST_USAGE: &str = "usage: cargo xtask ci step test <workspace|slt-coverage|cli|doctest|ffi|microbenchmarks|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|spill-pool-fuzz|hash-collisions|sqlite> [--explain]";+ const TEST_USAGE: &str = "usage: cargo xtask ci step test <workspace|slt-coverage|cli|doctest|ffi|microbenchmarks|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|fuzz|hash-collisions|sqlite> [--explain]";fn args(args: &[&str]) -> Vec<String> {args.iter().map(|arg| (*arg).to_string()).collect()@@ -609,22 +615,24 @@ mod tests {}/// A filter that matches no test passes without running anything, so a- /// renamed or moved spill pool fuzzer would silently stop running in CI.+ /// renamed or moved fuzzer would silently stop running in CI.#[test]- fn spill_pool_fuzz_test_exists() {- let (module, test) = SPILL_POOL_FUZZ_TEST- .rsplit_once("::tests::")- .expect("test path has a `tests` module");- let file = Path::new(env!("CARGO_MANIFEST_DIR"))- .join("../datafusion/physical-plan/src")- .join(format!("{}.rs", module.replace("::", "/")));- let source = std::fs::read_to_string(&file)- .unwrap_or_else(|e| panic!("cannot read {}: {e}", file.display()));- assert!(- source.contains("mod tests {") && source.contains(&format!("fn {test}(")),- "{} has no test `{test}` in `mod tests`; update SPILL_POOL_FUZZ_TEST",- file.display()- );+ fn fuzz_tests_exist() {+ for path in FUZZ_TESTS {+ let (module, test) = path+ .rsplit_once("::tests::")+ .expect("test path has a `tests` module");+ let file = Path::new(env!("CARGO_MANIFEST_DIR"))+ .join("../datafusion/physical-plan/src")+ .join(format!("{}.rs", module.replace("::", "/")));+ let source = std::fs::read_to_string(&file)+ .unwrap_or_else(|e| panic!("cannot read {}: {e}", file.display()));+ assert!(+ source.contains("mod tests {") && source.contains(&format!("fn {test}(")),+ "{} has no test `{test}` in `mod tests`; update FUZZ_TESTS",+ file.display()+ );+ }}/// The `extended` step runs only the `extended_tests` test target, which@@ -943,20 +951,22 @@ cargo test \},},Cmd {- command: &["test", "spill-pool-fuzz"],+ command: &["test", "fuzz"],expected: |actual| {insta::assert_snapshot!(actual, @r"-# Run more random spill pool fuzzer scenarios than the `workspace`-# step. This takes about a minute.+# Run more random fuzzer scenarios than the `workspace` step. This+# takes about a minute for the spill pool, seconds for the others.cd /workspace && \+DATAFUSION_KEY_RANGE_UNION_FUZZ_ITERATIONS=10000 \+DATAFUSION_ROUTED_LOOKUP_FUZZ_ITERATIONS=10000 \DATAFUSION_SPILL_POOL_FUZZ_ITERATIONS=1000 \RUST_BACKTRACE=1 \cargo test \--profile ci \-p datafusion-physical-plan \--lib \--- --exact spill::spill_pool::tests::spill_pool_scenario_fuzz+-- --exact spill::spill_pool::tests::spill_pool_scenario_fuzz joins::hash_join::partitioned_hash_eval::tests::routed_lookup_fuzz joins::key_range_bitmap::tests::union_fuzz")},},