Skip to content

Apache DataFusion SQL Query Engine

feat(perf): compute routing and lookup hashes once (#26249)

WeblWable77a370b9fe8
diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.yml
index a240f2c3e2..f632b9f2ef 100644
--- a/.github/workflows/extended.yml
+++ b/.github/workflows/extended.yml
@@ -84,8 +84,8 @@ jobs:
df -h
- name: Run tests (excluding doctests)
run: cargo xtask ci step test extended
- - name: Run spill pool fuzzer with more iterations
- run: cargo xtask ci step test spill-pool-fuzz
+ - name: Run fuzzers with more iterations
+ run: cargo xtask ci step test fuzz
- parallel:
- name: Verify Working Directory Clean
run: git diff --exit-code
diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs
index eeba0dc220..6b4f47227c 100644
--- a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs
+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs
@@ -1152,7 +1152,7 @@ async fn test_hashjoin_dynamic_filter_pushdown_partitioned() {
- RepartitionExec: partitioning=Hash([a@0, b@1], 12), input_partitions=1
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
- RepartitionExec: partitioning=Hash([a@0, b@1], 12), input_partitions=1
- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, e], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ CASE hash_repartition % 12 WHEN 5 THEN a@0 >= ab AND a@0 <= ab AND b@1 >= bb AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:ab,c1:bb}]) WHEN 8 THEN a@0 >= aa AND a@0 <= aa AND b@1 >= ba AND b@1 <= ba AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}]) ELSE false END ]
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, e], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:ab,c1:bb}, {c0:aa,c1:ba}]) ]
"
);
@@ -1368,7 +1368,7 @@ async fn test_hashjoin_dynamic_filter_pushdown_range_partitioned() {
- RepartitionExec: partitioning=Range([a@0 ASC, b@1 ASC], [(aa, bb)], 2), input_partitions=1
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
- RepartitionExec: partitioning=Range([a@0 ASC, b@1 ASC], [(aa, bb)], 2), input_partitions=1
- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, e], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ CASE range_partition WHEN 0 THEN a@0 >= aa AND a@0 <= aa AND b@1 >= ba AND b@1 <= ba AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}]) WHEN 1 THEN a@0 >= ab AND a@0 <= ab AND b@1 >= bb AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:ab,c1:bb}]) ELSE false END ]
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, e], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ]
"
);
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs
index 8273a99438..20a7def8f3 100644
--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs
@@ -1688,6 +1688,7 @@ impl ExecutionPlan for HashJoinExec {
repartition_random_state,
self.null_equality,
not_in,
+ &context.session_config().options().optimizer,
))
})))
})
@@ -9897,19 +9898,8 @@ mod tests {
dynamic_filter.wait_complete().await;
let filter = dynamic_filter.current()?.to_string();
- // The aggregated build-side NULL must widen the routed filter once, at the
- // top: `c2 IS NULL OR CASE hash(c2) % 4 WHEN ... END`.
- //
- // Under `force_hash_collisions` every key hashes to the same value, so the
- // repartition routes all build rows to a single partition and
- // `build_partitioned_filter` collapses to that partition's filter with no
- // routing `CASE`. The NULL widening this test guards is asserted either way.
- if cfg!(not(feature = "force_hash_collisions")) {
- assert!(
- filter.contains("CASE"),
- "expected a routed partitioned filter, got: {filter}"
- );
- }
+ // The aggregated build-side NULL must widen the whole merged filter once,
+ // at the top: `c2 IS NULL OR <membership over every partition>`.
assert!(
filter.starts_with("c2@0 IS NULL OR"),
"expected the IS NULL disjunct to wrap the whole routed filter, got: {filter}"
diff --git a/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rs b/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rs
index b580687e15..66c7c75f61 100644
--- a/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rs
@@ -7,21 +7,20 @@
use std::{fmt::Display, hash::Hash, sync::Arc};
use arrow::{
- array::{ArrayRef, UInt64Array},
+ array::{ArrayRef, BooleanArray, BooleanBufferBuilder, UInt64Array},
datatypes::{DataType, Schema},
record_batch::RecordBatch,
};
-use datafusion_common::Result;
use datafusion_common::hash_utils::RandomState;
use datafusion_common::hash_utils::{create_hashes, with_hashes};
-#[cfg(feature = "proto")]
-use datafusion_common::internal_err;
+use datafusion_common::{Result, internal_err};
use datafusion_expr::ColumnarValue;
use datafusion_expr_common::dyn_eq::DynHash;
use datafusion_physical_expr_common::physical_expr::{PhysicalExpr, PhysicalExprRef};
use crate::joins::Map;
use crate::joins::key_range_bitmap::KeyRangeBitmap;
+use crate::repartition::RangeExpr;
/// RandomState wrapper that preserves the seed used to create it.
///
@@ -261,29 +260,100 @@ impl HashExpr {
/// Physical expression that checks join keys in a [`Map`] (hash table or array map).
///
-/// Returns a [`BooleanArray`](arrow::array::BooleanArray) indicating if join keys (from `on_columns`) exist in the map.
+/// Returns a [`BooleanArray`] indicating if join keys (from `on_columns`) exist in the map.
// TODO: rename to MapLookupExpr
pub struct HashTableLookupExpr {
/// Columns in the ON clause used to compute the join key for lookups
on_columns: Vec<PhysicalExprRef>,
/// Random state for hashing (with seeds preserved for serialization)
random_state: SeededRandomState,
- /// Map to check against (hash table or array map)
- map: Arc<Map>,
+ /// Map(s) to check against
+ lookup: Lookup,
/// Description for display
description: String,
/// Shared pruning-only view of the build side's key range
pruning_bitmap: Option<Arc<KeyRangeBitmap>>,
}
+
+/// The build side a [`HashTableLookupExpr`] checks probe keys against.
+#[derive(Clone)]
+enum Lookup {
+ /// One map holding every build key, probed directly with no routing.
+ Single(Arc<Map>),
+ /// One hash table per build partition of a `Partitioned` join, `None` where
+ /// the partition is empty. A key is only ever built into the partition its
+ /// key routes to, so each probe row is looked up in that partition's table
+ /// alone: no `CASE` over per-partition filters, and no merged table.
+ Routed {
+ router: PartitionRouter,
+ maps: Arc<[Option<Arc<Map>>]>,
+ },
+}
+
+/// Assigns probe rows to the build partition their key was repartitioned to.
+///
+/// # Public Only for Internal Use:
+/// Public only because [`HashTableLookupExpr::new`] takes it; not part of the
+/// supported public API.
+#[doc(hidden)]
+#[derive(Clone)]
+pub enum PartitionRouter {
+ /// `hash(keys) % partitions`, as hash repartitioning routes rows.
+ Hash(SeededRandomState),
+ /// The probe side's range partitioning.
+ Range(Arc<RangeExpr>),
+}
+
+impl PartitionRouter {
+ /// Writes the partition of each row of `keys` to `partition_ids`, which
+ /// holds one slot per row.
+ fn route(
+ &self,
+ keys: &[ArrayRef],
+ partitions: usize,
+ partition_ids: &mut Vec<u64>,
+ ) -> Result<()> {
+ match self {
+ PartitionRouter::Hash(random_state) => {
+ partition_ids.resize(keys[0].len(), 0);
+ create_hashes(keys, random_state.random_state(), partition_ids)?;
+ let partitions = partitions as u64;
+ for id in partition_ids.iter_mut() {
+ *id %= partitions;
+ }
+ Ok(())
+ }
+ PartitionRouter::Range(range) => {
+ range.route_partition_ids(keys, partition_ids)
+ }
+ }
+ }
+
+ fn seed(&self) -> Option<u64> {
+ match self {
+ PartitionRouter::Hash(random_state) => Some(random_state.seed()),
+ PartitionRouter::Range(_) => None,
+ }
+ }
+}
+
impl HashTableLookupExpr {
- /// Create a new HashTableLookupExpr
+ /// Create a new HashTableLookupExpr that looks each probe row up in the
+ /// map of the build partition `router` assigns it to.
///
/// # Arguments
/// * `on_columns` - Columns in the ON clause used to compute the join key
/// * `random_state` - SeededRandomState for hashing
- /// * `map` - Map to check membership (hash table or array map)
+ /// * `router` - Routes each probe row to the partition its key was
+ /// repartitioned to; `None` when `maps` is a single map
+ /// * `maps` - One entry per build partition, `None` for an empty partition
/// * `description` - Description for debugging
- /// * `pruning_bitmap` - key-range summary for pruning only, or `None`
+ /// * `pruning_bitmap` - key-range summary covering every partition's keys
+ /// for pruning only, or `None`
+ ///
+ /// A single non-empty map is probed directly without routing, so it may be
+ /// any [`Map`] and `router` is ignored. Otherwise `router` is required and
+ /// every map must be a [`Map::HashMap`].
///
/// # Public Only for Internal Use:
/// `datafusion-proto` tests require this constructor, but it is not part of
@@ -292,23 +362,102 @@ impl HashTableLookupExpr {
pub fn new(
on_columns: Vec<PhysicalExprRef>,
random_state: SeededRandomState,
- map: Arc<Map>,
+ router: Option<PartitionRouter>,
+ maps: Vec<Option<Arc<Map>>>,
description: String,
pruning_bitmap: Option<Arc<KeyRangeBitmap>>,
- ) -> Self {
- Self {
+ ) -> Result<Self> {
+ let lookup = match (router, maps.as_slice()) {
+ // Every row routes to the only partition, so skip routing.
+ (_, [Some(map)]) => Lookup::Single(Arc::clone(map)),
+ (None, _) => {
+ return internal_err!(
+ "a lookup over anything but one non-empty map needs a partition router"
+ );
+ }
+ (Some(router), _) => {
+ if maps
+ .iter()
+ .flatten()
+ .any(|map| !matches!(map.as_ref(), Map::HashMap(_)))
+ {
+ return internal_err!(
+ "routed lookups only support hash table partitions"
+ );
+ }
+ Lookup::Routed {
+ router,
+ maps: maps.into(),
+ }
+ }
+ };
+ Ok(Self {
on_columns,
random_state,
- map,
+ lookup,
description,
pruning_bitmap,
- }
+ })
}
/// Which parts of this build side's key range hold keys, for container pruning.
pub fn pruning_bitmap(&self) -> Option<&Arc<KeyRangeBitmap>> {
self.pruning_bitmap.as_ref()
}
+
+ /// Membership of each row of `keys` in the hash table of its routed partition.
+ fn evaluate_routed(
+ &self,
+ keys: &[ArrayRef],
+ router: &PartitionRouter,
+ maps: &[Option<Arc<Map>>],
+ ) -> Result<BooleanArray> {
+ let num_rows = keys[0].len();
+ let mut partition_ids = Vec::with_capacity(num_rows);
+ router.route(keys, maps.len(), &mut partition_ids)?;
+
+ with_hashes(keys, self.random_state.random_state(), |hashes| {
+ // Counting sort of the rows by partition, so each table is probed
+ // once per batch over a contiguous run of its rows' hashes.
+ let mut starts = vec![0usize; maps.len() + 1];
+ for &id in &partition_ids {
+ starts[id as usize + 1] += 1;
+ }
+ for i in 0..maps.len() {
+ starts[i + 1] += starts[i];
+ }
+ let mut next = starts.clone();
+ let mut rows = vec![0u32; num_rows];
+ let mut sorted_hashes = vec![0u64; num_rows];
+ for (row, (&id, &hash)) in partition_ids.iter().zip(hashes).enumerate() {
+ let slot = &mut next[id as usize];
+ rows[*slot] = row as u32;
+ sorted_hashes[*slot] = hash;
+ *slot += 1;
+ }
+
+ let mut matched = BooleanBufferBuilder::new(num_rows);
+ matched.append_n(num_rows, false);
+ for (partition, map) in maps.iter().enumerate() {
+ // An empty partition holds no key, so its rows stay unmatched.
+ let Some(map) = map else { continue };
+ let Map::HashMap(map) = map.as_ref() else {
+ return internal_err!(
+ "routed lookups only support hash table partitions"
+ );
+ };
+ let run = starts[partition]..starts[partition + 1];
+ if run.is_empty() {
+ continue;
+ }
+ let hits = map.contain_hashes(&sorted_hashes[run.clone()]);
+ for hit in hits.values().set_indices() {
+ matched.set_bit(rows[run.start + hit] as usize, true);
+ }
+ }
+ Ok(BooleanArray::new(matched.finish(), None))
+ })
+ }
}
impl std::fmt::Debug for HashTableLookupExpr {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -319,7 +468,21 @@ impl std::fmt::Debug for HashTableLookupExpr {
.collect::<Vec<_>>()
.join(", ");
let seed = self.random_state.seed();
- write!(f, "{}({cols}, [{seed}])", self.description)
+ match &self.lookup {
+ Lookup::Single(_) => write!(f, "{}({cols}, [{seed}])", self.description),
+ Lookup::Routed { router, maps } => {
+ let router = match router.seed() {
+ Some(routing_seed) => format!("hash [{routing_seed}]"),
+ None => "range".to_string(),
+ };
+ write!(
+ f,
+ "{}({cols}, [{seed}], routed by {router} over {} partitions)",
+ self.description,
+ maps.len()
+ )
+ }
+ }
}
}
@@ -329,7 +492,7 @@ impl Hash for HashTableLookupExpr {
self.description.hash(state);
self.random_state.seed().hash(state);
// Hash immutable map identity, consistently with pointer equality below.
- Arc::as_ptr(&self.map).hash(state);
+ self.lookup.identity().hash(state);
}
}
@@ -340,12 +503,23 @@ impl PartialEq for HashTableLookupExpr {
self.on_columns == other.on_columns
&& self.description == other.description
&& self.random_state.seed() == other.random_state.seed()
- && Arc::ptr_eq(&self.map, &other.map)
+ && self.lookup.identity() == other.lookup.identity()
}
}
impl Eq for HashTableLookupExpr {}
+impl Lookup {
+ /// Identity of the immutable build-side maps. A routed lookup's router is
+ /// created together with its maps, so the maps identify it as well.
+ fn identity(&self) -> *const () {
+ match self {
+ Lookup::Single(map) => Arc::as_ptr(map).cast(),
+ Lookup::Routed { maps, .. } => Arc::as_ptr(maps).cast(),
+ }
+ }
+}
+
impl Display for HashTableLookupExpr {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.description)
@@ -364,7 +538,7 @@ impl PhysicalExpr for HashTableLookupExpr {
Ok(Arc::new(HashTableLookupExpr {
on_columns: children,
random_state: self.random_state.clone(),
- map: Arc::clone(&self.map),
+ lookup: self.lookup.clone(),
description: self.description.clone(),
pruning_bitmap: self.pruning_bitmap.clone(),
}))
@@ -382,15 +556,21 @@ impl PhysicalExpr for HashTableLookupExpr {
// Evaluate columns
let join_keys = evaluate_columns(&self.on_columns, batch)?;
- match self.map.as_ref() {
- Map::HashMap(map) => {
- with_hashes(&join_keys, self.random_state.random_state(), |hashes| {
- let array = map.contain_hashes(hashes);
+ match &self.lookup {
+ Lookup::Single(map) => match map.as_ref() {
+ Map::HashMap(map) => {
+ with_hashes(&join_keys, self.random_state.random_state(), |hashes| {
+ let array = map.contain_hashes(hashes);
+ Ok(ColumnarValue::Array(Arc::new(array)))
+ })
+ }
+ Map::ArrayMap(map) => {
+ let array = map.contain_keys(&join_keys)?;
Ok(ColumnarValue::Array(Arc::new(array)))
- })
- }
- Map::ArrayMap(map) => {
- let array = map.contain_keys(&join_keys)?;
+ }
+ },
+ Lookup::Routed { router, maps } => {
+ let array = self.evaluate_routed(&join_keys, router, maps)?;
Ok(ColumnarValue::Array(Arc::new(array)))
}
}
@@ -411,7 +591,7 @@ impl PhysicalExpr for HashTableLookupExpr {
// with lit(true), see the comment below
on_columns: _,
random_state: _,
- map: _,
+ lookup: _,
description: _,
pruning_bitmap: _,
} = self;
@@ -474,7 +654,7 @@ mod tests {
/// The pruning bitmap must survive `with_new_children`, which a scan calls
/// once per file.
#[test]
- fn test_pruning_bitmap_shared_with_derived_children() {
+ fn test_pruning_bitmap_shared_with_derived_children() -> Result<()> {
let keys: ArrayRef = Arc::new(arrow::array::Int64Array::from(vec![0, 1_000_000]));
let bitmap = Arc::new(
KeyRangeBitmap::try_new(
@@ -483,22 +663,28 @@ mod tests {
&ScalarValue::Int64(Some(1_000_000)),
2,
)
- .expect("two keys a million apart leave prunable gaps"),
+ .ok_or_else(|| {
+ datafusion_common::internal_datafusion_err!(
+ "two keys a million apart leave prunable gaps"
+ )
+ })?,
);
let expr = Arc::new(HashTableLookupExpr::new(
vec![Arc::new(Column::new("a", 0))],
SeededRandomState::with_seed(1),
- Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(10)))),
+ None,
+ vec![Some(Arc::new(Map::HashMap(Box::new(
+ JoinHashMapU32::with_capacity(10),
+ ))))],
"hash_lookup".to_string(),
Some(Arc::clone(&bitmap)),
- ));
+ )?);
- let derived = expr
- .with_new_children(vec![Arc::new(Column::new("a", 7))])
- .unwrap();
+ let derived = expr.with_new_children(vec![Arc::new(Column::new("a", 7))])?;
let derived = derived.downcast_ref::<HashTableLookupExpr>().unwrap();
assert_eq!(derived.children()[0].to_string(), "a@7");
assert!(Arc::ptr_eq(&bitmap, derived.pruning_bitmap().unwrap()));
+ Ok(())
}
#[test]
@@ -769,7 +955,7 @@ mod tests {
}
#[test]
- fn test_hash_table_lookup_expr_eq_same() {
+ fn test_hash_table_lookup_expr_eq_same() -> Result<()> {
let col_a: PhysicalExprRef = Arc::new(Column::new("a", 0));
let hash_map =
Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(10))));
@@ -777,24 +963,27 @@ mod tests {
let expr1 = HashTableLookupExpr::new(
vec![Arc::clone(&col_a)],
SeededRandomState::with_seed(1),
- Arc::clone(&hash_map),
+ None,
+ vec![Some(Arc::clone(&hash_map))],
"lookup".to_string(),
None,
- );
+ )?;
let expr2 = HashTableLookupExpr::new(
vec![Arc::clone(&col_a)],
SeededRandomState::with_seed(1),
- Arc::clone(&hash_map),
+ None,
+ vec![Some(Arc::clone(&hash_map))],
"lookup".to_string(),
None,
- );
+ )?;
assert_eq!(expr1, expr2);
+ Ok(())
}
#[test]
- fn test_hash_table_lookup_expr_eq_different_columns() {
+ fn test_hash_table_lookup_expr_eq_different_columns() -> Result<()> {
let col_a: PhysicalExprRef = Arc::new(Column::new("a", 0));
let col_b: PhysicalExprRef = Arc::new(Column::new("b", 1));
@@ -804,24 +993,27 @@ mod tests {
let expr1 = HashTableLookupExpr::new(
vec![Arc::clone(&col_a)],
SeededRandomState::with_seed(1),
- Arc::clone(&hash_map),
+ None,
+ vec![Some(Arc::clone(&hash_map))],
"lookup".to_string(),
None,
- );
+ )?;
let expr2 = HashTableLookupExpr::new(
vec![Arc::clone(&col_b)],
SeededRandomState::with_seed(1),
- Arc::clone(&hash_map),
+ None,
+ vec![Some(Arc::clone(&hash_map))],
"lookup".to_string(),
None,
- );
+ )?;
assert_ne!(expr1, expr2);
+ Ok(())
}
#[test]
- fn test_hash_table_lookup_expr_eq_different_description() {
+ fn test_hash_table_lookup_expr_eq_different_description() -> Result<()> {
let col_a: PhysicalExprRef = Arc::new(Column::new("a", 0));
let hash_map =
Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(10))));
@@ -829,24 +1021,27 @@ mod tests {
let expr1 = HashTableLookupExpr::new(
vec![Arc::clone(&col_a)],
SeededRandomState::with_seed(1),
- Arc::clone(&hash_map),
+ None,
+ vec![Some(Arc::clone(&hash_map))],
"lookup_one".to_string(),
None,
- );
+ )?;
let expr2 = HashTableLookupExpr::new(
vec![Arc::clone(&col_a)],
SeededRandomState::with_seed(1),
- Arc::clone(&hash_map),
+ None,
+ vec![Some(Arc::clone(&hash_map))],
"lookup_two".to_string(),
None,
- );
+ )?;
assert_ne!(expr1, expr2);
+ Ok(())
}
#[test]
- fn test_hash_table_lookup_expr_eq_different_hash_map() {
+ fn test_hash_table_lookup_expr_eq_different_hash_map() -> Result<()> {
let col_a: PhysicalExprRef = Arc::new(Column::new("a", 0));
// Two different Arc pointers (even with same content) should not be equal
@@ -857,25 +1052,28 @@ mod tests {
let expr1 = HashTableLookupExpr::new(
vec![Arc::clone(&col_a)],
SeededRandomState::with_seed(1),
- hash_map1,
+ None,
+ vec![Some(hash_map1)],
"lookup".to_string(),
None,
- );
+ )?;
let expr2 = HashTableLookupExpr::new(
vec![Arc::clone(&col_a)],
SeededRandomState::with_seed(1),
- hash_map2,
+ None,
+ vec![Some(hash_map2)],
"lookup".to_string(),
None,
- );
+ )?;
// Different Arc pointers means not equal (uses Arc::ptr_eq)
assert_ne!(expr1, expr2);
+ Ok(())
}
#[test]
- fn test_hash_table_lookup_expr_hash_consistency() {
+ fn test_hash_table_lookup_expr_hash_consistency() -> Result<()> {
let col_a: PhysicalExprRef = Arc::new(Column::new("a", 0));
let hash_map =
Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(10))));
@@ -883,21 +1081,142 @@ mod tests {
let expr1 = HashTableLookupExpr::new(
vec![Arc::clone(&col_a)],
SeededRandomState::with_seed(1),
- Arc::clone(&hash_map),
+ None,
+ vec![Some(Arc::clone(&hash_map))],
"lookup".to_string(),
None,
- );
+ )?;
let expr2 = HashTableLookupExpr::new(
vec![Arc::clone(&col_a)],
SeededRandomState::with_seed(1),
- Arc::clone(&hash_map),
+ None,
+ vec![Some(Arc::clone(&hash_map))],
"lookup".to_string(),
None,
- );
+ )?;
// Equal expressions should have equal hashes
assert_eq!(expr1, expr2);
assert_eq!(compute_hash(&expr1), compute_hash(&expr2));
+ Ok(())
+ }
+
+ /// Probes `probe` against `expr`, one row per key of column `a`.
+ fn lookup(expr: &HashTableLookupExpr, probe: &ArrayRef) -> Result<BooleanArray> {
+ let schema = Arc::new(Schema::new(vec![arrow::datatypes::Field::new(
+ "a",
+ DataType::Int32,
+ true,
+ )]));
+ let batch = RecordBatch::try_new(schema, vec![Arc::clone(probe)])?;
+ let matched = expr.evaluate(&batch)?.into_array(batch.num_rows())?;
+ Ok(arrow::array::AsArray::as_boolean(&matched).clone())
+ }
+
+ /// A hash table of `keys`, hashed with `random_state` as the build side
+ /// hashes them.
+ fn hash_table(keys: &ArrayRef, random_state: &SeededRandomState) -> Result<Arc<Map>> {
+ use crate::joins::join_hash_map::JoinHashMapType;
+ let mut hashes = vec![0; keys.len()];
+ create_hashes([keys], random_state.random_state(), &mut hashes)?;
+ let mut map = JoinHashMapU32::with_capacity(keys.len());
+ map.update_from_iter(Box::new(hashes.iter().enumerate()), 0);
+ Ok(Arc::new(Map::HashMap(Box::new(map))))
+ }
+
+ /// One randomly generated scenario, see [`routed_lookup_fuzz`].
+ fn run_routed_lookup_scenario(seed: u64) -> Result<()> {
+ use crate::repartition::{BatchPartitioner, REPARTITION_RANDOM_STATE};
+ use rand::rngs::StdRng;
+ use rand::{Rng, SeedableRng};
+
+ let mut rng = StdRng::seed_from_u64(seed);
+ let num_partitions = rng.random_range(1..8usize);
+ // A key domain near the row counts, so probes both hit and miss.
+ let build: Vec<i32> = (0..rng.random_range(0..128))
+ .map(|_| rng.random_range(-64..64))
+ .collect();
+ let probe: Vec<Option<i32>> = (0..rng.random_range(0..128))
+ .map(|_| rng.random_bool(0.9).then(|| rng.random_range(-80..80)))
+ .collect();
+ let context = format!(
+ "seed {seed}: {num_partitions} partitions, {} build rows, {} probe rows",
+ build.len(),
+ probe.len()
+ );
+
+ let column: PhysicalExprRef = Arc::new(Column::new("a", 0));
+ let schema = Arc::new(Schema::new(vec![arrow::datatypes::Field::new(
+ "a",
+ DataType::Int32,
+ false,
+ )]));
+ let build: ArrayRef = Arc::new(arrow::array::Int32Array::from(build));
+ let probe: ArrayRef = Arc::new(arrow::array::Int32Array::from(probe));
+ let random_state = SeededRandomState::with_seed(1);
+
+ // Split the build side exactly as hash repartitioning does.
+ let mut maps = vec![None; num_partitions];
+ BatchPartitioner::new_hash_partitioner(
+ vec![Arc::clone(&column)],
+ num_partitions,
+ crate::metrics::Time::new(),
+ )?
+ .partition(
+ RecordBatch::try_new(schema, vec![Arc::clone(&build)])?,
+ |partition, batch| {
+ maps[partition] = Some(hash_table(batch.column(0), &random_state)?);
+ Ok(())
+ },
+ )?;
+ let routed = HashTableLookupExpr::new(
+ vec![Arc::clone(&column)],
+ random_state.clone(),
+ Some(PartitionRouter::Hash(REPARTITION_RANDOM_STATE)),
+ maps,
+ "routed".to_string(),
+ None,
+ )?;
+ // Reference: the single-table lookup over the whole build side.
+ let single = HashTableLookupExpr::new(
+ vec![column],
+ random_state.clone(),
+ None,
+ vec![Some(hash_table(&build, &random_state)?)],
+ "single".to_string(),
+ None,
+ )?;
+
+ assert_eq!(
+ lookup(&routed, &probe)?,
+ lookup(&single, &probe)?,
+ "{context}"
+ );
+ Ok(())
+ }
+
+ /// Routing each probe row to its build partition's table matches one table
+ /// over the whole build side, when the build side is split the way hash
+ /// repartitioning splits it.
+ ///
+ /// `DATAFUSION_ROUTED_LOOKUP_FUZZ_ITERATIONS` and
+ /// `DATAFUSION_ROUTED_LOOKUP_FUZZ_SEED` select how many scenarios run and
+ /// from which seed; a failure names its seed so it can be replayed.
+ #[test]
+ fn routed_lookup_fuzz() -> Result<()> {
+ let env_u64 = |name: &str, default: u64| {
+ std::env::var(name)
+ .ok()
+ .and_then(|v| v.parse().ok())
+ .unwrap_or(default)
+ };
+ let first_seed = env_u64("DATAFUSION_ROUTED_LOOKUP_FUZZ_SEED", 0);
+ let iterations = env_u64("DATAFUSION_ROUTED_LOOKUP_FUZZ_ITERATIONS", 256);
+ // `saturating_add` so a replay seed near `u64::MAX` cannot overflow.
+ for seed in first_seed..first_seed.saturating_add(iterations) {
+ run_routed_lookup_scenario(seed)?;
+ }
+ Ok(())
}
}
diff --git a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs
index c3c37a193e..321ca4b10d 100644
--- a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs
@@ -17,16 +17,19 @@ use crate::joins::hash_join::exec::HASH_JOIN_SEED;
use crate::joins::hash_join::inlist_builder::build_struct_fields;
use crate::joins::hash_join::not_in::NotIn;
use crate::joins::hash_join::partitioned_hash_eval::{
- HashExpr, HashTableLookupExpr, SeededRandomState,
+ HashExpr, HashTableLookupExpr, PartitionRouter, SeededRandomState,
};
-use crate::joins::key_range_bitmap::KeyRangeBitmap;
+use crate::joins::join_hash_map::{JoinHashMapType, JoinHashMapU32};
+use crate::joins::key_range_bitmap::{KeyRangeBitmap, KeyRangePart};
use crate::repartition::RangeExpr;
-use arrow::array::ArrayRef;
+use arrow::array::{ArrayRef, AsArray};
+use arrow::compute::concat;
use arrow::datatypes::{DataType, Field, Schema};
-use datafusion_common::config::ConfigOptions;
+use datafusion_common::config::{ConfigOptions, OptimizerOptions};
+use datafusion_common::hash_utils::create_hashes;
use datafusion_common::{
DataFusionError, NullEquality, Result, ScalarValue, SharedResult,
- assert_or_internal_err,
+ assert_or_internal_err, internal_err,
};
use datafusion_expr::Operator;
use datafusion_functions::core::r#struct as struct_func;
@@ -46,7 +49,7 @@ use tokio::sync::Notify;
pub(crate) struct ColumnBounds {
/// The minimum value observed for this column
pub(crate) min: ScalarValue,
- /// The maximum value observed for this column
+ /// The maximum value observed for this column
pub(crate) max: ScalarValue,
}
@@ -73,6 +76,37 @@ impl PartitionBounds {
pub(crate) fn get_column_bounds(&self, index: usize) -> Option<&ColumnBounds> {
self.column_bounds.get(index)
}
+
+ /// Bounds covering both `self` and `other`, which must describe the same
+ /// columns. A NULL bound means the partition holds no non-null key in that
+ /// column, so it never widens the other side's bound.
+ fn union(self, other: &PartitionBounds) -> PartitionBounds {
+ // Every non-empty partition of a join reports one bound per join key.
+ debug_assert_eq!(
+ self.column_bounds.len(),
+ other.column_bounds.len(),
+ "partitions report bounds for different numbers of columns"
+ );
+ let column_bounds = self
+ .column_bounds
+ .into_iter()
+ .zip(&other.column_bounds)
+ .map(|(a, b)| {
+ let min = if !b.min.is_null() && (a.min.is_null() || b.min < a.min) {
+ b.min.clone()
+ } else {
+ a.min
+ };
+ let max = if !b.max.is_null() && (a.max.is_null() || b.max > a.max) {
+ b.max.clone()
+ } else {
+ a.max
+ };
+ ColumnBounds::new(min, max)
+ })
+ .collect();
+ PartitionBounds::new(column_bounds)
+ }
}
/// Creates a membership predicate for filter pushdown.
@@ -129,10 +163,11 @@ fn create_membership_predicate(
Ok(Some(Arc::new(HashTableLookupExpr::new(
on_right.to_vec(),
random_state.clone(),
- hash_map,
+ None,
+ vec![Some(hash_map)],
"hash_lookup".to_string(),
pruning_bitmap,
- )) as Arc<dyn PhysicalExpr>))
+ )?) as Arc<dyn PhysicalExpr>))
}
// Empty partition - should not create a filter for this
PushdownStrategy::Empty => Ok(None),
@@ -262,6 +297,12 @@ pub(crate) struct SharedBuildAccumulator {
/// Null-aware anti join (`NOT IN`). A probe-side NULL must reach the join so its
/// three-valued logic can collapse the result, so the pushed filter keeps NULL rows.
not_in: Option<NotIn>,
+ /// Largest merged IN list, in bytes, that partitioned filters keep as an IN
+ /// list: `hash_join_inlist_pushdown_max_size`, as for a single build.
+ in_list_max_size: usize,
+ /// Largest merged IN list, in values, that partitioned filters keep as an IN
+ /// list: `hash_join_inlist_pushdown_max_distinct_values`, as for a single build.
+ in_list_max_values: usize,
}
/// Strategy for filter pushdown (decided at collection time)
@@ -374,6 +415,7 @@ impl SharedBuildAccumulator {
repartition_random_state: SeededRandomState,
null_equality: NullEquality,
not_in: Option<NotIn>,
+ optimizer: &OptimizerOptions,
) -> Self {
// Troubleshooting: If partition counts are incorrect, verify this logic matches
// the actual execution pattern in collect_build_side()
@@ -431,6 +473,8 @@ impl SharedBuildAccumulator {
probe_range_partitioning,
null_equality,
not_in,
+ in_list_max_size: optimizer.hash_join_inlist_pushdown_max_size,
+ in_list_max_values: optimizer.hash_join_inlist_pushdown_max_distinct_values,
}
}
@@ -656,6 +700,10 @@ impl SharedBuildAccumulator {
let mut empty_partition_ids = Vec::new();
let mut has_canceled_unknown = false;
let mut keys_have_null = false;
+ // Every non-empty partition's membership strategy and bounds, merged into
+ // one membership filter below; `None` once a partition cannot be merged.
+ let mut members = Some(Vec::with_capacity(partitions.len()));
+ let mut merged_bounds: Option<PartitionBounds> = None;
for (partition_id, partition) in partitions.into_iter().enumerate() {
match partition {
@@ -673,6 +721,22 @@ impl SharedBuildAccumulator {
}) => {
real_partition_ids.push(partition_id);
keys_have_null |= partition_keys_have_null;
+ let mergeable = match &pushdown {
+ PushdownStrategy::Map(map, _) => {
+ matches!(map.as_ref(), Map::HashMap(_))
+ }
+ PushdownStrategy::InList(_) => true,
+ PushdownStrategy::Empty => false,
+ };
+ members = members.filter(|_| mergeable).map(|mut members| {
+ members.push((partition_id, pushdown.clone(), bounds.clone()));
+ members
+ });
+ merged_bounds = Some(match merged_bounds {
+ None => bounds.clone(),
+ Some(merged) => merged.union(&bounds),
+ });
+
let membership_expr = create_membership_predicate(
&self.on_right,
pushdown,
@@ -688,6 +752,8 @@ impl SharedBuildAccumulator {
PartitionStatus::CanceledUnknown => {
has_canceled_unknown = true;
partition_filters.push(lit(true));
+ // Rows routed here must pass, which a merged filter cannot express.
+ members = None;
// A canceled partition's build content is unknown, so it
// may hold a NULL key.
keys_have_null = true;
@@ -717,6 +783,12 @@ impl SharedBuildAccumulator {
// Only one build partition contains rows, so its filter covers every
// possible probe match without routing.
Arc::clone(&partition_filters[real_partition_ids[0]])
+ } else if let Some(members) = members {
+ self.merged_membership_filter(
+ &members,
+ merged_bounds,
+ partition_filters.len(),
+ )?
} else {
// Builds the shared sparse `CASE` for partition filter routing.
// Without cancellation, omitted branches are known empty and safely fall
@@ -791,6 +863,145 @@ impl SharedBuildAccumulator {
.update(self.preserve_probe_nulls(filter_expr, keys_have_null)?)
}
+ /// `members` holds each non-empty partition's id, strategy (an IN list or a
+ /// hash table) and bounds, out of `num_partitions` partitions.
+ fn merged_membership_filter(
+ &self,
+ members: &[(usize, PushdownStrategy, PartitionBounds)],
+ merged_bounds: Option<PartitionBounds>,
+ num_partitions: usize,
+ ) -> Result<Arc<dyn PhysicalExpr>> {
+ let lists = members
+ .iter()
+ .filter_map(|(_, pushdown, _)| match pushdown {
+ PushdownStrategy::InList(values) => Some(values),
+ _ => None,
+ })
+ .collect::<Vec<_>>();
+
+ // Every partition is an IN list, and together they still fit the limits a
+ // single build uses to pick an IN list: one combined list keeps the
+ // statistics and bloom filter pruning an IN list allows. Past those
+ // limits, a hash lookup is cheaper per row and carries a pruning bitmap.
+ let combined_in_list = lists.len() == members.len() && {
+ let values = lists.iter().map(|list| list.len()).sum::<usize>();
+ let bytes = lists
+ .iter()
+ .map(|list| list.get_array_memory_size())
+ .sum::<usize>();
+ values <= self.in_list_max_values && bytes <= self.in_list_max_size
+ };
+ let membership = if combined_in_list {
+ let values = concat(&lists.iter().map(|l| l.as_ref()).collect::<Vec<_>>())?;
+ create_membership_predicate(
+ &self.on_right,
+ PushdownStrategy::InList(values),
+ &HASH_JOIN_SEED,
+ self.probe_schema.as_ref(),
+ )?
+ .expect("an IN list always has a membership predicate")
+ } else {
+ // Each probe row is looked up in the table of the partition its key
+ // routes to, so the partitions' tables are shared as they are: no
+ // merged table to build while every probe stream waits on the filter.
+ let mut maps = vec![None; num_partitions];
+ for (partition_id, pushdown, _) in members {
+ maps[*partition_id] = Some(match pushdown {
+ PushdownStrategy::Map(map, _) => Arc::clone(map),
+ PushdownStrategy::InList(values) => self.in_list_hash_map(values)?,
+ PushdownStrategy::Empty => {
+ return internal_err!("empty partitions have no members");
+ }
+ });
+ }
+ Arc::new(HashTableLookupExpr::new(
+ self.on_right.clone(),
+ HASH_JOIN_SEED,
+ Some(self.partition_router(num_partitions)?),
+ maps,
+ String::from("hash_lookup"),
+ self.merged_pruning_bitmap(members, merged_bounds.as_ref()),
+ )?)
+ };
+
+ let bounds = merged_bounds
+ .and_then(|bounds| create_bounds_predicate(&self.on_right, &bounds));
+ Ok(combine_membership_and_bounds(Some(membership), bounds)
+ .expect("membership predicate is always present"))
+ }
+
+ /// How probe rows find their build partition: as the probe side was
+ /// partitioned.
+ fn partition_router(&self, num_partitions: usize) -> Result<PartitionRouter> {
+ Ok(match &self.probe_range_partitioning {
+ Some(range_partitioning) => {
+ assert_or_internal_err!(
+ num_partitions == range_partitioning.partition_count(),
+ "Dynamic filter partition count {} does not match Range partition count {}",
+ num_partitions,
+ range_partitioning.partition_count()
+ );
+ PartitionRouter::Range(Arc::new(RangeExpr::try_new_with_schema(
+ self.on_right.clone(),
+ range_partitioning,
+ &self.probe_schema,
+ )?))
+ }
+ None => PartitionRouter::Hash(self.repartition_random_state.clone()),
+ })
+ }
+
+ /// A hash table of an IN list partition's keys, hashed as the probe side
+ /// hashes its join keys: one column per join key.
+ fn in_list_hash_map(&self, values: &ArrayRef) -> Result<Arc<Map>> {
+ let columns = match values.as_struct_opt() {
+ Some(columns) if self.on_right.len() > 1 => columns.columns(),
+ _ => std::slice::from_ref(values),
+ };
+ let mut hashes = vec![0; values.len()];
+ create_hashes(columns, HASH_JOIN_SEED.random_state(), &mut hashes)?;
+ let mut map = JoinHashMapU32::with_capacity(hashes.len());
+ map.update_from_iter(Box::new(hashes.iter().enumerate()), 0);
+ Ok(Arc::new(Map::HashMap(Box::new(map))))
+ }
+
+ /// Container pruning reads one bitmap, so merge the partitions' key ranges:
+ /// a hash table's bitmap, an IN list's keys, or else a partition's whole
+ /// bounds. `None` when no partition narrows its bounds, as the bounds
+ /// predicate already prunes that much.
+ fn merged_pruning_bitmap(
+ &self,
+ members: &[(usize, PushdownStrategy, PartitionBounds)],
+ merged_bounds: Option<&PartitionBounds>,
+ ) -> Option<Arc<KeyRangeBitmap>> {
+ if self.on_right.len() != 1 {
+ return None;
+ }
+ let merged = merged_bounds?.get_column_bounds(0)?;
+ let parts = members
+ .iter()
+ .map(|(_, pushdown, bounds)| match pushdown {
+ PushdownStrategy::Map(_, Some(bitmap)) => {
+ Some(KeyRangePart::Bitmap(bitmap))
+ }
+ PushdownStrategy::InList(keys) => Some(KeyRangePart::Keys(keys.as_ref())),
+ _ => bounds
+ .get_column_bounds(0)
+ .map(|bounds| KeyRangePart::Range {
+ min: &bounds.min,
+ max: &bounds.max,
+ }),
+ })
+ .collect::<Option<Vec<_>>>()?;
+ if parts
+ .iter()
+ .all(|part| matches!(part, KeyRangePart::Range { .. }))
+ {
+ return None;
+ }
+ KeyRangeBitmap::union(&merged.min, &merged.max, &parts).map(Arc::new)
+ }
+
/// Keeps probe rows with a NULL key when the join semantics need them.
///
/// The build-side predicate drops probe rows whose key is NULL. A null-aware join
@@ -887,6 +1098,8 @@ pub(super) fn make_partitioned_accumulator_for_test(
probe_range_partitioning: None,
null_equality: NullEquality::NullEqualsNothing,
not_in: None,
+ in_list_max_size: 128 * 1024,
+ in_list_max_values: 150,
}
}
@@ -908,6 +1121,7 @@ mod tests {
use super::*;
use crate::joins::hash_join::not_in::{NotInOutput, NotInScope};
+ use crate::joins::join_hash_map::JoinHashMapU32;
use arrow::array::{ArrayRef, BooleanArray, Float64Array, Int32Array};
use arrow::compute::SortOptions;
use arrow::record_batch::RecordBatch;
@@ -953,6 +1167,8 @@ mod tests {
probe_range_partitioning: None,
null_equality: NullEquality::NullEqualsNothing,
not_in: None,
+ in_list_max_size: 128 * 1024,
+ in_list_max_values: 150,
}
}
@@ -1124,6 +1340,157 @@ mod tests {
assert_literal_bool(&expr, true);
}
+ fn hash_map() -> PushdownStrategy {
+ PushdownStrategy::Map(
+ Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(1)))),
+ None,
+ )
+ }
+
+ #[test]
+ fn partitioned_hash_map_partitions_route_without_case() {
+ let acc = make_partitioned_expr_accumulator_for_test(3);
+
+ acc.build_filter(FinalizeInput::Partitioned(vec![
+ reported(hash_map(), bounds(5, 10)),
+ reported(PushdownStrategy::Empty, no_bounds()),
+ reported(hash_map(), bounds(1, 7)),
+ ]))
+ .unwrap();
+
+ let expr = current_expr(&acc);
+ assert!(expr.downcast_ref::<CaseExpr>().is_none());
+ assert_eq!(
+ expr.to_string(),
+ "probe_key@0 >= 1 AND probe_key@0 <= 10 AND hash_lookup"
+ );
+ }
+
+ /// A hash table holding `keys`, hashed as the build side hashes them.
+ #[cfg(not(feature = "force_hash_collisions"))]
+ fn hash_map_of(keys: &[i32]) -> PushdownStrategy {
+ use crate::joins::join_hash_map::JoinHashMapType;
+ let keys: ArrayRef = Arc::new(Int32Array::from(keys.to_vec()));
+ let mut hashes = vec![0; keys.len()];
+ create_hashes([&keys], HASH_JOIN_SEED.random_state(), &mut hashes).unwrap();
+ let mut map = JoinHashMapU32::with_capacity(keys.len());
+ map.update_from_iter(Box::new(hashes.iter().enumerate()), 0);
+ PushdownStrategy::Map(Arc::new(Map::HashMap(Box::new(map))), None)
+ }
+
+ /// Splits `keys` across `num_partitions` build partitions as hash
+ /// repartitioning would, reporting each non-empty partition with the
+ /// strategy `strategy` picks for it.
+ #[cfg(not(feature = "force_hash_collisions"))]
+ fn routed_partitions(
+ acc: &SharedBuildAccumulator,
+ num_partitions: usize,
+ keys: &[i32],
+ strategy: impl Fn(usize, &[i32]) -> PushdownStrategy,
+ ) -> Vec<PartitionStatus> {
+ let array: ArrayRef = Arc::new(Int32Array::from(keys.to_vec()));
+ let mut hashes = vec![0; keys.len()];
+ create_hashes(
+ [&array],
+ acc.repartition_random_state.random_state(),
+ &mut hashes,
+ )
+ .unwrap();
+ (0..num_partitions)
+ .map(|partition| {
+ let held: Vec<i32> = keys
+ .iter()
+ .zip(&hashes)
+ .filter(|(_, hash)| *hash % num_partitions as u64 == partition as u64)
+ .map(|(key, _)| *key)
+ .collect();
+ if held.is_empty() {
+ reported(PushdownStrategy::Empty, no_bounds())
+ } else {
+ reported(strategy(partition, &held), no_bounds())
+ }
+ })
+ .collect()
+ }
+
+ /// Partitions that chose an IN list and partitions that chose a hash table
+ /// merge into one lookup without a `CASE`: a probe row passes exactly when
+ /// the partition its key routes to holds the key.
+ #[test]
+ #[cfg(not(feature = "force_hash_collisions"))]
+ fn partitioned_in_list_and_hash_map_partitions_merge() {
+ let acc = make_partitioned_expr_accumulator_for_test(4);
+ let build: Vec<i32> = (0..40).step_by(2).collect();
+
+ acc.build_filter(FinalizeInput::Partitioned(routed_partitions(
+ &acc,
+ 4,
+ &build,
+ |partition, keys| {
+ if partition % 2 == 0 {
+ hash_map_of(keys)
+ } else {
+ in_list(keys)
+ }
+ },
+ )))
+ .unwrap();
+
+ assert!(
+ current_expr(&acc).downcast_ref::<CaseExpr>().is_none(),
+ "expected a merged lookup, got: {}",
+ current_expr(&acc)
+ );
+ let probe: Vec<i32> = (0..40).collect();
+ let expected: Vec<bool> = probe.iter().map(|key| key % 2 == 0).collect();
+ assert_matches(&acc, &probe, &expected);
+ }
+
+ /// IN list partitions whose combined list passes the IN list limits merge
+ /// into a hash lookup rather than one oversized IN list, which must still
+ /// accept every partition's keys.
+ #[test]
+ #[cfg(not(feature = "force_hash_collisions"))]
+ fn partitioned_in_lists_past_limit_merge_into_hash_lookup() {
+ let mut acc = make_partitioned_expr_accumulator_for_test(3);
+ acc.in_list_max_values = 2;
+ let build: Vec<i32> = (1..=9).collect();
+
+ acc.build_filter(FinalizeInput::Partitioned(routed_partitions(
+ &acc,
+ 3,
+ &build,
+ |_, keys| in_list(keys),
+ )))
+ .unwrap();
+
+ let expr = current_expr(&acc);
+ assert!(
+ expr.downcast_ref::<CaseExpr>().is_none()
+ && expr.downcast_ref::<InListExpr>().is_none(),
+ "expected a hash lookup, got: {expr}"
+ );
+ let probe: Vec<i32> = (0..20).collect();
+ let expected: Vec<bool> = probe.iter().map(|key| (1..=9).contains(key)).collect();
+ assert_matches(&acc, &probe, &expected);
+ }
+
+ /// Evaluates the current filter over probe keys `keys`.
+ #[cfg(not(feature = "force_hash_collisions"))]
+ fn assert_matches(acc: &SharedBuildAccumulator, keys: &[i32], expected: &[bool]) {
+ let probe = RecordBatch::try_new(
+ test_probe_schema(),
+ vec![Arc::new(Int32Array::from(keys.to_vec()))],
+ )
+ .unwrap();
+ let matched = current_expr(acc)
+ .evaluate(&probe)
+ .unwrap()
+ .into_array(keys.len())
+ .unwrap();
+ assert_eq!(matched.as_boolean(), &BooleanArray::from(expected.to_vec()));
+ }
+
#[test]
fn partitioned_one_real_partition_with_rest_empty_skips_case() {
let acc = make_partitioned_expr_accumulator_for_test(3);
@@ -1444,6 +1811,8 @@ mod tests {
scope: NotInScope::Uncorrelated,
output: NotInOutput::Rows,
}),
+ in_list_max_size: 128 * 1024,
+ in_list_max_values: 150,
}
}
@@ -1491,9 +1860,9 @@ mod tests {
acc
}
- /// The IS NULL disjunct must wrap the routed `CASE` as a whole. A probe NULL
- /// takes exactly one route, so widening only the branch of the partition that
- /// reported the NULL would still drop it on every other route.
+ /// The IS NULL disjunct must wrap the merged filter as a whole. A probe NULL
+ /// belongs to exactly one partition, so widening only the filter of the
+ /// partition that reported the NULL would still drop it everywhere else.
#[test]
fn partitioned_null_keys_in_one_partition_widen_whole_routed_filter() {
let acc = null_equal_partitioned_accumulator(2);
@@ -1511,7 +1880,7 @@ mod tests {
widened.left().downcast_ref::<IsNullExpr>().is_some(),
"expected the IS NULL disjunct first, got: {expr}"
);
- case_expr(widened.right());
+ assert_in_list_column_values(widened.right(), "probe_key", 0, &[1, 2]);
assert_eq!(
format!("{expr}").matches("IS NULL").count(),
1,
@@ -1659,6 +2028,8 @@ mod tests {
scope: NotInScope::Correlated { value_keys },
output: NotInOutput::Mark,
}),
+ in_list_max_size: 128 * 1024,
+ in_list_max_values: 150,
};
let bounds = PartitionBounds::new(
diff --git a/datafusion/physical-plan/src/joins/key_range_bitmap.rs b/datafusion/physical-plan/src/joins/key_range_bitmap.rs
index 60631dd2c5..47030f5cad 100644
--- a/datafusion/physical-plan/src/joins/key_range_bitmap.rs
+++ b/datafusion/physical-plan/src/joins/key_range_bitmap.rs
@@ -15,6 +15,19 @@ use crate::joins::array_map::ArrayMap;
/// Largest bitmap, in bits (128 KiB), which bounds what one build side can hold.
const MAX_BUCKETS: u64 = 1 << 20;
+/// One build partition's contribution to [`KeyRangeBitmap::union`].
+pub(crate) enum KeyRangePart<'a> {
+ /// The partition's own bitmap.
+ Bitmap(&'a KeyRangeBitmap),
+ /// The partition's build keys, which must lie within its `[min, max]`.
+ Keys(&'a dyn Array),
+ /// The partition has no bitmap, so any key in `[min, max]` may be present.
+ Range {
+ min: &'a ScalarValue,
+ max: &'a ScalarValue,
+ },
+}
+
/// Which parts of a build side's key range hold at least one key.
///
/// Keys map order-preservingly onto buckets spanning `[min, max]`, one bit each,
@@ -88,6 +101,104 @@ impl KeyRangeBitmap {
})
}
+ /// One bitmap over `[min, max]` marking every bucket that any of `parts`
+ /// may hold a key in. `[min, max]` must cover every part.
+ ///
+ /// Coarser than the parts' own bitmaps where its buckets are wider, but
+ /// never clears a bucket a part may hold a key in. `None` when the key
+ /// type has no `u64` ordering, or when nothing can be excluded.
+ pub(crate) fn union(
+ min: &ScalarValue,
+ max: &ScalarValue,
+ parts: &[KeyRangePart<'_>],
+ ) -> Option<Self> {
+ let offset = ArrayMap::key_to_u64(min)?;
+ let span = ArrayMap::key_to_u64(max)?.wrapping_sub(offset);
+ let buckets = span.saturating_add(1).min(MAX_BUCKETS).next_power_of_two();
+ let shift = (0u32..63).find(|s| (span >> s) < buckets).unwrap_or(63);
+
+ // One bit per bucket, in words, so a part's bitmap ORs in a word at a time.
+ let mut words =
+ MutableBuffer::from_len_zeroed((buckets as usize).div_ceil(64) * 8);
+ let reachable = ((span >> shift) + 1) as usize;
+ for part in parts {
+ match part {
+ KeyRangePart::Bitmap(part) => {
+ let words = words.typed_data_mut::<u64>();
+ // Every key below `part.offset` is outside `[min, max]`, so
+ // this wrapping distance is the part's true position.
+ let base = part.offset.wrapping_sub(offset);
+ let last = (part.span >> part.shift) as usize;
+ if part.shift == shift {
+ // Same bucket width: part bucket `b` covers union bucket
+ // `b + q`, and `b + q + 1` too when the part's buckets
+ // start `r` keys into a union bucket. Only the last part
+ // bucket, which may be narrower, can stop short of it.
+ let q = (base >> shift) as usize;
+ let r = base & ((1u64 << shift) - 1);
+ or_shifted(words, &part.bits, last + 1, q);
+ if r != 0 {
+ or_shifted(words, &part.bits, last, q + 1);
+ if part.bits.value(last) {
+ let lo = base + ((last as u64) << shift);
+ let hi = (base + part.span).min(span);
+ set_bit_range(
+ words,
+ (lo >> shift) as usize,
+ (hi >> shift) as usize,
+ );
+ }
+ }
+ } else {
+ let width = (1u64 << part.shift) - 1;
+ for bucket in part.bits.slice(0, last + 1).set_indices() {
+ let lo = base + ((bucket as u64) << part.shift);
+ let hi = (base
+ + ((bucket as u64) << part.shift)
+ .saturating_add(width)
+ .min(part.span))
+ .min(span);
+ set_bit_range(
+ words,
+ (lo >> shift) as usize,
+ (hi >> shift) as usize,
+ );
+ }
+ }
+ }
+ KeyRangePart::Keys(keys) => {
+ if !ArrayMap::is_supported_type(keys.data_type()) {
+ return None;
+ }
+ set_bucket_bits(*keys, offset, shift, buckets, words.as_slice_mut());
+ }
+ KeyRangePart::Range { min, max } => {
+ let lo = ArrayMap::key_to_u64(min)?.wrapping_sub(offset);
+ let hi = ArrayMap::key_to_u64(max)?.wrapping_sub(offset).min(span);
+ if lo <= hi {
+ set_bit_range(
+ words.typed_data_mut::<u64>(),
+ (lo >> shift) as usize,
+ (hi >> shift) as usize,
+ );
+ }
+ }
+ }
+ // Bits are only ever added, so once every reachable bucket is set
+ // the remaining parts cannot make the bitmap exclude anything.
+ if all_set(words.typed_data::<u64>(), reachable) {
+ return None;
+ }
+ }
+
+ Some(Self {
+ offset,
+ span,
+ shift,
+ bits: BooleanBuffer::new(words.into(), 0, buckets as usize),
+ })
+ }
+
/// One Boolean per `[min[i], max[i]]` container interval: `false` proves no
/// build key lies in that container, `None` where a bound is absent or the
/// statistics are not integer keys.
@@ -144,9 +255,20 @@ impl KeyRangeBitmap {
/// One bit per bucket, set for each bucket holding a non-null key of `keys`.
fn bucket_bits(keys: &dyn Array, offset: u64, shift: u32, buckets: u64) -> BooleanBuffer {
let mut words = MutableBuffer::new_null(buckets as usize);
+ set_bucket_bits(keys, offset, shift, buckets, words.as_slice_mut());
+ BooleanBuffer::new(words.into(), 0, buckets as usize)
+}
+
+/// Sets the bit of each bucket holding a non-null key of `keys` in `bytes`.
+fn set_bucket_bits(
+ keys: &dyn Array,
+ offset: u64,
+ shift: u32,
+ buckets: u64,
+ bytes: &mut [u8],
+) {
downcast_integer_array!(
keys => {
- let bytes = words.as_slice_mut();
let mut set = |key: u64| {
let bucket = (key.wrapping_sub(offset) >> shift).min(buckets - 1);
bit_util::set_bit(bytes, bucket as usize);
@@ -159,7 +281,45 @@ fn bucket_bits(keys: &dyn Array, offset: u64, shift: u32, buckets: u64) -> Boole
}
_ => unreachable!("guarded by ArrayMap::is_supported_type"),
);
- BooleanBuffer::new(words.into(), 0, buckets as usize)
+}
+
+/// ORs bits `0..len` of `src` into `words`, moved `by` bits further along.
+fn or_shifted(words: &mut [u64], src: &BooleanBuffer, len: usize, by: usize) {
+ let (skip, bit) = (by / 64, by % 64);
+ let chunks = src.inner().bit_chunks(src.offset(), len);
+ for (i, chunk) in chunks.iter_padded().enumerate() {
+ if chunk == 0 {
+ continue;
+ }
+ words[skip + i] |= chunk << bit;
+ if bit != 0 {
+ // A set bit always lands on a bucket, so it never spills past the end.
+ if let Some(next) = words.get_mut(skip + i + 1) {
+ *next |= chunk >> (64 - bit);
+ }
+ }
+ }
+}
+
+/// Sets bits `from..=to` of `words`.
+fn set_bit_range(words: &mut [u64], from: usize, to: usize) {
+ let (first, last) = (from / 64, to / 64);
+ let head = u64::MAX << (from % 64);
+ let tail = u64::MAX >> (63 - to % 64);
+ if first == last {
+ words[first] |= head & tail;
+ } else {
+ words[first] |= head;
+ words[first + 1..last].fill(u64::MAX);
+ words[last] |= tail;
+ }
+}
+
+/// Whether bits `0..len` of `words` are all set.
+fn all_set(words: &[u64], len: usize) -> bool {
+ let (full, rest) = (len / 64, len % 64);
+ words[..full].iter().all(|word| *word == u64::MAX)
+ && (rest == 0 || words[full] | (u64::MAX << rest) == u64::MAX)
}
#[cfg(test)]
@@ -201,6 +361,49 @@ mod tests {
assert!(kept <= 220, "expected ~200 kept, got {kept}");
}
+ /// Partitions spanning negative and positive keys, one without a bitmap:
+ /// the union must keep every container holding a key of any partition, and
+ /// still prune the gaps no partition covers.
+ #[test]
+ fn union_never_drops_a_partition_key() {
+ let clustered: Vec<i64> = (-1_000_000..-999_000).chain(0..1000).collect();
+ let sparse: Vec<i64> = (0..400).map(|i| 500_000 + 1000 * i).collect();
+ let dense = (-200_000i64, -190_000i64);
+ let a = bitmap(&clustered).expect("not saturated");
+ let b = bitmap(&sparse).expect("not saturated");
+ let (min, max) = (
+ ScalarValue::Int64(Some(-1_000_000)),
+ ScalarValue::Int64(Some(899_000)),
+ );
+ let (dense_min, dense_max) = (
+ ScalarValue::Int64(Some(dense.0)),
+ ScalarValue::Int64(Some(dense.1)),
+ );
+ let union = KeyRangeBitmap::union(
+ &min,
+ &max,
+ &[
+ KeyRangePart::Bitmap(&a),
+ KeyRangePart::Bitmap(&b),
+ KeyRangePart::Range {
+ min: &dense_min,
+ max: &dense_max,
+ },
+ ],
+ )
+ .expect("not saturated");
+
+ for key in clustered
+ .iter()
+ .chain(&sparse)
+ .chain(&[dense.0, -195_000, dense.1])
+ {
+ assert!(may(&union, *key, *key), "dropped key {key}");
+ }
+ assert!(!may(&union, -900_000, -800_000));
+ assert!(!may(&union, 100_000, 400_000));
+ }
+
#[test]
fn contiguous_keys_no_bitmap() {
assert!(bitmap(&(0..5000i64).collect::<Vec<_>>()).is_none());
@@ -222,4 +425,189 @@ mod tests {
assert!(may(&s, -1, 1));
assert!(!may(&s, 1 << 50, 1 << 51));
}
+
+ /// `(offset, span, shift, bits)`, to compare bitmaps by value.
+ fn parts_of(b: &KeyRangeBitmap) -> (u64, u64, u32, &BooleanBuffer) {
+ (b.offset, b.span, b.shift, &b.bits)
+ }
+
+ /// One randomly generated scenario, see [`union_fuzz`].
+ fn run_union_scenario(seed: u64) {
+ use rand::rngs::StdRng;
+ use rand::{Rng, SeedableRng};
+
+ let mut rng = StdRng::seed_from_u64(seed);
+ // Domain widths from one key per bucket to many keys per bucket, so
+ // the union's buckets are sometimes exact and sometimes coarse.
+ let width = [64i64, 1 << 12, 1 << 24, 1 << 40][rng.random_range(0..4)];
+ let start = rng.random_range(-(1i64 << 50)..(1i64 << 50));
+ let key = |rng: &mut StdRng| start + rng.random_range(0..width);
+ let keys_array =
+ |keys: &[i64]| -> ArrayRef { Arc::new(Int64Array::from(keys.to_vec())) };
+ let scalar = |key: i64| ScalarValue::Int64(Some(key));
+
+ enum Part {
+ Keys(ArrayRef),
+ Range(ScalarValue, ScalarValue),
+ Bitmap(KeyRangeBitmap),
+ }
+ // Each part, and every key any part may hold.
+ let mut parts = vec![];
+ let mut all_keys = vec![];
+ let mut has_bitmap = false;
+ for _ in 0..rng.random_range(1..5) {
+ if rng.random_bool(1.0 / 3.0) {
+ // Narrow, so every key it may hold can be listed.
+ let lo = key(&mut rng);
+ let hi = lo + rng.random_range(0..64);
+ parts.push(Part::Range(scalar(lo), scalar(hi)));
+ all_keys.extend(lo..=hi);
+ continue;
+ }
+ let keys: Vec<i64> = (0..rng.random_range(1..50))
+ .map(|_| key(&mut rng))
+ .collect();
+ let (lo, hi) = (*keys.iter().min().unwrap(), *keys.iter().max().unwrap());
+ let bitmap = rng
+ .random_bool(0.5)
+ .then(|| {
+ KeyRangeBitmap::try_new(
+ &keys_array(&keys),
+ &scalar(lo),
+ &scalar(hi),
+ 0,
+ )
+ })
+ .flatten();
+ parts.push(match bitmap {
+ Some(bitmap) => {
+ has_bitmap = true;
+ Part::Bitmap(bitmap)
+ }
+ None => Part::Keys(keys_array(&keys)),
+ });
+ all_keys.extend(keys);
+ }
+ let min = ScalarValue::Int64(all_keys.iter().min().copied());
+ let max = ScalarValue::Int64(all_keys.iter().max().copied());
+ let context = format!(
+ "seed {seed}: width {width}, {} parts, {} keys",
+ parts.len(),
+ all_keys.len()
+ );
+
+ let union = KeyRangeBitmap::union(
+ &min,
+ &max,
+ &parts
+ .iter()
+ .map(|part| match part {
+ Part::Keys(keys) => KeyRangePart::Keys(keys.as_ref()),
+ Part::Range(min, max) => KeyRangePart::Range { min, max },
+ Part::Bitmap(bitmap) => KeyRangePart::Bitmap(bitmap),
+ })
+ .collect::<Vec<_>>(),
+ );
+ // Reference: one bitmap built straight from every key any part may hold.
+ let exact = KeyRangeBitmap::try_new(&keys_array(&all_keys), &min, &max, 0);
+
+ // Never excludes a key any part may hold.
+ if let Some(union) = &union {
+ for key in &all_keys {
+ assert!(may(union, *key, *key), "{context}: dropped key {key}");
+ }
+ }
+ if has_bitmap {
+ // A part's bitmap can only widen buckets: every bucket the exact
+ // bitmap sets, the union sets too.
+ match (&union, &exact) {
+ (_, None) => assert!(union.is_none(), "{context}"),
+ (None, Some(_)) => {}
+ (Some(union), Some(exact)) => {
+ assert_eq!(
+ (union.offset, union.span, union.shift),
+ (exact.offset, exact.span, exact.shift),
+ "{context}"
+ );
+ for bucket in exact.bits.set_indices() {
+ assert!(union.bits.value(bucket), "{context}: bucket {bucket}");
+ }
+ }
+ }
+ // Each set bucket of a part's bitmap stands for its whole key range,
+ // so the union equals the one over those ranges given as `Range`s.
+ let ranges: Vec<(ScalarValue, ScalarValue)> = parts
+ .iter()
+ .flat_map(|part| match part {
+ Part::Bitmap(bitmap) => bitmap
+ .bits
+ .set_indices()
+ .map(|bucket| {
+ let lo = (bucket as u64) << bitmap.shift;
+ let hi = lo
+ .saturating_add((1u64 << bitmap.shift) - 1)
+ .min(bitmap.span);
+ let key =
+ |at: u64| scalar(bitmap.offset.wrapping_add(at) as i64);
+ (key(lo), key(hi))
+ })
+ .collect(),
+ _ => vec![],
+ })
+ .collect();
+ let expanded = KeyRangeBitmap::union(
+ &min,
+ &max,
+ &parts
+ .iter()
+ .filter_map(|part| match part {
+ Part::Keys(keys) => Some(KeyRangePart::Keys(keys.as_ref())),
+ Part::Range(min, max) => Some(KeyRangePart::Range { min, max }),
+ Part::Bitmap(_) => None,
+ })
+ .chain(
+ ranges
+ .iter()
+ .map(|(min, max)| KeyRangePart::Range { min, max }),
+ )
+ .collect::<Vec<_>>(),
+ );
+ assert_eq!(
+ union.as_ref().map(parts_of),
+ expanded.as_ref().map(parts_of),
+ "{context}: bitmap parts differ from their bucket ranges"
+ );
+ } else {
+ // Keys and ranges mark exactly the buckets their keys fall in.
+ assert_eq!(
+ union.as_ref().map(parts_of),
+ exact.as_ref().map(parts_of),
+ "{context}"
+ );
+ }
+ }
+
+ /// Randomized partitions against [`KeyRangeBitmap::union`]'s contract: it
+ /// never excludes a key any part may hold, equals the bitmap built from all
+ /// parts' keys when no part brings its own bitmap, and only widens that
+ /// bitmap's buckets when one does.
+ ///
+ /// `DATAFUSION_KEY_RANGE_UNION_FUZZ_ITERATIONS` and
+ /// `DATAFUSION_KEY_RANGE_UNION_FUZZ_SEED` select how many scenarios run and
+ /// from which seed; a failure names its seed so it can be replayed.
+ #[test]
+ fn union_fuzz() {
+ let env_u64 = |name: &str, default: u64| {
+ std::env::var(name)
+ .ok()
+ .and_then(|v| v.parse().ok())
+ .unwrap_or(default)
+ };
+ let first_seed = env_u64("DATAFUSION_KEY_RANGE_UNION_FUZZ_SEED", 0);
+ let iterations = env_u64("DATAFUSION_KEY_RANGE_UNION_FUZZ_ITERATIONS", 256);
+ // `saturating_add` so a replay seed near `u64::MAX` cannot overflow.
+ for seed in first_seed..first_seed.saturating_add(iterations) {
+ run_union_scenario(seed);
+ }
+ }
}
diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rs
index 0ecea79ef4..929f52fd15 100644
--- a/datafusion/physical-plan/src/repartition/mod.rs
+++ b/datafusion/physical-plan/src/repartition/mod.rs
@@ -783,6 +783,20 @@ impl RangeExpr {
pub fn sort_options(&self) -> &[SortOptions] {
self.router.sort_options()
}
+
+ /// Appends the partition of each row of `arrays`, the evaluated
+ /// `on_columns`, to `partition_ids`.
+ pub(crate) fn route_partition_ids(
+ &self,
+ arrays: &[arrow::array::ArrayRef],
+ partition_ids: &mut Vec<u64>,
+ ) -> Result<()> {
+ if self.router.split_points().is_empty() {
+ partition_ids.resize(partition_ids.len() + arrays[0].len(), 0);
+ return Ok(());
+ }
+ self.router.route_partition_ids(arrays, partition_ids)
+ }
}
impl Display for RangeExpr {
diff --git a/datafusion/proto/tests/cases/plans/exprs.rs b/datafusion/proto/tests/cases/plans/exprs.rs
index a76d40325a..8c193283ad 100644
--- a/datafusion/proto/tests/cases/plans/exprs.rs
+++ b/datafusion/proto/tests/cases/plans/exprs.rs
@@ -446,10 +446,11 @@ fn roundtrip_hash_table_lookup_expr_to_lit() -> Result<()> {
let lookup_expr: Arc<dyn PhysicalExpr> = Arc::new(HashTableLookupExpr::new(
on_columns,
datafusion::physical_plan::joins::SeededRandomState::with_seed(0),
- hash_map,
+ None,
+ vec![Some(hash_map)],
"test_lookup".to_string(),
None,
- ));
+ )?);
// Create a filter with the lookup expression
let filter = Arc::new(FilterExec::try_new(lookup_expr, input)?);
diff --git a/datafusion/pruning/src/pruning_predicate.rs b/datafusion/pruning/src/pruning_predicate.rs
index aa48b75ae7..2e3c84673f 100644
--- a/datafusion/pruning/src/pruning_predicate.rs
+++ b/datafusion/pruning/src/pruning_predicate.rs
@@ -7470,7 +7470,7 @@ mod tests {
fn hash_lookup(
column: &Arc<dyn PhysicalExpr>,
keys: &[i64],
- ) -> Arc<dyn PhysicalExpr> {
+ ) -> Result<Arc<dyn PhysicalExpr>> {
let array: ArrayRef = Arc::new(Int64Array::from(keys.to_vec()));
let bitmap = KeyRangeBitmap::try_new(
&array,
@@ -7478,26 +7478,28 @@ mod tests {
&ScalarValue::Int64(Some(*keys.iter().max().unwrap())),
keys.len(),
);
- Arc::new(HashTableLookupExpr::new(
+ Ok(Arc::new(HashTableLookupExpr::new(
vec![Arc::clone(column)],
SeededRandomState::with_seed(1),
- Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(1)))),
+ None,
+ vec![Some(Arc::new(Map::HashMap(Box::new(
+ JoinHashMapU32::with_capacity(1),
+ ))))],
"hash_lookup".to_string(),
bitmap.map(Arc::new),
- ))
+ )?))
}
#[test]
- fn test_hash_lookup_pruning_via_min_max() {
+ fn test_hash_lookup_pruning_via_min_max() -> Result<()> {
let schema = Arc::new(Schema::new(vec![Field::new("b", DataType::Int64, true)]));
let column: Arc<dyn PhysicalExpr> = Arc::new(phys_expr::Column::new("b", 0));
// Gapped keys: 10,20,30 then a far-away 100000, so the bitmap is sparse.
- let lookup = hash_lookup(&column, &[10, 20, 30, 100_000]);
+ let lookup = hash_lookup(&column, &[10, 20, 30, 100_000])?;
let predicate = PruningPredicateBuilder::new()
.with_file_schema(Arc::clone(&schema))
- .try_build(lookup)
- .unwrap();
+ .try_build(lookup)?;
// An opaque lookup yields no literal guarantees, so `contained()` is never
// consulted and every exclusion below comes from the min/max rewrite.
@@ -7511,10 +7513,11 @@ mod tests {
),
);
- let result = predicate.prune(&statistics).unwrap();
+ let result = predicate.prune(&statistics)?;
// Containers 0 and 2 sit in the gap between 30 and 100000; container 1
// holds 20. The bitmap excludes the first and third.
assert_eq!(result, vec![false, true, false]);
+ Ok(())
}
#[rstest]
@@ -7528,18 +7531,17 @@ mod tests {
#[case] file_type: DataType,
#[case] cast_type: DataType,
#[case] pruned: bool,
- ) {
+ ) -> Result<()> {
let schema =
Arc::new(Schema::new(vec![Field::new("b", file_type.clone(), true)]));
let column: Arc<dyn PhysicalExpr> = Arc::new(phys_expr::Column::new("b", 0));
let cast_column: Arc<dyn PhysicalExpr> =
Arc::new(phys_expr::CastExpr::new(column, cast_type, None));
- let lookup = hash_lookup(&cast_column, &[10, 20, 30, 100_000]);
+ let lookup = hash_lookup(&cast_column, &[10, 20, 30, 100_000])?;
let predicate = PruningPredicateBuilder::new()
.with_file_schema(Arc::clone(&schema))
- .try_build(lookup)
- .unwrap();
+ .try_build(lookup)?;
let to_file_type = |values: Vec<Option<i64>>| -> ArrayRef {
let values: ArrayRef = Arc::new(values.into_iter().collect::<Int64Array>());
@@ -7552,7 +7554,7 @@ mod tests {
.with_max(to_file_type(vec![Some(8000), Some(25), Some(60000)])),
);
- let result = predicate.prune(&statistics).unwrap();
+ let result = predicate.prune(&statistics)?;
// Containers 0 and 2 sit in the gap between 30 and 100000; container 1
// holds 20. A working cast excludes the first and third; a declined one
// falls back to keeping every container.
@@ -7562,10 +7564,11 @@ mod tests {
vec![true, true, true]
};
assert_eq!(result, expected);
+ Ok(())
}
#[test]
- fn test_partition_routed_hash_lookup_pruning() {
+ fn test_partition_routed_hash_lookup_pruning() -> Result<()> {
let schema = Arc::new(Schema::new(vec![Field::new("b", DataType::Int32, true)]));
let column: Arc<dyn PhysicalExpr> = Arc::new(phys_expr::Column::new("b", 0));
let literal = |value: ScalarValue| -> Arc<dyn PhysicalExpr> {
@@ -7573,29 +7576,25 @@ mod tests {
};
// One branch per build partition, each holding its own slice of the build
// side, as `build_partitioned_filter` produces them.
- let case: Arc<dyn PhysicalExpr> = Arc::new(
- phys_expr::CaseExpr::try_new(
- Some(literal(ScalarValue::UInt64(Some(0)))),
- vec![
- (
- literal(ScalarValue::UInt64(Some(0))),
- hash_lookup(&column, &[10, 20, 30]),
- ),
- (
- literal(ScalarValue::UInt64(Some(1))),
- hash_lookup(&column, &[40, 50, 60]),
- ),
- ],
- // Partitions with no build rows reject everything routed to them.
- Some(literal(ScalarValue::Boolean(Some(false)))),
- )
- .unwrap(),
- );
+ let case: Arc<dyn PhysicalExpr> = Arc::new(phys_expr::CaseExpr::try_new(
+ Some(literal(ScalarValue::UInt64(Some(0)))),
+ vec![
+ (
+ literal(ScalarValue::UInt64(Some(0))),
+ hash_lookup(&column, &[10, 20, 30])?,
+ ),
+ (
+ literal(ScalarValue::UInt64(Some(1))),
+ hash_lookup(&column, &[40, 50, 60])?,
+ ),
+ ],
+ // Partitions with no build rows reject everything routed to them.
+ Some(literal(ScalarValue::Boolean(Some(false)))),
+ )?);
let predicate = PruningPredicateBuilder::new()
.with_file_schema(Arc::clone(&schema))
- .try_build(case)
- .unwrap();
+ .try_build(case)?;
let statistics = TestStatistics::new().with(
"b",
@@ -7605,10 +7604,11 @@ mod tests {
),
);
- let result = predicate.prune(&statistics).unwrap();
+ let result = predicate.prune(&statistics)?;
// Container 1 ([15,25]) holds 20 from the first branch and container 2
// ([45,55]) holds 50 from the second, so a branch may match in each. The
// other two intersect neither branch, nor the `ELSE`.
assert_eq!(result, vec![false, true, true, false]);
+ Ok(())
}
}
diff --git a/datafusion/sqllogictest/test_files/range_partitioning.slt b/datafusion/sqllogictest/test_files/range_partitioning.slt
index 4573f28b7f..f389e265f8 100644
--- a/datafusion/sqllogictest/test_files/range_partitioning.slt
+++ b/datafusion/sqllogictest/test_files/range_partitioning.slt
@@ -1886,10 +1886,9 @@ SELECT range_key, SUM(value) FROM (
##########
# TEST 48: Hash Join Dynamic Filter Pushdown on Compatible Range Inputs
-# The Parquet-backed probe accepts the partition-routed dynamic filter. Matching
-# Range split points keep build filter i aligned with probe partition i. The
-# build has rows only in partitions 0 and 2, so the sparse runtime CASE includes
-# only those branches and lets empty partitions fall through to false.
+# The Parquet-backed probe accepts the dynamic filter. The build has rows only in
+# partitions 0 and 2, and both chose an IN list, so they merge into one IN list
+# with combined bounds instead of a per-partition CASE.
##########
statement ok
@@ -1930,7 +1929,7 @@ JOIN range_partitioned p ON b.range_key = p.range_key;
Plan with Metrics
01)HashJoinExec: mode=Partitioned<slt:ignore>metrics=[output_rows=2,<slt:ignore>]
02)--DataSourceExec: <slt:ignore>file_type=parquet, predicate=range_key@0 IN (SET) ([5, 20])<slt:ignore>metrics=[output_rows=2,<slt:ignore>]
-03)--DataSourceExec: <slt:ignore>file_type=parquet, predicate=DynamicFilter [ CASE range_partition WHEN 0 THEN range_key@0 >= 5 AND range_key@0 <= 5 AND range_key@0 IN (SET) ([5]) WHEN 2 THEN range_key@0 >= 20 AND range_key@0 <= 20 AND range_key@0 IN (SET) ([20]) ELSE false END ]<slt:ignore>metrics=[output_rows=2,<slt:ignore>]
+03)--DataSourceExec: <slt:ignore>file_type=parquet, predicate=DynamicFilter [ range_key@0 >= 5 AND range_key@0 <= 20 AND range_key@0 IN (SET) ([5, 20]) ]<slt:ignore>metrics=[output_rows=2,<slt:ignore>]
query III
SELECT b.range_key, b.value, p.value
diff --git a/xtask/src/ci_steps.rs b/xtask/src/ci_steps.rs
index a23dd563af..4ece399a79 100644
--- a/xtask/src/ci_steps.rs
+++ b/xtask/src/ci_steps.rs
@@ -15,8 +15,12 @@ type StepRunner = fn(&StepContext, &[String]) -> StepResult<CommandSpec>;
const CI_COMMAND: &str = "cargo xtask ci step";
const CI_SHORTCUT: &str = "cargo ci-step";
const TPCH_DATA_DIR: &str = "datafusion/sqllogictest/test_files/tpch/data";
-/// The `datafusion-physical-plan` unit test that the `spill-pool-fuzz` step runs.
-const SPILL_POOL_FUZZ_TEST: &str = "spill::spill_pool::tests::spill_pool_scenario_fuzz";
+/// The `datafusion-physical-plan` unit tests that the `fuzz` step runs.
+const FUZZ_TESTS: &[&str] = &[
+ "spill::spill_pool::tests::spill_pool_scenario_fuzz",
+ "joins::hash_join::partitioned_hash_eval::tests::routed_lookup_fuzz",
+ "joins::key_range_bitmap::tests::union_fuzz",
+];
/// Metadata and implementation for one CI step.
///
@@ -40,7 +44,7 @@ static CI_STEPS: &[StepInfo] = &[
},
StepInfo {
command: "test",
- help_usage: "test <workspace|slt-coverage|cli|doctest|ffi|microbenchmarks|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|spill-pool-fuzz|hash-collisions|sqlite> [--explain]",
+ help_usage: "test <workspace|slt-coverage|cli|doctest|ffi|microbenchmarks|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|fuzz|hash-collisions|sqlite> [--explain]",
help_examples: &["test cli", "test workspace"],
help_description: "Run a CI test suite",
runner: StepContext::run_test,
@@ -459,10 +463,10 @@ impl StepContext {
])
.env("RUST_BACKTRACE", "1");
}
- "spill-pool-fuzz" => {
+ "fuzz" => {
cmd_builder.doc(
- "Run more random spill pool fuzzer scenarios than the `workspace`\n\
- step. This takes about a minute.",
+ "Run more random fuzzer scenarios than the `workspace` step. This\n\
+ takes about a minute for the spill pool, seconds for the others.",
);
cmd_builder
.program("cargo")
@@ -475,10 +479,12 @@ impl StepContext {
"--lib",
"--",
"--exact",
- SPILL_POOL_FUZZ_TEST,
])
+ .args(FUZZ_TESTS)
.env("RUST_BACKTRACE", "1")
- .env("DATAFUSION_SPILL_POOL_FUZZ_ITERATIONS", "1000");
+ .env("DATAFUSION_SPILL_POOL_FUZZ_ITERATIONS", "1000")
+ .env("DATAFUSION_ROUTED_LOOKUP_FUZZ_ITERATIONS", "10000")
+ .env("DATAFUSION_KEY_RANGE_UNION_FUZZ_ITERATIONS", "10000");
}
"hash-collisions" => {
cmd_builder.doc(
@@ -559,7 +565,7 @@ mod tests {
use super::*;
use std::ffi::OsStr;
- const TEST_USAGE: &str = "usage: cargo xtask ci step test <workspace|slt-coverage|cli|doctest|ffi|microbenchmarks|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|spill-pool-fuzz|hash-collisions|sqlite> [--explain]";
+ const TEST_USAGE: &str = "usage: cargo xtask ci step test <workspace|slt-coverage|cli|doctest|ffi|microbenchmarks|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|fuzz|hash-collisions|sqlite> [--explain]";
fn args(args: &[&str]) -> Vec<String> {
args.iter().map(|arg| (*arg).to_string()).collect()
@@ -609,22 +615,24 @@ mod tests {
}
/// A filter that matches no test passes without running anything, so a
- /// renamed or moved spill pool fuzzer would silently stop running in CI.
+ /// renamed or moved fuzzer would silently stop running in CI.
#[test]
- fn spill_pool_fuzz_test_exists() {
- let (module, test) = SPILL_POOL_FUZZ_TEST
- .rsplit_once("::tests::")
- .expect("test path has a `tests` module");
- let file = Path::new(env!("CARGO_MANIFEST_DIR"))
- .join("../datafusion/physical-plan/src")
- .join(format!("{}.rs", module.replace("::", "/")));
- let source = std::fs::read_to_string(&file)
- .unwrap_or_else(|e| panic!("cannot read {}: {e}", file.display()));
- assert!(
- source.contains("mod tests {") && source.contains(&format!("fn {test}(")),
- "{} has no test `{test}` in `mod tests`; update SPILL_POOL_FUZZ_TEST",
- file.display()
- );
+ fn fuzz_tests_exist() {
+ for path in FUZZ_TESTS {
+ let (module, test) = path
+ .rsplit_once("::tests::")
+ .expect("test path has a `tests` module");
+ let file = Path::new(env!("CARGO_MANIFEST_DIR"))
+ .join("../datafusion/physical-plan/src")
+ .join(format!("{}.rs", module.replace("::", "/")));
+ let source = std::fs::read_to_string(&file)
+ .unwrap_or_else(|e| panic!("cannot read {}: {e}", file.display()));
+ assert!(
+ source.contains("mod tests {") && source.contains(&format!("fn {test}(")),
+ "{} has no test `{test}` in `mod tests`; update FUZZ_TESTS",
+ file.display()
+ );
+ }
}
/// The `extended` step runs only the `extended_tests` test target, which
@@ -943,20 +951,22 @@ cargo test \
},
},
Cmd {
- command: &["test", "spill-pool-fuzz"],
+ command: &["test", "fuzz"],
expected: |actual| {
insta::assert_snapshot!(actual, @r"
-# Run more random spill pool fuzzer scenarios than the `workspace`
-# step. This takes about a minute.
+# Run more random fuzzer scenarios than the `workspace` step. This
+# takes about a minute for the spill pool, seconds for the others.
cd /workspace && \
+DATAFUSION_KEY_RANGE_UNION_FUZZ_ITERATIONS=10000 \
+DATAFUSION_ROUTED_LOOKUP_FUZZ_ITERATIONS=10000 \
DATAFUSION_SPILL_POOL_FUZZ_ITERATIONS=1000 \
RUST_BACKTRACE=1 \
cargo test \
--profile ci \
-p datafusion-physical-plan \
--lib \
--- --exact spill::spill_pool::tests::spill_pool_scenario_fuzz
+-- --exact spill::spill_pool::tests::spill_pool_scenario_fuzz joins::hash_join::partitioned_hash_eval::tests::routed_lookup_fuzz joins::key_range_bitmap::tests::union_fuzz
")
},
},