Skip to content

Apache DataFusion SQL Query Engine

perf: reuse evaluated hash join build keys (#27285)

Phoenixbc89cf94b3a5
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs
index 3dc705d34e..f1d393e6ff 100644
--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs
@@ -160,30 +160,25 @@ fn subtree_max_len(data: &ArrayData) -> usize {
///
/// On `Some`, the memory of the [`ArrayMap`] has already been added to
/// `reservation`; the caller creates the map with [`ArrayMap::try_new`].
-#[expect(clippy::too_many_arguments)]
fn array_map_key_range(
bounds: Option<&PartitionBounds>,
- schema: &SchemaRef,
- batches: &[RecordBatch],
- on_left: &[PhysicalExprRef],
+ keys: &[ArrayRef],
reservation: &mut MemoryReservation,
perfect_hash_join_small_build_threshold: usize,
perfect_hash_join_min_key_density: f64,
null_equality: NullEquality,
) -> Result<Option<(u64, u64)>> {
+ let [key] = keys else {
+ return Ok(None);
+ };
// `bounds` are also collected for dynamic filters, on any key type, so the
// key type cannot be inferred from their presence.
- if !is_perfect_hash_join_candidate(on_left, schema)? {
+ if !ArrayMap::is_supported_type(key.data_type()) {
return Ok(None);
}
- if null_equality == NullEquality::NullEqualsNull {
- for batch in batches.iter() {
- let arrays = evaluate_expressions_to_arrays(on_left, batch)?;
- if arrays[0].null_count() > 0 {
- return Ok(None);
- }
- }
+ if null_equality == NullEquality::NullEqualsNull && key.null_count() > 0 {
+ return Ok(None);
}
let (min_val, max_val) = if let Some(bounds) = bounds {
@@ -213,10 +208,10 @@ fn array_map_key_range(
};
let range = ArrayMap::calculate_range(min_val, max_val);
- let num_row: usize = batches.iter().map(|x| x.num_rows()).sum();
+ let num_rows = key.len();
// TODO: support create ArrayMap<u64>
- if num_row >= u32::MAX as usize {
+ if num_rows >= u32::MAX as usize {
return Ok(None);
}
@@ -226,7 +221,7 @@ fn array_map_key_range(
return Ok(None);
}
- let dense_ratio = (num_row as f64) / ((range + 1) as f64);
+ let dense_ratio = (num_rows as f64) / ((range + 1) as f64);
if range >= perfect_hash_join_small_build_threshold as u64
&& dense_ratio <= perfect_hash_join_min_key_density
@@ -234,7 +229,7 @@ fn array_map_key_range(
return Ok(None);
}
- let mem_size = ArrayMap::estimate_memory_size(min_val, max_val, num_row);
+ let mem_size = ArrayMap::estimate_memory_size(min_val, max_val, num_rows);
reservation.try_grow(mem_size)?;
Ok(Some((min_val, max_val)))
@@ -3269,9 +3264,7 @@ fn build_in_memory(
let join_hash_map = if let Some((min_val, max_val)) = array_map_key_range(
bounds.as_ref(),
- schema,
- std::slice::from_ref(&batch),
- on_left,
+ &left_values,
&mut map_reservation,
config.execution.perfect_hash_join_small_build_threshold,
config.execution.perfect_hash_join_min_key_density,