Skip to content

Apache DataFusion SQL Query Engine

perf: Improve NLJ for very small right side case (#17562)

Yongting You2a6f3aafcaf3
diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rs
index 9377ace33a..a6b3df8819 100644
--- a/datafusion/physical-plan/src/joins/nested_loop_join.rs
+++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs
@@ -50,7 +50,7 @@ use crate::{
use arrow::array::{
new_null_array, Array, BooleanArray, BooleanBufferBuilder, RecordBatchOptions,
- UInt64Array,
+ UInt32Array, UInt64Array,
};
use arrow::buffer::BooleanBuffer;
use arrow::compute::{
@@ -1269,11 +1269,51 @@ impl NestedLoopJoinStream {
// and push the result into output_buffer
// ========
+ // Special case:
+ // When the right batch is very small, join with multiple left rows at once,
+ //
+ // The regular implementation is not efficient if the plan's right child is
+ // very small (e.g. 1 row total), because inside the inner loop of NLJ, it's
+ // handling one input right batch at once, if it's not large enough, the
+ // overheads like filter evaluation can't be amortized through vectorization.
+ debug_assert_ne!(
+ right_batch.num_rows(),
+ 0,
+ "When fetching the right batch, empty batches will be skipped"
+ );
+
+ let l_row_cnt_ratio = self.batch_size / right_batch.num_rows();
+ if l_row_cnt_ratio > 10 {
+ // Calculate max left rows to handle at once. This operator tries to handle
+ // up to `datafusion.execution.batch_size` rows at once in the intermediate
+ // batch.
+ let l_row_count = std::cmp::min(
+ l_row_cnt_ratio,
+ left_data.batch().num_rows() - self.left_probe_idx,
+ );
+
+ debug_assert!(l_row_count != 0, "This function should only be entered when there are remaining left rows to process");
+ let joined_batch = self.process_left_range_join(
+ &left_data,
+ &right_batch,
+ self.left_probe_idx,
+ l_row_count,
+ )?;
+
+ if let Some(batch) = joined_batch {
+ self.output_buffer.push_batch(batch)?;
+ }
+
+ self.left_probe_idx += l_row_count;
+
+ return Ok(true);
+ }
+
let l_idx = self.left_probe_idx;
- let join_batch =
+ let joined_batch =
self.process_single_left_row_join(&left_data, &right_batch, l_idx)?;
- if let Some(batch) = join_batch {
+ if let Some(batch) = joined_batch {
self.output_buffer.push_batch(batch)?;
}
@@ -1286,8 +1326,196 @@ impl NestedLoopJoinStream {
Ok(true)
}
+ /// Process [l_start_index, l_start_index + l_count) JOIN right_batch
+ /// Returns a RecordBatch containing the join results (None if empty)
+ ///
+ /// Side Effect: If the join type requires, left or right side matched bitmap
+ /// will be set for matched indices.
+ fn process_left_range_join(
+ &mut self,
+ left_data: &JoinLeftData,
+ right_batch: &RecordBatch,
+ l_start_index: usize,
+ l_row_count: usize,
+ ) -> Result<Option<RecordBatch>> {
+ // Construct the Cartesian product between the specified range of left rows
+ // and the entire right_batch. First, it calculates the index vectors, then
+ // materializes the intermediate batch, and finally applies the join filter
+ // to it.
+ // -----------------------------------------------------------
+ let right_rows = right_batch.num_rows();
+ let total_rows = l_row_count * right_rows;
+
+ // Build index arrays for cartesian product: left_range X right_batch
+ let left_indices: UInt32Array =
+ UInt32Array::from_iter_values((0..l_row_count).flat_map(|i| {
+ std::iter::repeat_n((l_start_index + i) as u32, right_rows)
+ }));
+ let right_indices: UInt32Array = UInt32Array::from_iter_values(
+ (0..l_row_count).flat_map(|_| 0..right_rows as u32),
+ );
+
+ debug_assert!(
+ left_indices.len() == right_indices.len()
+ && right_indices.len() == total_rows,
+ "The length or cartesian product should be (left_size * right_size)",
+ );
+
+ // Evaluate the join filter (if any) over an intermediate batch built
+ // using the filter's own schema/column indices.
+ let bitmap_combined = if let Some(filter) = &self.join_filter {
+ // Build the intermediate batch for filter evaluation
+ let intermediate_batch = if filter.schema.fields().is_empty() {
+ // Constant predicate (e.g., TRUE/FALSE). Use an empty schema with row_count
+ create_record_batch_with_empty_schema(
+ Arc::new((*filter.schema).clone()),
+ total_rows,
+ )?
+ } else {
+ let mut filter_columns: Vec<Arc<dyn Array>> =
+ Vec::with_capacity(filter.column_indices().len());
+ for column_index in filter.column_indices() {
+ let array = if column_index.side == JoinSide::Left {
+ let col = left_data.batch().column(column_index.index);
+ take(col.as_ref(), &left_indices, None)?
+ } else {
+ let col = right_batch.column(column_index.index);
+ take(col.as_ref(), &right_indices, None)?
+ };
+ filter_columns.push(array);
+ }
+
+ RecordBatch::try_new(Arc::new((*filter.schema).clone()), filter_columns)?
+ };
+
+ let filter_result = filter
+ .expression()
+ .evaluate(&intermediate_batch)?
+ .into_array(intermediate_batch.num_rows())?;
+ let filter_arr = as_boolean_array(&filter_result)?;
+
+ // Combine with null bitmap to get a unified mask
+ boolean_mask_from_filter(filter_arr)
+ } else {
+ // No filter: all pairs match
+ BooleanArray::from(vec![true; total_rows])
+ };
+
+ // Update the global left or right bitmap for matched indices
+ // -----------------------------------------------------------
+
+ // None means we don't have to update left bitmap for this join type
+ let mut left_bitmap = if need_produce_result_in_final(self.join_type) {
+ Some(left_data.bitmap().lock())
+ } else {
+ None
+ };
+
+ // 'local' meaning: we want to collect 'is_matched' flag for the current
+ // right batch, after it has joining all of the left buffer, here it's only
+ // the partial result for joining given left range
+ let mut local_right_bitmap = if self.should_track_unmatched_right {
+ let mut current_right_batch_bitmap = BooleanBufferBuilder::new(right_rows);
+ // Ensure builder has logical length so set_bit is in-bounds
+ current_right_batch_bitmap.append_n(right_rows, false);
+ Some(current_right_batch_bitmap)
+ } else {
+ None
+ };
+
+ // Set the matched bit for left and right side bitmap
+ for (i, is_matched) in bitmap_combined.iter().enumerate() {
+ let is_matched = is_matched.ok_or_else(|| {
+ internal_datafusion_err!("Must be Some after the previous combining step")
+ })?;
+
+ let l_index = l_start_index + i / right_rows;
+ let r_index = i % right_rows;
+
+ if let Some(bitmap) = left_bitmap.as_mut() {
+ if is_matched {
+ // Map local index back to absolute left index within the batch
+ bitmap.set_bit(l_index, true);
+ }
+ }
+
+ if let Some(bitmap) = local_right_bitmap.as_mut() {
+ if is_matched {
+ bitmap.set_bit(r_index, true);
+ }
+ }
+ }
+
+ // Apply the local right bitmap to the global bitmap
+ if self.should_track_unmatched_right {
+ // Remember to put it back after update
+ let global_right_bitmap =
+ std::mem::take(&mut self.current_right_batch_matched).ok_or_else(
+ || internal_datafusion_err!("right batch's bitmap should be present"),
+ )?;
+ let (buf, nulls) = global_right_bitmap.into_parts();
+ debug_assert!(nulls.is_none());
+
+ let current_right_bitmap = local_right_bitmap
+ .ok_or_else(|| {
+ internal_datafusion_err!(
+ "Should be Some if the current join type requires right bitmap"
+ )
+ })?
+ .finish();
+ let updated_global_right_bitmap = buf.bitor(&current_right_bitmap);
+
+ self.current_right_batch_matched =
+ Some(BooleanArray::new(updated_global_right_bitmap, None));
+ }
+
+ // For the following join types: only bitmaps are updated; do not emit rows now
+ if matches!(
+ self.join_type,
+ JoinType::LeftAnti
+ | JoinType::LeftSemi
+ | JoinType::LeftMark
+ | JoinType::RightAnti
+ | JoinType::RightMark
+ | JoinType::RightSemi
+ ) {
+ return Ok(None);
+ }
+
+ // Build the projected output batch (using output schema/column_indices),
+ // then apply the bitmap filter to it.
+ if self.output_schema.fields().is_empty() {
+ // Empty projection: only row count matters
+ let row_count = bitmap_combined.true_count();
+ return Ok(Some(create_record_batch_with_empty_schema(
+ Arc::clone(&self.output_schema),
+ row_count,
+ )?));
+ }
+
+ let mut out_columns: Vec<Arc<dyn Array>> =
+ Vec::with_capacity(self.output_schema.fields().len());
+ for column_index in &self.column_indices {
+ let array = if column_index.side == JoinSide::Left {
+ let col = left_data.batch().column(column_index.index);
+ take(col.as_ref(), &left_indices, None)?
+ } else {
+ let col = right_batch.column(column_index.index);
+ take(col.as_ref(), &right_indices, None)?
+ };
+ out_columns.push(array);
+ }
+ let pre_filtered =
+ RecordBatch::try_new(Arc::clone(&self.output_schema), out_columns)?;
+ let filtered = filter_record_batch(&pre_filtered, &bitmap_combined)?;
+ Ok(Some(filtered))
+ }
+
/// Process a single left row join with the current right batch.
/// Returns a RecordBatch containing the join results (None if empty)
+ ///
+ /// Side Effect: If the join type requires, left or right side matched bitmap
+ /// will be set for matched indices.
fn process_single_left_row_join(
&mut self,
left_data: &JoinLeftData,
@@ -1584,22 +1812,26 @@ fn apply_filter_to_row_join_batch(
.into_array(intermediate_batch.num_rows())?;
let filter_arr = as_boolean_array(&filter_result)?;
- // [Caution] This step has previously introduced bugs
- // The filter result is NOT a bitmap; it contains true/false/null values.
- // For example, 1 < NULL is evaluated to NULL. Therefore, we must combine (AND)
- // the boolean array with its null bitmap to construct a unified bitmap.
- let (is_filtered, nulls) = filter_arr.clone().into_parts();
- let bitmap_combined = match nulls {
- Some(nulls) => {
- let combined = nulls.inner() & &is_filtered;
- BooleanArray::new(combined, None)
- }
- None => BooleanArray::new(is_filtered, None),
- };
+ // Convert boolean array with potential nulls into a unified mask bitmap
+ let bitmap_combined = boolean_mask_from_filter(filter_arr);
Ok(bitmap_combined)
}
+/// Convert a boolean filter array into a unified mask bitmap.
+///
+/// Caution: The filter result is NOT a bitmap; it contains true/false/null values.
+/// For example, `1 < NULL` evaluates to NULL. Therefore, we must combine (AND)
+/// the boolean array with its null bitmap to construct a unified bitmap.
+#[inline]
+fn boolean_mask_from_filter(filter_arr: &BooleanArray) -> BooleanArray {
+ let (values, nulls) = filter_arr.clone().into_parts();
+ match nulls {
+ Some(nulls) => BooleanArray::new(nulls.inner() & &values, None),
+ None => BooleanArray::new(values, None),
+ }
+}
+
/// This function performs the following steps:
/// 1. Apply filter to probe-side batch
/// 2. Broadcast the left row (build_side_batch\[build_side_index\]) to the