Skip to content

Apache DataFusion SQL Query Engine

chore: refactor `MaterializingSortMergeJoinStream` into generators and simplify code to be textbook like as possible (#23976)

Raz Luvaton39d50641424d
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs
index 3b597323b2..1abcd9d6c7 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs
@@ -81,8 +81,7 @@ use datafusion_physical_expr_common::sort_expr::{LexOrdering, OrderingRequiremen
/// on the output batch size of the execution plan. There is no spilling support for streamed input.
/// The comparisons are performed from values of join keys in streamed input with the values of
/// join keys in buffered input. One row in streamed record batch could be matched with multiple rows in
-/// buffered input batches. The streamed input is managed through the states in `StreamedState`
-/// and streamed input batches are represented by `StreamedBatch`.
+/// buffered input batches. Streamed input batches are represented by `StreamedBatch`.
///
/// Buffered input is buffered for all record batches having the same value of join key.
/// If the memory limit increases beyond the specified value and spilling is enabled,
@@ -92,8 +91,7 @@ use datafusion_physical_expr_common::sort_expr::{LexOrdering, OrderingRequiremen
/// memory/disk depends on the number of rows of buffered input having the same value
/// of join key as that of streamed input rows currently present in memory. Due to pre-sorted inputs,
/// the algorithm understands when it is not needed anymore, and releases the buffered batches
-/// from memory/disk. The buffered input is managed through the states in `BufferedState`
-/// and buffered input batches are represented by `BufferedBatch`.
+/// from memory/disk. Buffered input batches are represented by `BufferedBatch`.
///
/// Depending on the type of join, left or right input may be selected as streamed or buffered
/// respectively. For example, in a left-outer join, the left execution plan will be selected as
@@ -546,7 +544,7 @@ impl ExecutionPlan for SortMergeJoinExec {
context.runtime_env(),
)
} else {
- Ok(Box::pin(MaterializingSortMergeJoinStream::try_new(
+ MaterializingSortMergeJoinStream::try_new(
Arc::clone(&self.schema),
self.sort_options.clone(),
self.null_equality,
@@ -561,7 +559,7 @@ impl ExecutionPlan for SortMergeJoinExec {
reservation,
spill_manager,
context.runtime_env(),
- )?))
+ )
}
}
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs
index 51cf38b9ab..3baa0c4a3e 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs
@@ -17,20 +17,17 @@
//! Sort-Merge Join execution
//!
-//! This module implements the runtime state machine for the Sort-Merge Join
-//! operator. It drives two sorted input streams (the *streamed* side and the
-//! *buffered* side), compares join keys, and produces joined `RecordBatch`es.
+//! This module implements the Sort-Merge Join operator as an async
+//! generator running a merge scan: it drives two sorted input streams (the
+//! *streamed* side and the *buffered* side), compares join keys, and
+//! produces joined `RecordBatch`es.
use std::cmp::Ordering;
use std::collections::{HashMap, VecDeque};
use std::fmt::Debug;
use std::mem::size_of;
use std::ops::Range;
-use std::pin::Pin;
use std::sync::Arc;
-use std::sync::atomic::AtomicUsize;
-use std::sync::atomic::Ordering::Relaxed;
-use std::task::{Context, Poll};
use crate::joins::sort_merge_join::filter::{
FilterMetadata, filter_record_batch_by_join_type, get_corrected_filter_mask,
@@ -38,10 +35,10 @@ use crate::joins::sort_merge_join::filter::{
};
use crate::joins::sort_merge_join::metrics::SortMergeJoinMetrics;
use crate::joins::utils::{JoinFilter, JoinKeyComparator};
-use crate::metrics::RecordOutput;
+use crate::metrics::Time;
use crate::spill::spill_manager::SpillManager;
-use crate::stream::EmptyRecordBatchStream;
-use crate::{PhysicalExpr, RecordBatchStream, SendableRecordBatchStream};
+use crate::stream::{EmptyRecordBatchStream, ObservedStream, RecordBatchStreamAdapter};
+use crate::{PhysicalExpr, SendableRecordBatchStream};
use arrow::array::{types::UInt64Type, *};
use arrow::compute::{
@@ -50,56 +47,16 @@ use arrow::compute::{
};
use arrow::datatypes::SchemaRef;
use datafusion_common::cast::as_uint64_array;
-use datafusion_common::{JoinType, NullEquality, Result, exec_err, internal_err};
-use datafusion_execution::SpillFile;
+use datafusion_common::instant::Instant;
+use datafusion_common::{
+ DataFusionError, JoinType, NullEquality, Result, exec_err, internal_err,
+};
use datafusion_execution::memory_pool::MemoryReservation;
use datafusion_execution::runtime_env::RuntimeEnv;
+use datafusion_execution::{SpillFile, TryEmitter, async_try_stream};
use datafusion_physical_expr_common::physical_expr::PhysicalExprRef;
-use futures::{Stream, StreamExt, ready};
-
-/// State of SMJ stream
-#[derive(Debug, PartialEq, Eq)]
-pub(super) enum SortMergeJoinState {
- /// Init joining with a new streamed row or a new buffered batches
- Init,
- /// Polling one streamed row or one buffered batch, or both
- Polling,
- /// Joining polled data and making output
- JoinOutput,
- /// Emit ready data if have any and then go back to [`Self::Init`] state
- EmitReadyThenInit,
- /// No more output
- Exhausted,
-}
-
-/// State of streamed data stream
-#[derive(Debug, PartialEq, Eq)]
-pub(super) enum StreamedState {
- /// Init polling
- Init,
- /// Polling one streamed row
- Polling,
- /// Ready to produce one streamed row
- Ready,
- /// No more streamed row
- Exhausted,
-}
-
-/// State of buffered data stream
-#[derive(Debug, PartialEq, Eq)]
-pub(super) enum BufferedState {
- /// Init polling
- Init,
- /// Polling first row in the next batch
- PollingFirst,
- /// Polling rest rows in the next batch
- PollingRest,
- /// Ready to produce one batch
- Ready,
- /// No more buffered batches
- Exhausted,
-}
+use futures::StreamExt;
/// Represents a chunk of joined data from streamed and buffered side
pub(super) struct StreamedJoinedChunk {
@@ -335,6 +292,9 @@ pub(super) struct MaterializingSortMergeJoinStream {
pub filter: Option<JoinFilter>,
/// How the join is performed
pub join_type: JoinType,
+ /// Cached `needs_deferred_filtering(filter, join_type)` — both inputs
+ /// are fixed at construction time.
+ pub deferred_filtering: bool,
/// Target output batch size
pub batch_size: usize,
@@ -348,10 +308,8 @@ pub(super) struct MaterializingSortMergeJoinStream {
pub streamed: SendableRecordBatchStream,
/// Current processing record batch of streamed
pub streamed_batch: StreamedBatch,
- /// (used in outer join) Is current streamed row joined at least once?
- pub streamed_joined: bool,
- /// State of streamed
- pub streamed_state: StreamedState,
+ /// True once the streamed input has no more rows
+ pub streamed_exhausted: bool,
/// Join key columns of streamed
pub on_streamed: Vec<PhysicalExprRef>,
@@ -365,10 +323,11 @@ pub(super) struct MaterializingSortMergeJoinStream {
pub buffered: SendableRecordBatchStream,
/// Current buffered data
pub buffered_data: BufferedData,
- /// (used in outer join) Is current buffered batches joined at least once?
- pub buffered_joined: bool,
- /// State of buffered
- pub buffered_state: BufferedState,
+ /// Has any streamed row matched the current buffered key group?
+ /// (FULL join: an unmatched group is emitted null-joined when passed.)
+ pub buffered_group_matched: bool,
+ /// True once the buffered input has no more rows and no group remains
+ pub buffered_exhausted: bool,
/// Join key columns of buffered
pub on_buffered: Vec<PhysicalExprRef>,
@@ -377,23 +336,26 @@ pub(super) struct MaterializingSortMergeJoinStream {
// These fields track the execution state of merge join and are updated
// during the execution.
// ========================================================================
- /// Current state of the stream
- pub state: SortMergeJoinState,
/// Staging output array builders
pub joined_record_batches: JoinedRecordBatches,
/// Output buffer. Currently used by filtering as it requires double buffering
- /// to avoid small/empty batches. Non-filtered join outputs directly from `staging_output_record_batches.batches`
+ /// to avoid small/empty batches. Non-filtered joins output directly from
+ /// `joined_record_batches.joined_batches`
pub output: BatchCoalescer,
- /// The comparison result of current streamed row and buffered batches
- pub current_ordering: Ordering,
/// Manages the process of spilling and reading back intermediate data
pub spill_manager: SpillManager,
- /// Tracks the active stream when loading spilled buffered batches back in memory
- pub spill_stream: Option<SendableRecordBatchStream>,
/// Tracks the number of batches currently spilled
pub spilled_batch_count: usize,
+ /// Time spent doing the join's own work (including spill write and
+ /// read-back). The clock is stopped while awaiting the child inputs or
+ /// the consumer taking an emitted batch — see [`Self::stop_join_time`].
+ pub join_time: Time,
+ /// Start of the currently running `join_time` span; `None` while the
+ /// clock is stopped.
+ pub join_time_start: Option<Instant>,
+
// ========================================================================
// CACHED COMPARATORS:
// Pre-built comparators to avoid per-row type dispatch in hot loops.
@@ -413,8 +375,9 @@ pub(super) struct MaterializingSortMergeJoinStream {
pub reservation: MemoryReservation,
/// Runtime env
pub runtime_env: Arc<RuntimeEnv>,
- /// A unique number for each batch
- pub streamed_batch_counter: AtomicUsize,
+ /// A unique id per streamed batch, tagging deferred-filter metadata so
+ /// `get_corrected_filter_mask` can group output rows by input batch.
+ pub streamed_batch_counter: usize,
}
/// Staging area for joined data before output
@@ -560,264 +523,6 @@ impl JoinedRecordBatches {
self.debug_assert_empty_consistency();
}
}
-impl RecordBatchStream for MaterializingSortMergeJoinStream {
- fn schema(&self) -> SchemaRef {
- Arc::clone(&self.schema)
- }
-}
-
-impl Stream for MaterializingSortMergeJoinStream {
- type Item = Result<RecordBatch>;
-
- fn poll_next(
- mut self: Pin<&mut Self>,
- cx: &mut Context<'_>,
- ) -> Poll<Option<Self::Item>> {
- let join_time = self.join_metrics.join_time().clone();
- let _timer = join_time.timer();
- loop {
- match &self.state {
- SortMergeJoinState::Init => {
- let streamed_exhausted =
- self.streamed_state == StreamedState::Exhausted;
- let buffered_exhausted =
- self.buffered_state == BufferedState::Exhausted;
- self.state = if streamed_exhausted && buffered_exhausted {
- SortMergeJoinState::Exhausted
- } else {
- match self.current_ordering {
- Ordering::Less | Ordering::Equal => {
- if !streamed_exhausted {
- // Batch deferred filtering: process_filtered_batches()
- // only when >= batch_size rows have accumulated.
- // Without this gate, unique keys cause per-row pipeline
- // execution (concat + correct_mask + filter_by_type),
- // which dominates runtime.
- //
- // Accumulated rows are bounded to ~2*batch_size:
- // one batch_size worth from freeze_dequeuing_buffered()
- // (when an input batch is fully consumed), plus up to
- // batch_size pairs accumulating toward the next freeze.
- // This does not reintroduce the unbounded buffering
- // fixed by PR #20482. Exhausted state flushes remainder.
- if needs_deferred_filtering(
- &self.filter,
- self.join_type,
- ) {
- let accumulated = self.num_unfrozen_pairs()
- + self
- .joined_record_batches
- .filter_metadata
- .filter_mask
- .len();
- if accumulated >= self.batch_size {
- // Ensure required spilled batches are restored to memory
- // before processing, as this path invokes freeze_all().
- let needed = self.get_required_batch_indices(
- self.buffered_data.batches.len(),
- );
- if let Err(e) = ready!(
- self.poll_spilled_batches(cx, &needed)
- ) {
- return Poll::Ready(Some(Err(e)));
- }
- match self.process_filtered_batches()? {
- Poll::Ready(Some(batch)) => {
- return Poll::Ready(Some(Ok(batch)));
- }
- Poll::Ready(None) | Poll::Pending => {}
- }
- }
- }
-
- self.streamed_joined = false;
- self.streamed_state = StreamedState::Init;
- }
- }
- Ordering::Greater => {
- if !buffered_exhausted {
- self.buffered_joined = false;
- self.buffered_state = BufferedState::Init;
- }
- }
- }
- SortMergeJoinState::Polling
- };
- }
- SortMergeJoinState::Polling => {
- if ![StreamedState::Exhausted, StreamedState::Ready]
- .contains(&self.streamed_state)
- {
- match self.poll_streamed_row(cx)? {
- Poll::Ready(_) => {}
- Poll::Pending => return Poll::Pending,
- }
- }
-
- if ![BufferedState::Exhausted, BufferedState::Ready]
- .contains(&self.buffered_state)
- {
- match self.poll_buffered_batches(cx)? {
- Poll::Ready(_) => {}
- Poll::Pending => return Poll::Pending,
- }
- }
- let streamed_exhausted =
- self.streamed_state == StreamedState::Exhausted;
- let buffered_exhausted =
- self.buffered_state == BufferedState::Exhausted;
- if streamed_exhausted && buffered_exhausted {
- self.state = SortMergeJoinState::Exhausted;
- continue;
- }
- self.current_ordering = self.compare_streamed_buffered()?;
- self.state = SortMergeJoinState::JoinOutput;
- }
- SortMergeJoinState::EmitReadyThenInit => {
- // If have data to emit, emit it and if no more, change to next
-
- // Verify metadata alignment before checking if we have batches to output
- self.joined_record_batches
- .filter_metadata
- .debug_assert_metadata_aligned();
-
- // For filtered joins, skip output and let Init state handle it
- if needs_deferred_filtering(&self.filter, self.join_type) {
- self.state = SortMergeJoinState::Init;
- continue;
- }
-
- // For non-filtered joins, only output if we have a completed batch
- // (opportunistic output when target batch size is reached)
- if self
- .joined_record_batches
- .joined_batches
- .has_completed_batch()
- {
- let record_batch = self
- .joined_record_batches
- .joined_batches
- .next_completed_batch()
- .expect("has_completed_batch was true");
- (&record_batch)
- .record_output(&self.join_metrics.baseline_metrics());
- return Poll::Ready(Some(Ok(record_batch)));
- }
- self.state = SortMergeJoinState::Init;
- }
- SortMergeJoinState::JoinOutput => {
- // If the batch size limit is reached, restore required spilled batches to memory and freeze.
- // Guarding at the top of the loop safely handles re-entry from Poll::Pending.
- if self.num_unfrozen_pairs() >= self.batch_size {
- let needed = self
- .get_required_batch_indices(self.buffered_data.batches.len());
- ready!(self.poll_spilled_batches(cx, &needed))?;
-
- self.freeze_all()?;
-
- // Verify metadata alignment before checking if we have batches to output
- self.joined_record_batches
- .filter_metadata
- .debug_assert_metadata_aligned();
-
- // For filtered joins, skip output and let Init state handle it
- if needs_deferred_filtering(&self.filter, self.join_type) {
- continue;
- }
-
- // For non-filtered joins, only output if we have a completed batch
- if self
- .joined_record_batches
- .joined_batches
- .has_completed_batch()
- {
- let record_batch = self
- .joined_record_batches
- .joined_batches
- .next_completed_batch()
- .expect("has_completed_batch was true");
- (&record_batch)
- .record_output(&self.join_metrics.baseline_metrics());
- return Poll::Ready(Some(Ok(record_batch)));
- }
-
- // Otherwise keep buffering (don't output yet)
- continue;
- }
-
- self.join_partial()?;
-
- if self.num_unfrozen_pairs() < self.batch_size
- && self.buffered_data.scanning_finished()
- {
- self.buffered_data.scanning_reset();
- self.state = SortMergeJoinState::EmitReadyThenInit;
- }
- // Note: If join_partial() reached the batch size, the loop repeats to freeze the data.
- }
- SortMergeJoinState::Exhausted => {
- let needed =
- self.get_required_batch_indices(self.buffered_data.batches.len());
- ready!(self.poll_spilled_batches(cx, &needed))?;
-
- self.freeze_all()?;
-
- // Verify metadata alignment before final output
- self.joined_record_batches
- .filter_metadata
- .debug_assert_metadata_aligned();
-
- // For filtered joins, must concat and filter ALL data at once
- if needs_deferred_filtering(&self.filter, self.join_type)
- && !self.joined_record_batches.joined_batches.is_empty()
- {
- let record_batch = self.filter_joined_batch()?;
- (&record_batch)
- .record_output(&self.join_metrics.baseline_metrics());
- return Poll::Ready(Some(Ok(record_batch)));
- }
-
- // For non-filtered joins, finish buffered data first
- if !self.joined_record_batches.joined_batches.is_empty() {
- self.joined_record_batches
- .joined_batches
- .finish_buffered_batch()?;
- }
-
- // Output one completed batch at a time (stay in Exhausted until empty)
- if self
- .joined_record_batches
- .joined_batches
- .has_completed_batch()
- {
- let record_batch = self
- .joined_record_batches
- .joined_batches
- .next_completed_batch()
- .expect("has_completed_batch was true");
- (&record_batch)
- .record_output(&self.join_metrics.baseline_metrics());
- return Poll::Ready(Some(Ok(record_batch)));
- }
-
- // Finally check self.output BatchCoalescer (used by filtered joins)
- return if !self.output.is_empty() {
- self.output.finish_buffered_batch()?;
- let record_batch = self
- .output
- .next_completed_batch()
- .expect("Failed to get last batch");
- (&record_batch)
- .record_output(&self.join_metrics.baseline_metrics());
- Poll::Ready(Some(Ok(record_batch)))
- } else {
- Poll::Ready(None)
- };
- }
- }
- }
- }
-}
impl MaterializingSortMergeJoinStream {
#[expect(clippy::too_many_arguments)]
@@ -836,7 +541,7 @@ impl MaterializingSortMergeJoinStream {
reservation: MemoryReservation,
spill_manager: SpillManager,
runtime_env: Arc<RuntimeEnv>,
- ) -> Result<Self> {
+ ) -> Result<SendableRecordBatchStream> {
let streamed_schema = streamed.schema();
let buffered_schema = buffered.schema();
debug_assert!(
@@ -847,8 +552,8 @@ impl MaterializingSortMergeJoinStream {
"MaterializingSortMergeJoinStream does not handle {join_type:?}; \
semi/anti/mark joins use BitwiseSortMergeJoinStream"
);
- Ok(Self {
- state: SortMergeJoinState::Init,
+ let join_time = join_metrics.join_time();
+ let mut this = Self {
sort_options,
null_equality,
schema: Arc::clone(&schema),
@@ -858,13 +563,12 @@ impl MaterializingSortMergeJoinStream {
buffered,
streamed_batch: StreamedBatch::new_empty(streamed_schema),
buffered_data: BufferedData::default(),
- streamed_joined: false,
- buffered_joined: false,
- streamed_state: StreamedState::Init,
- buffered_state: BufferedState::Init,
- current_ordering: Ordering::Equal,
+ buffered_group_matched: false,
+ streamed_exhausted: false,
+ buffered_exhausted: false,
on_streamed,
on_buffered,
+ deferred_filtering: needs_deferred_filtering(&filter, join_type),
filter,
joined_record_batches: JoinedRecordBatches {
joined_batches: BatchCoalescer::new(Arc::clone(&schema), batch_size)
@@ -879,12 +583,299 @@ impl MaterializingSortMergeJoinStream {
reservation,
runtime_env,
spill_manager,
- spill_stream: None,
spilled_batch_count: 0,
+ join_time,
+ join_time_start: None,
streamed_buffered_cmp: None,
buffered_equality_cmp: None,
- streamed_batch_counter: AtomicUsize::new(0),
- })
+ streamed_batch_counter: 0,
+ };
+
+ let schema = Arc::clone(&this.schema);
+ let baseline_metrics = this.join_metrics.baseline_metrics();
+
+ let stream = async_try_stream(|mut emitter| async move {
+ this.start_join_time();
+ let result = this.join(&mut emitter).await;
+ this.stop_join_time();
+ result
+ });
+ // ObservedStream records the baseline metrics (output rows/batches,
+ // end time).
+ Ok(Box::pin(ObservedStream::new(
+ Box::pin(RecordBatchStreamAdapter::new(schema, stream)),
+ baseline_metrics,
+ None,
+ )))
+ }
+
+ /// Main loop: the textbook sort-merge join.
+ ///
+ /// Both inputs arrive sorted on the join keys. The streamed side is
+ /// consumed one row at a time; the buffered side one key *group* (all
+ /// contiguous rows sharing a key) at a time
+ async fn join(
+ &mut self,
+ emitter: &mut TryEmitter<RecordBatch, DataFusionError>,
+ ) -> Result<()> {
+ // 1. Load the first streamed row and the first buffered key group.
+ self.load_next_streamed_batch().await?;
+ self.advance_buffered_group().await?;
+
+ // 2. Merge-scan while either input still has rows.
+ while !(self.streamed_exhausted && self.buffered_exhausted) {
+ // Flush the deferred-filtering pipeline once a full batch of
+ // rows accumulated (filtered outer joins output through it).
+ if self.deferred_filtering
+ && self.deferred_rows_accumulated() >= self.batch_size
+ {
+ self.emit_deferred_output(emitter).await?;
+ }
+
+ // 3. Compare the join keys at both cursors. An exhausted side
+ // compares as the larger one, so the other side keeps
+ // draining through its own arm.
+ match self.compare_streamed_buffered()? {
+ // 3a. The streamed row can never match: null-join it (outer
+ // joins emit it; inner joins drop it), then advance.
+ Ordering::Less => {
+ self.null_join_streamed_row();
+ if self.num_unfrozen_pairs() >= self.batch_size {
+ self.freeze_and_emit(emitter).await?;
+ }
+ if !self.try_advance_streamed_row() {
+ self.load_next_streamed_batch().await?;
+ }
+ }
+ // 3b. The buffered group can never match again: null-join
+ // it if nothing matched it (FULL join), then advance to
+ // the next key group.
+ Ordering::Greater => {
+ self.null_join_buffered_group();
+ if !self.try_advance_buffered_group()? {
+ self.advance_buffered_group().await?;
+ }
+ }
+ // 3c. Match: pair the streamed row with the whole group —
+ // materializing ("freezing") mid-scan whenever a full
+ // batch of pairs accumulates — then advance streamed.
+ // The group stays for the next streamed row.
+ Ordering::Equal => {
+ while !self.pair_streamed_row_with_group() {
+ self.freeze_and_emit(emitter).await?;
+ }
+ if !self.try_advance_streamed_row() {
+ self.load_next_streamed_batch().await?;
+ }
+ }
+ }
+
+ // 4. Emit completed output batches (filtered joins emit
+ // through the deferred-filtering pipeline above instead).
+ if !self.deferred_filtering
+ && self
+ .joined_record_batches
+ .joined_batches
+ .has_completed_batch()
+ {
+ self.emit_completed_joined_batches(emitter).await;
+ }
+ }
+
+ // 5. Flush everything that remains.
+ self.on_children_exhausted(emitter).await
+ }
+
+ /// `Equal`: pair the current streamed row with every row of the
+ /// buffered key group, and mark the group as matched.
+ ///
+ /// Returns false when a full batch of pairs has accumulated (the scan
+ /// may or may not be complete): the caller must materialize
+ /// (`freeze_and_emit`) and call again, which resumes the scan where it
+ /// paused. Returns true when the group scan is complete and there is
+ /// room for more pairs.
+ fn pair_streamed_row_with_group(&mut self) -> bool {
+ while !self.buffered_data.scanning_finished()
+ && self.num_unfrozen_pairs() < self.batch_size
+ {
+ let scanning_idx = self.buffered_data.scanning_idx();
+ self.streamed_batch.append_output_pair(
+ Some(self.buffered_data.scanning_batch_idx),
+ Some(scanning_idx),
+ self.batch_size,
+ );
+ self.buffered_data.scanning_advance();
+ }
+ if self.num_unfrozen_pairs() >= self.batch_size {
+ return false;
+ }
+
+ self.buffered_group_matched = true;
+ self.buffered_data.scanning_reset();
+ true
+ }
+
+ /// `Less` (outer joins): no buffered row matches the current streamed
+ /// row — emit it joined to NULLs. Inner joins emit nothing.
+ fn null_join_streamed_row(&mut self) {
+ if matches!(
+ self.join_type,
+ JoinType::Left | JoinType::Right | JoinType::Full
+ ) {
+ let scanning_batch_idx = if self.buffered_data.scanning_finished() {
+ None
+ } else {
+ Some(self.buffered_data.scanning_batch_idx)
+ };
+ self.streamed_batch.append_output_pair(
+ scanning_batch_idx,
+ None,
+ self.batch_size,
+ );
+ }
+ self.buffered_data.scanning_reset();
+ }
+
+ /// `Greater` (FULL join): the buffered group can never match a streamed
+ /// row anymore — if nothing matched it, mark all its rows for
+ /// null-joined output (produced when the group's batches are dequeued).
+ fn null_join_buffered_group(&mut self) {
+ if self.join_type == JoinType::Full && !self.buffered_group_matched {
+ while !self.buffered_data.scanning_finished() {
+ let scanning_idx = self.buffered_data.scanning_idx();
+ self.buffered_data
+ .scanning_batch_mut()
+ .null_joined
+ .push(scanning_idx);
+ self.buffered_data.scanning_advance();
+ }
+ }
+ self.buffered_data.scanning_reset();
+ }
+
+ /// Start (resume) the `join_time` clock.
+ fn start_join_time(&mut self) {
+ debug_assert!(self.join_time_start.is_none(), "join_time already running");
+ self.join_time_start = Some(Instant::now());
+ }
+
+ /// Stop (pause) the `join_time` clock, accumulating the elapsed span.
+ ///
+ /// Called around awaits whose duration is not the join's own work: the
+ /// child input streams' `next()` and `emitter.emit()` (where the
+ /// consumer processes the batch). The join's own spill write and
+ /// read-back are NOT excluded — that time is join work.
+ fn stop_join_time(&mut self) {
+ if let Some(start) = self.join_time_start.take() {
+ self.join_time.add_elapsed(start);
+ }
+ }
+
+ /// Number of rows currently waiting in the deferred-filtering pipeline.
+ ///
+ /// Typically bounded to ~2*batch_size: one batch_size worth from
+ /// freeze_dequeuing_buffered() (when an input batch is fully consumed),
+ /// plus up to batch_size pairs accumulating toward the next freeze. A
+ /// single streamed row matching a very large key group can exceed that
+ /// (its pairs freeze into the pipeline before the gate runs again — same
+ /// as the pre-generator design). This does not reintroduce the unbounded
+ /// buffering fixed by PR #20482; `on_children_exhausted` flushes the
+ /// remainder.
+ fn deferred_rows_accumulated(&self) -> usize {
+ self.num_unfrozen_pairs()
+ + self.joined_record_batches.filter_metadata.filter_mask.len()
+ }
+
+ /// Run the deferred-filtering pipeline over everything accumulated so
+ /// far and emit its completed output, if any. Clears the accumulation
+ /// it processed.
+ ///
+ /// The caller gates this on `deferred_rows_accumulated() >= batch_size`:
+ /// running the pipeline per row instead (concat + correct_mask +
+ /// filter_by_type) would dominate runtime for unique keys.
+ async fn emit_deferred_output(
+ &mut self,
+ emitter: &mut TryEmitter<RecordBatch, DataFusionError>,
+ ) -> Result<()> {
+ // Ensure required spilled batches are restored to memory before
+ // processing, as this path invokes freeze_all().
+ self.restore_spilled_batches_for_freeze().await?;
+ if let Some(batch) = self.process_filtered_batches()? {
+ // While the emitted batch is in the consumer's hands the join
+ // isn't doing any work.
+ self.stop_join_time();
+ emitter.emit(batch).await;
+ self.start_join_time();
+ }
+ Ok(())
+ }
+
+ /// Restore every spilled buffered batch that the next freeze needs.
+ async fn restore_spilled_batches_for_freeze(&mut self) -> Result<()> {
+ let needed = self.get_required_batch_indices(self.buffered_data.batches.len());
+ self.restore_spilled_batches(&needed).await
+ }
+
+ /// Emit all completed joined batches to the stream consumer.
+ async fn emit_completed_joined_batches(
+ &mut self,
+ emitter: &mut TryEmitter<RecordBatch, DataFusionError>,
+ ) {
+ while let Some(record_batch) = self
+ .joined_record_batches
+ .joined_batches
+ .next_completed_batch()
+ {
+ // While the emitted batch is in the consumer's hands the join
+ // isn't doing any work.
+ self.stop_join_time();
+ emitter.emit(record_batch).await;
+ self.start_join_time();
+ }
+ }
+
+ /// Flush everything that remains once both inputs are exhausted.
+ async fn on_children_exhausted(
+ &mut self,
+ emitter: &mut TryEmitter<RecordBatch, DataFusionError>,
+ ) -> Result<()> {
+ // Freeze the remaining pairs, restoring any spilled batches needed.
+ self.restore_spilled_batches_for_freeze().await?;
+ self.freeze_all()?;
+
+ // Verify metadata alignment before final output
+ self.joined_record_batches
+ .filter_metadata
+ .debug_assert_metadata_aligned();
+
+ if self.deferred_filtering {
+ // Filtered joins must concat and filter ALL remaining data at once
+ if !self.joined_record_batches.joined_batches.is_empty() {
+ let record_batch = self.filter_joined_batch()?;
+ self.stop_join_time();
+ emitter.emit(record_batch).await;
+ self.start_join_time();
+ }
+ } else if !self.joined_record_batches.joined_batches.is_empty() {
+ // For non-filtered joins, finish buffered data first, then emit
+ // every completed batch.
+ self.joined_record_batches
+ .joined_batches
+ .finish_buffered_batch()?;
+ self.emit_completed_joined_batches(emitter).await;
+ }
+
+ // Drain the double-buffering coalescer used by filtered joins.
+ if !self.output.is_empty() {
+ self.output.finish_buffered_batch()?;
+ while let Some(record_batch) = self.output.next_completed_batch() {
+ self.stop_join_time();
+ emitter.emit(record_batch).await;
+ self.start_join_time();
+ }
+ }
+
+ Ok(())
}
/// Build a comparator for streamed vs buffered head batch keys.
@@ -927,9 +918,9 @@ impl MaterializingSortMergeJoinStream {
/// Process accumulated batches for filtered joins
///
- /// Freezes unfrozen pairs, applies deferred filtering, and outputs if ready.
- /// Returns Poll::Ready with a batch if one is available, otherwise Poll::Pending.
- fn process_filtered_batches(&mut self) -> Poll<Option<Result<RecordBatch>>> {
+ /// Freezes unfrozen pairs, applies deferred filtering, and returns a
+ /// completed output batch if one is ready.
+ fn process_filtered_batches(&mut self) -> Result<Option<RecordBatch>> {
self.freeze_all()?;
self.joined_record_batches
@@ -947,12 +938,11 @@ impl MaterializingSortMergeJoinStream {
.output
.next_completed_batch()
.expect("Failed to get output batch");
- (&record_batch).record_output(&self.join_metrics.baseline_metrics());
- return Poll::Ready(Some(Ok(record_batch)));
+ return Ok(Some(record_batch));
}
}
- Poll::Pending
+ Ok(None)
}
/// Identifies which buffered batches are needed for the upcoming freeze operation
@@ -981,11 +971,10 @@ impl MaterializingSortMergeJoinStream {
/// Asynchronously reads spilled batches back into memory.
/// Only processes the required indices to avoid OOMs.
- fn poll_spilled_batches(
+ async fn restore_spilled_batches(
&mut self,
- cx: &mut Context<'_>,
required_indices: &[usize],
- ) -> Poll<Result<()>> {
+ ) -> Result<()> {
for &idx in required_indices {
// Guard against indices that might be out of bounds if the queue was cleared
if idx >= self.buffered_data.batches.len() {
@@ -995,15 +984,12 @@ impl MaterializingSortMergeJoinStream {
let bb = &mut self.buffered_data.batches[idx];
if let BufferedBatchState::Spilled(spill_file) = &bb.batch {
- if self.spill_stream.is_none() {
- let stream = self
- .spill_manager
- .read_spill_as_stream(Arc::clone(spill_file), None)?;
- self.spill_stream = Some(stream);
- }
+ let mut spill_stream = self
+ .spill_manager
+ .read_spill_as_stream(Arc::clone(spill_file), None)?;
- match ready!(self.spill_stream.as_mut().unwrap().poll_next_unpin(cx)) {
- Some(Ok(batch)) => {
+ match spill_stream.next().await.transpose()? {
+ Some(batch) => {
// Transition the batch back to InMemory
bb.batch = BufferedBatchState::InMemory(batch);
self.spilled_batch_count -= 1;
@@ -1016,78 +1002,65 @@ impl MaterializingSortMergeJoinStream {
self.join_metrics
.peak_mem_used()
.set_max(self.reservation.size());
-
- self.spill_stream = None;
- }
- Some(Err(e)) => {
- self.spill_stream = None;
- return Poll::Ready(Err(e));
}
None => {
- self.spill_stream = None;
- return Poll::Ready(internal_err!("Spill file was empty"));
+ return internal_err!("Spill file was empty");
}
}
}
}
- Poll::Ready(Ok(()))
+
+ Ok(())
}
- /// Poll next streamed row
- fn poll_streamed_row(&mut self, cx: &mut Context) -> Poll<Option<Result<()>>> {
+ /// Sync fast path of advancing the streamed cursor: move to the next row
+ /// of the current batch. Returns false at the batch boundary, where the
+ /// caller must load the next batch via
+ /// [`Self::load_next_streamed_batch`].
+ fn try_advance_streamed_row(&mut self) -> bool {
+ if self.streamed_batch.idx + 1 < self.streamed_batch.batch.num_rows() {
+ self.streamed_batch.idx += 1;
+ return true;
+ }
+ false
+ }
+
+ /// Load the next streamed batch (freezing the finished one) and point
+ /// the streamed cursor at its first row. Sets `streamed_exhausted` when
+ /// the streamed input has no more rows.
+ async fn load_next_streamed_batch(&mut self) -> Result<()> {
loop {
- match &self.streamed_state {
- StreamedState::Init => {
- if self.streamed_batch.idx + 1 < self.streamed_batch.batch.num_rows()
- {
- self.streamed_batch.idx += 1;
- self.streamed_state = StreamedState::Ready;
- return Poll::Ready(Some(Ok(())));
- } else {
- self.streamed_state = StreamedState::Polling;
- }
+ // Loading a new streamed batch freezes the current one, which
+ // materializes buffered columns — restore any spilled buffered
+ // batches it needs first.
+ self.restore_spilled_batches_for_freeze().await?;
+
+ // The child's execution time is its own, not join_time.
+ self.stop_join_time();
+ let item = self.streamed.next().await.transpose();
+ self.start_join_time();
+ match item? {
+ None => {
+ // Release the streamed input pipeline's resources.
+ let streamed_schema = self.streamed.schema();
+ self.streamed =
+ Box::pin(EmptyRecordBatchStream::new(streamed_schema));
+ self.streamed_exhausted = true;
+ return Ok(());
}
- StreamedState::Polling => {
- let needed =
- self.get_required_batch_indices(self.buffered_data.batches.len());
- if let Err(e) = ready!(self.poll_spilled_batches(cx, &needed)) {
- return Poll::Ready(Some(Err(e)));
- }
-
- match self.streamed.poll_next_unpin(cx)? {
- Poll::Pending => {
- return Poll::Pending;
- }
- Poll::Ready(None) => {
- // Release the streamed input pipeline's resources.
- let streamed_schema = self.streamed.schema();
- self.streamed =
- Box::pin(EmptyRecordBatchStream::new(streamed_schema));
- self.streamed_state = StreamedState::Exhausted;
- }
- Poll::Ready(Some(batch)) => {
- if batch.num_rows() > 0 {
- self.freeze_streamed()?;
- self.join_metrics.input_batches().add(1);
- self.join_metrics.input_rows().add(batch.num_rows());
- self.streamed_batch =
- StreamedBatch::new(batch, &self.on_streamed);
- self.rebuild_streamed_buffered_cmp()?;
- // Every incoming streaming batch should have its unique id
- // Check `JoinedRecordBatches.self.streamed_batch_counter` documentation
- self.streamed_batch_counter
- .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
- self.streamed_state = StreamedState::Ready;
- }
- }
+ Some(batch) => {
+ if batch.num_rows() > 0 {
+ self.freeze_streamed()?;
+ self.join_metrics.input_batches().add(1);
+ self.join_metrics.input_rows().add(batch.num_rows());
+ self.streamed_batch =
+ StreamedBatch::new(batch, &self.on_streamed);
+ self.rebuild_streamed_buffered_cmp()?;
+ // Every incoming streamed batch gets a unique id.
+ self.streamed_batch_counter += 1;
+ return Ok(());
}
}
- StreamedState::Ready => {
- return Poll::Ready(Some(Ok(())));
- }
- StreamedState::Exhausted => {
- return Poll::Ready(None);
- }
}
}
}
@@ -1150,146 +1123,193 @@ impl MaterializingSortMergeJoinStream {
Ok(())
}
- /// Poll next buffered batches
- fn poll_buffered_batches(&mut self, cx: &mut Context) -> Poll<Option<Result<()>>> {
- loop {
- match &self.buffered_state {
- BufferedState::Init => {
- // pop previous buffered batches
- let mut head_changed = false;
- while !self.buffered_data.batches.is_empty() {
- let head_batch = self.buffered_data.head_batch();
- // If the head batch is fully processed, dequeue it and produce output of it.
- if head_batch.range.end == head_batch.num_rows {
- // load the spilled head batch before dequeuing
- let needed = self.get_required_batch_indices(1);
- if let Err(e) = ready!(self.poll_spilled_batches(cx, &needed))
- {
- return Poll::Ready(Some(Err(e)));
- }
+ /// Sync fast path of [`Self::advance_buffered_group`]: when the next
+ /// group starts in the single remaining buffered batch and provably ends
+ /// within it (the common case — a group only reaches a batch boundary
+ /// once per batch), advance entirely synchronously. Returns false —
+ /// leaving all state unchanged — when the async path must run instead.
+ fn try_advance_buffered_group(&mut self) -> Result<bool> {
+ if self.buffered_data.batches.len() != 1 {
+ return Ok(false);
+ }
+ let head_batch = self.buffered_data.head_batch();
+ if head_batch.range.end == head_batch.num_rows {
+ // Fully consumed — needs dequeuing (and loading the next batch).
+ return Ok(false);
+ }
- self.freeze_dequeuing_buffered()?;
- if let Some(mut buffered_batch) =
- self.buffered_data.batches.pop_front()
- {
- self.produce_buffered_not_matched(&mut buffered_batch)?;
- self.free_reservation(&buffered_batch);
- if matches!(
- buffered_batch.batch,
- BufferedBatchState::Spilled(_)
- ) {
- self.spilled_batch_count -= 1;
- }
- head_changed = true;
- }
- } else {
- // If the head batch is not fully processed, break the loop.
- // Streamed batch will be joined with the head batch in the next step.
- break;
- }
- }
- if head_changed {
+ if self.buffered_equality_cmp.is_none() {
+ self.rebuild_buffered_equality_cmp()?;
+ }
+ let cmp = self.buffered_equality_cmp.as_ref().unwrap();
+
+ // Scan the next group's extent before committing any state, so a
+ // bail-out (the group may span into the next batch) leaves
+ // everything untouched for the async path.
+ let batch = self.buffered_data.head_batch();
+ let group_start = batch.range.end;
+ let mut group_end = group_start + 1;
+ while group_end < batch.num_rows && cmp.is_equal(group_start, group_end) {
+ group_end += 1;
+ }
+ if group_end == batch.num_rows {
+ return Ok(false);
+ }
+
+ let batch = self.buffered_data.tail_batch_mut();
+ batch.range.start = group_start;
+ batch.range.end = group_end;
+ self.buffered_group_matched = false;
+ Ok(true)
+ }
+
+ /// Advance the buffered side to the next key group: dequeue batches
+ /// fully consumed by the previous group, then collect all contiguous
+ /// rows sharing the next join key (the group may span multiple buffered
+ /// batches). Sets `buffered_exhausted` when no group remains.
+ async fn advance_buffered_group(&mut self) -> Result<()> {
+ self.buffered_group_matched = false;
+ self.dequeue_consumed_buffered_batches().await?;
+
+ if self.buffered_data.batches.is_empty() {
+ // Load the batch holding the first row of the next group.
+ if !self.load_next_buffered_batch().await? {
+ self.buffered_exhausted = true;
+ return Ok(());
+ }
+ } else {
+ // Seed the next group at the first unconsumed row of the
+ // remaining batch.
+ let tail_batch = self.buffered_data.tail_batch_mut();
+ tail_batch.range.start = tail_batch.range.end;
+ tail_batch.range.end += 1;
+ }
+
+ self.extend_buffered_group().await
+ }
+
+ /// Dequeue buffered batches fully consumed by the previous group,
+ /// producing their pending output (e.g. Full-join null-joined rows).
+ async fn dequeue_consumed_buffered_batches(&mut self) -> Result<()> {
+ let mut head_changed = false;
+ while !self.buffered_data.batches.is_empty() {
+ let head_batch = self.buffered_data.head_batch();
+ if head_batch.range.end != head_batch.num_rows {
+ // The next group starts within the head batch: streamed rows
+ // will be joined with the head batch in the next step.
+ break;
+ }
+ // load the spilled head batch before dequeuing
+ let needed = self.get_required_batch_indices(1);
+ self.restore_spilled_batches(&needed).await?;
+
+ self.freeze_dequeuing_buffered()?;
+ if let Some(mut buffered_batch) = self.buffered_data.batches.pop_front() {
+ self.produce_buffered_not_matched(&mut buffered_batch)?;
+ self.free_reservation(&buffered_batch);
+ if matches!(buffered_batch.batch, BufferedBatchState::Spilled(_)) {
+ self.spilled_batch_count -= 1;
+ }
+ head_changed = true;
+ }
+ }
+ if head_changed {
+ self.streamed_buffered_cmp = None;
+ self.buffered_equality_cmp = None;
+ }
+ Ok(())
+ }
+
+ /// Load the next non-empty buffered batch and seed a new group with its
+ /// first row. Returns false when the buffered input is exhausted.
+ async fn load_next_buffered_batch(&mut self) -> Result<bool> {
+ loop {
+ // The child's execution time is its own, not join_time.
+ self.stop_join_time();
+ let item = self.buffered.next().await.transpose();
+ self.start_join_time();
+ match item? {
+ None => {
+ // Release the buffered input pipeline's resources.
+ let buffered_schema = self.buffered.schema();
+ self.buffered =
+ Box::pin(EmptyRecordBatchStream::new(buffered_schema));
+ return Ok(false);
+ }
+ Some(batch) => {
+ self.join_metrics.input_batches().add(1);
+ self.join_metrics.input_rows().add(batch.num_rows());
+
+ if batch.num_rows() > 0 {
+ let buffered_batch =
+ BufferedBatch::new(batch, 0..1, &self.on_buffered);
+ self.allocate_reservation(buffered_batch)?;
self.streamed_buffered_cmp = None;
- self.buffered_equality_cmp = None;
+ return Ok(true);
}
- if self.buffered_data.batches.is_empty() {
- self.buffered_state = BufferedState::PollingFirst;
+ }
+ }
+ }
+ }
+
+ /// Extend the current group with every following row that shares its
+ /// key, loading more buffered batches as needed.
+ async fn extend_buffered_group(&mut self) -> Result<()> {
+ loop {
+ if self.buffered_data.tail_batch().range.end
+ < self.buffered_data.tail_batch().num_rows
+ {
+ if self.buffered_equality_cmp.is_none() {
+ self.rebuild_buffered_equality_cmp()?;
+ }
+ while self.buffered_data.tail_batch().range.end
+ < self.buffered_data.tail_batch().num_rows
+ {
+ if self.buffered_equality_cmp.as_ref().unwrap().is_equal(
+ self.buffered_data.head_batch().range.start,
+ self.buffered_data.tail_batch().range.end,
+ ) {
+ self.buffered_data.tail_batch_mut().range.end += 1;
} else {
- let tail_batch = self.buffered_data.tail_batch_mut();
- tail_batch.range.start = tail_batch.range.end;
- tail_batch.range.end += 1;
- self.buffered_state = BufferedState::PollingRest;
+ // Group complete within the current batch.
+ return Ok(());
}
}
- BufferedState::PollingFirst => match self.buffered.poll_next_unpin(cx)? {
- Poll::Pending => {
- return Poll::Pending;
- }
- Poll::Ready(None) => {
+ } else {
+ // The child's execution time is its own, not join_time.
+ self.stop_join_time();
+ let item = self.buffered.next().await.transpose();
+ self.start_join_time();
+ match item? {
+ None => {
+ // Group complete; the input is done but the group is
+ // still valid — `buffered_exhausted` is only set once
+ // it has been fully consumed and dequeued.
// Release the buffered input pipeline's resources.
let buffered_schema = self.buffered.schema();
self.buffered =
Box::pin(EmptyRecordBatchStream::new(buffered_schema));
- self.buffered_state = BufferedState::Exhausted;
- return Poll::Ready(None);
+ return Ok(());
}
- Poll::Ready(Some(batch)) => {
+ Some(batch) => {
+ // Polling batches coming concurrently as multiple partitions
self.join_metrics.input_batches().add(1);
self.join_metrics.input_rows().add(batch.num_rows());
-
if batch.num_rows() > 0 {
let buffered_batch =
- BufferedBatch::new(batch, 0..1, &self.on_buffered);
-
+ BufferedBatch::new(batch, 0..0, &self.on_buffered);
self.allocate_reservation(buffered_batch)?;
- self.streamed_buffered_cmp = None;
- self.buffered_state = BufferedState::PollingRest;
- }
- }
- },
- BufferedState::PollingRest => {
- if self.buffered_data.tail_batch().range.end
- < self.buffered_data.tail_batch().num_rows
- {
- if self.buffered_equality_cmp.is_none() {
- self.rebuild_buffered_equality_cmp()?;
- }
- while self.buffered_data.tail_batch().range.end
- < self.buffered_data.tail_batch().num_rows
- {
- if self.buffered_equality_cmp.as_ref().unwrap().is_equal(
- self.buffered_data.head_batch().range.start,
- self.buffered_data.tail_batch().range.end,
- ) {
- self.buffered_data.tail_batch_mut().range.end += 1;
- } else {
- self.buffered_state = BufferedState::Ready;
- return Poll::Ready(Some(Ok(())));
- }
- }
- } else {
- match self.buffered.poll_next_unpin(cx)? {
- Poll::Pending => {
- return Poll::Pending;
- }
- Poll::Ready(None) => {
- // Release the buffered input pipeline's resources.
- let buffered_schema = self.buffered.schema();
- self.buffered = Box::pin(EmptyRecordBatchStream::new(
- buffered_schema,
- ));
- self.buffered_state = BufferedState::Ready;
- }
- Poll::Ready(Some(batch)) => {
- // Polling batches coming concurrently as multiple partitions
- self.join_metrics.input_batches().add(1);
- self.join_metrics.input_rows().add(batch.num_rows());
- if batch.num_rows() > 0 {
- let buffered_batch = BufferedBatch::new(
- batch,
- 0..0,
- &self.on_buffered,
- );
- self.allocate_reservation(buffered_batch)?;
- self.buffered_equality_cmp = None;
- }
- }
+ self.buffered_equality_cmp = None;
}
}
}
- BufferedState::Ready => {
- return Poll::Ready(Some(Ok(())));
- }
- BufferedState::Exhausted => {
- return Poll::Ready(None);
- }
}
}
}
/// Get comparison result of streamed row and buffered batches
fn compare_streamed_buffered(&mut self) -> Result<Ordering> {
- if self.streamed_state == StreamedState::Exhausted {
+ if self.streamed_exhausted {
return Ok(Ordering::Greater);
}
if !self.buffered_data.has_buffered_rows() {
@@ -1305,81 +1325,23 @@ impl MaterializingSortMergeJoinStream {
))
}
- /// Produce join and fill output buffer until reaching target batch size
- /// or the join is finished
- fn join_partial(&mut self) -> Result<()> {
- // Whether to join streamed rows
- let mut join_streamed = false;
- // Whether to join buffered rows
- let mut join_buffered = false;
-
- // determine whether we need to join streamed/buffered rows
- match self.current_ordering {
- Ordering::Less => {
- if matches!(
- self.join_type,
- JoinType::Left | JoinType::Right | JoinType::Full
- ) {
- join_streamed = !self.streamed_joined;
- }
- }
- Ordering::Equal => {
- join_streamed = true;
- join_buffered = true;
- }
- Ordering::Greater => {
- if self.join_type == JoinType::Full {
- join_buffered = !self.buffered_joined;
- };
- }
- }
- if !join_streamed && !join_buffered {
- // no joined data
- self.buffered_data.scanning_finish();
- return Ok(());
- }
-
- if join_buffered {
- // joining streamed/nulls and buffered
- while !self.buffered_data.scanning_finished()
- && self.num_unfrozen_pairs() < self.batch_size
- {
- let scanning_idx = self.buffered_data.scanning_idx();
- if join_streamed {
- // Join streamed row and buffered row
- self.streamed_batch.append_output_pair(
- Some(self.buffered_data.scanning_batch_idx),
- Some(scanning_idx),
- self.batch_size,
- );
- } else {
- // Join nulls and buffered row for FULL join
- self.buffered_data
- .scanning_batch_mut()
- .null_joined
- .push(scanning_idx);
- }
- self.buffered_data.scanning_advance();
+ /// Materialize ("freeze") the accumulated pairs — restoring any spilled
+ /// batches they reference first — and emit completed output batches
+ /// (filtered joins emit through the deferred-filtering gate instead).
+ async fn freeze_and_emit(
+ &mut self,
+ emitter: &mut TryEmitter<RecordBatch, DataFusionError>,
+ ) -> Result<()> {
+ self.restore_spilled_batches_for_freeze().await?;
+ self.freeze_all()?;
- if self.buffered_data.scanning_finished() {
- self.streamed_joined = join_streamed;
- self.buffered_joined = true;
- }
- }
- } else {
- // joining streamed and nulls
- let scanning_batch_idx = if self.buffered_data.scanning_finished() {
- None
- } else {
- Some(self.buffered_data.scanning_batch_idx)
- };
- self.streamed_batch.append_output_pair(
- scanning_batch_idx,
- None,
- self.batch_size,
- );
- self.buffered_data.scanning_finish();
- self.streamed_joined = true;
+ if !self.deferred_filtering
+ && self
+ .joined_record_batches
+ .joined_batches
+ .has_completed_batch()
+ {
+ self.emit_completed_joined_batches(emitter).await;
}
Ok(())
}
@@ -1513,7 +1475,7 @@ impl MaterializingSortMergeJoinStream {
// but must flow through the same pipeline as matched rows to
// preserve output ordering. Use null metadata as a sentinel so
// get_corrected_filter_mask() passes them through unchanged.
- if needs_deferred_filtering(&self.filter, self.join_type) {
+ if self.deferred_filtering {
self.joined_record_batches
.push_batch_with_null_metadata(batch, self.join_type);
} else {
@@ -1616,12 +1578,12 @@ impl MaterializingSortMergeJoinStream {
filter_result_mask.clone()
};
- if needs_deferred_filtering(&self.filter, self.join_type) {
+ if self.deferred_filtering {
self.joined_record_batches.push_batch_with_filter_metadata(
output_batch,
&combined_left_indices,
&mask,
- self.streamed_batch_counter.load(Relaxed),
+ self.streamed_batch_counter,
self.join_type,
);
} else {
@@ -1968,9 +1930,9 @@ fn fetch_right_columns_from_batch_by_idxs(
pub(super) struct BufferedData {
/// Buffered batches with the same key
pub batches: VecDeque<BufferedBatch>,
- /// current scanning batch index used in join_partial()
+ /// current scanning batch index used by the group-scan phase
pub scanning_batch_idx: usize,
- /// current scanning offset used in join_partial()
+ /// current scanning offset used by the group-scan phase
pub scanning_offset: usize,
}
@@ -2023,11 +1985,6 @@ impl BufferedData {
pub fn scanning_finished(&self) -> bool {
self.scanning_batch_idx == self.batches.len()
}
-
- pub fn scanning_finish(&mut self) {
- self.scanning_batch_idx = self.batches.len();
- self.scanning_offset = 0;
- }
}
/// Get join array refs of given batch and join columns
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
index 313818cf6c..3dbb50eba0 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
@@ -4315,6 +4315,147 @@ async fn join_time_excludes_consumer_wait() -> Result<()> {
.await
}
+/// Three 2-row batches with unique matching keys, right-side column names.
+fn join_time_batches_right() -> Vec<RecordBatch> {
+ vec![
+ build_table_i32(
+ ("a2", &vec![0, 1]),
+ ("b2", &vec![1, 2]),
+ ("c2", &vec![7, 8]),
+ ),
+ build_table_i32(
+ ("a2", &vec![2, 3]),
+ ("b2", &vec![3, 4]),
+ ("c2", &vec![7, 8]),
+ ),
+ build_table_i32(
+ ("a2", &vec![4, 5]),
+ ("b2", &vec![5, 6]),
+ ("c2", &vec![7, 8]),
+ ),
+ ]
+}
+
+/// Build a no-filter Inner materializing join over the given input streams.
+/// The small batch size makes the output surface as multiple batches, so a
+/// slow consumer test sees multiple emits.
+fn materializing_join_time_test_join(
+ streamed: SendableRecordBatchStream,
+ buffered: SendableRecordBatchStream,
+) -> (SendableRecordBatchStream, ExecutionPlanMetricsSet) {
+ use crate::joins::sort_merge_join::materializing_stream::MaterializingSortMergeJoinStream;
+ use crate::joins::sort_merge_join::metrics::SortMergeJoinMetrics;
+
+ let metrics = ExecutionPlanMetricsSet::new();
+ let out_schema = Arc::new(Schema::new(
+ streamed
+ .schema()
+ .fields()
+ .iter()
+ .chain(buffered.schema().fields().iter())
+ .map(|f| f.as_ref().clone())
+ .collect::<Vec<_>>(),
+ ));
+ let (reservation, spill_manager, runtime_env) =
+ test_stream_resources(buffered.schema(), &metrics);
+ let stream = MaterializingSortMergeJoinStream::try_new(
+ out_schema,
+ vec![SortOptions::default()],
+ NullEquality::NullEqualsNothing,
+ streamed,
+ buffered,
+ vec![Arc::new(Column::new("b1", 1)) as _],
+ vec![Arc::new(Column::new("b2", 1)) as _],
+ None,
+ Inner,
+ 2,
+ SortMergeJoinMetrics::new(0, &metrics),
+ reservation,
+ spill_manager,
+ runtime_env,
+ )
+ .unwrap();
+ (stream, metrics)
+}
+
+/// join_time must not include time spent waiting for the streamed input.
+#[tokio::test]
+async fn materializing_join_time_excludes_streamed_input_wait() -> Result<()> {
+ check_join_time_excluded(|delay| async move {
+ let streamed = delayed_stream(join_time_batches(), delay);
+ let buffered = delayed_stream(join_time_batches_right(), Duration::ZERO);
+ let (stream, metrics) = materializing_join_time_test_join(streamed, buffered);
+
+ let start = Instant::now();
+ let batches = collect_stream(stream).await?;
+ let wall = start.elapsed();
+
+ let rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+ assert_eq!(rows, 6, "all rows should match");
+ assert!(
+ wall >= delay * 3,
+ "streamed delays should dominate wall time, got {wall:?}"
+ );
+ Ok((join_time_of(&metrics), wall))
+ })
+ .await
+}
+
+/// join_time must not include time spent waiting for the buffered input.
+#[tokio::test]
+async fn materializing_join_time_excludes_buffered_input_wait() -> Result<()> {
+ check_join_time_excluded(|delay| async move {
+ let streamed = delayed_stream(join_time_batches(), Duration::ZERO);
+ let buffered = delayed_stream(join_time_batches_right(), delay);
+ let (stream, metrics) = materializing_join_time_test_join(streamed, buffered);
+
+ let start = Instant::now();
+ let batches = collect_stream(stream).await?;
+ let wall = start.elapsed();
+
+ let rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+ assert_eq!(rows, 6, "all rows should match");
+ assert!(
+ wall >= delay * 3,
+ "buffered delays should dominate wall time, got {wall:?}"
+ );
+ Ok((join_time_of(&metrics), wall))
+ })
+ .await
+}
+
+/// join_time must not include time the consumer spends holding an emitted
+/// batch (the generator is suspended inside `emitter.emit` meanwhile).
+#[tokio::test]
+async fn materializing_join_time_excludes_consumer_wait() -> Result<()> {
+ check_join_time_excluded(|delay| async move {
+ let streamed = delayed_stream(join_time_batches(), Duration::ZERO);
+ let buffered = delayed_stream(join_time_batches_right(), Duration::ZERO);
+ let (mut stream, metrics) = materializing_join_time_test_join(streamed, buffered);
+
+ let start = Instant::now();
+ let mut output_batches = 0u32;
+ while let Some(batch) = stream.next().await {
+ batch?;
+ output_batches += 1;
+ // Simulate a slow consumer between emitted batches.
+ tokio::time::sleep(delay).await;
+ }
+ let wall = start.elapsed();
+
+ assert!(
+ output_batches >= 3,
+ "expected multiple emitted batches, got {output_batches}"
+ );
+ assert!(
+ wall >= delay * output_batches,
+ "consumer delays should dominate wall time, got {wall:?}"
+ );
+ Ok((join_time_of(&metrics), wall))
+ })
+ .await
+}
+
/// An inner key group spanning multiple inner batches must survive the inner
/// input returning Pending mid-way: inner rows delivered before the Pending
/// still take part in the filter evaluation.