Apache DataFusion SQL Query Engine
chore: refactor `MaterializingSortMergeJoinStream` into generators and simplify code to be textbook like as possible (#23976)
Raz Luvaton39d50641424d
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rsindex 3b597323b2..1abcd9d6c7 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs@@ -81,8 +81,7 @@ use datafusion_physical_expr_common::sort_expr::{LexOrdering, OrderingRequiremen/// on the output batch size of the execution plan. There is no spilling support for streamed input./// The comparisons are performed from values of join keys in streamed input with the values of/// join keys in buffered input. One row in streamed record batch could be matched with multiple rows in-/// buffered input batches. The streamed input is managed through the states in `StreamedState`-/// and streamed input batches are represented by `StreamedBatch`.+/// buffered input batches. Streamed input batches are represented by `StreamedBatch`.////// Buffered input is buffered for all record batches having the same value of join key./// If the memory limit increases beyond the specified value and spilling is enabled,@@ -92,8 +91,7 @@ use datafusion_physical_expr_common::sort_expr::{LexOrdering, OrderingRequiremen/// memory/disk depends on the number of rows of buffered input having the same value/// of join key as that of streamed input rows currently present in memory. Due to pre-sorted inputs,/// the algorithm understands when it is not needed anymore, and releases the buffered batches-/// from memory/disk. The buffered input is managed through the states in `BufferedState`-/// and buffered input batches are represented by `BufferedBatch`.+/// from memory/disk. Buffered input batches are represented by `BufferedBatch`.////// Depending on the type of join, left or right input may be selected as streamed or buffered/// respectively. For example, in a left-outer join, the left execution plan will be selected as@@ -546,7 +544,7 @@ impl ExecutionPlan for SortMergeJoinExec {context.runtime_env(),)} else {- Ok(Box::pin(MaterializingSortMergeJoinStream::try_new(+ MaterializingSortMergeJoinStream::try_new(Arc::clone(&self.schema),self.sort_options.clone(),self.null_equality,@@ -561,7 +559,7 @@ impl ExecutionPlan for SortMergeJoinExec {reservation,spill_manager,context.runtime_env(),- )?))+ )}}diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rsindex 51cf38b9ab..3baa0c4a3e 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs@@ -17,20 +17,17 @@//! Sort-Merge Join execution//!-//! This module implements the runtime state machine for the Sort-Merge Join-//! operator. It drives two sorted input streams (the *streamed* side and the-//! *buffered* side), compares join keys, and produces joined `RecordBatch`es.+//! This module implements the Sort-Merge Join operator as an async+//! generator running a merge scan: it drives two sorted input streams (the+//! *streamed* side and the *buffered* side), compares join keys, and+//! produces joined `RecordBatch`es.use std::cmp::Ordering;use std::collections::{HashMap, VecDeque};use std::fmt::Debug;use std::mem::size_of;use std::ops::Range;-use std::pin::Pin;use std::sync::Arc;-use std::sync::atomic::AtomicUsize;-use std::sync::atomic::Ordering::Relaxed;-use std::task::{Context, Poll};use crate::joins::sort_merge_join::filter::{FilterMetadata, filter_record_batch_by_join_type, get_corrected_filter_mask,@@ -38,10 +35,10 @@ use crate::joins::sort_merge_join::filter::{};use crate::joins::sort_merge_join::metrics::SortMergeJoinMetrics;use crate::joins::utils::{JoinFilter, JoinKeyComparator};-use crate::metrics::RecordOutput;+use crate::metrics::Time;use crate::spill::spill_manager::SpillManager;-use crate::stream::EmptyRecordBatchStream;-use crate::{PhysicalExpr, RecordBatchStream, SendableRecordBatchStream};+use crate::stream::{EmptyRecordBatchStream, ObservedStream, RecordBatchStreamAdapter};+use crate::{PhysicalExpr, SendableRecordBatchStream};use arrow::array::{types::UInt64Type, *};use arrow::compute::{@@ -50,56 +47,16 @@ use arrow::compute::{};use arrow::datatypes::SchemaRef;use datafusion_common::cast::as_uint64_array;-use datafusion_common::{JoinType, NullEquality, Result, exec_err, internal_err};-use datafusion_execution::SpillFile;+use datafusion_common::instant::Instant;+use datafusion_common::{+ DataFusionError, JoinType, NullEquality, Result, exec_err, internal_err,+};use datafusion_execution::memory_pool::MemoryReservation;use datafusion_execution::runtime_env::RuntimeEnv;+use datafusion_execution::{SpillFile, TryEmitter, async_try_stream};use datafusion_physical_expr_common::physical_expr::PhysicalExprRef;-use futures::{Stream, StreamExt, ready};--/// State of SMJ stream-#[derive(Debug, PartialEq, Eq)]-pub(super) enum SortMergeJoinState {- /// Init joining with a new streamed row or a new buffered batches- Init,- /// Polling one streamed row or one buffered batch, or both- Polling,- /// Joining polled data and making output- JoinOutput,- /// Emit ready data if have any and then go back to [`Self::Init`] state- EmitReadyThenInit,- /// No more output- Exhausted,-}--/// State of streamed data stream-#[derive(Debug, PartialEq, Eq)]-pub(super) enum StreamedState {- /// Init polling- Init,- /// Polling one streamed row- Polling,- /// Ready to produce one streamed row- Ready,- /// No more streamed row- Exhausted,-}--/// State of buffered data stream-#[derive(Debug, PartialEq, Eq)]-pub(super) enum BufferedState {- /// Init polling- Init,- /// Polling first row in the next batch- PollingFirst,- /// Polling rest rows in the next batch- PollingRest,- /// Ready to produce one batch- Ready,- /// No more buffered batches- Exhausted,-}+use futures::StreamExt;/// Represents a chunk of joined data from streamed and buffered sidepub(super) struct StreamedJoinedChunk {@@ -335,6 +292,9 @@ pub(super) struct MaterializingSortMergeJoinStream {pub filter: Option<JoinFilter>,/// How the join is performedpub join_type: JoinType,+ /// Cached `needs_deferred_filtering(filter, join_type)` — both inputs+ /// are fixed at construction time.+ pub deferred_filtering: bool,/// Target output batch sizepub batch_size: usize,@@ -348,10 +308,8 @@ pub(super) struct MaterializingSortMergeJoinStream {pub streamed: SendableRecordBatchStream,/// Current processing record batch of streamedpub streamed_batch: StreamedBatch,- /// (used in outer join) Is current streamed row joined at least once?- pub streamed_joined: bool,- /// State of streamed- pub streamed_state: StreamedState,+ /// True once the streamed input has no more rows+ pub streamed_exhausted: bool,/// Join key columns of streamedpub on_streamed: Vec<PhysicalExprRef>,@@ -365,10 +323,11 @@ pub(super) struct MaterializingSortMergeJoinStream {pub buffered: SendableRecordBatchStream,/// Current buffered datapub buffered_data: BufferedData,- /// (used in outer join) Is current buffered batches joined at least once?- pub buffered_joined: bool,- /// State of buffered- pub buffered_state: BufferedState,+ /// Has any streamed row matched the current buffered key group?+ /// (FULL join: an unmatched group is emitted null-joined when passed.)+ pub buffered_group_matched: bool,+ /// True once the buffered input has no more rows and no group remains+ pub buffered_exhausted: bool,/// Join key columns of bufferedpub on_buffered: Vec<PhysicalExprRef>,@@ -377,23 +336,26 @@ pub(super) struct MaterializingSortMergeJoinStream {// These fields track the execution state of merge join and are updated// during the execution.// ========================================================================- /// Current state of the stream- pub state: SortMergeJoinState,/// Staging output array builderspub joined_record_batches: JoinedRecordBatches,/// Output buffer. Currently used by filtering as it requires double buffering- /// to avoid small/empty batches. Non-filtered join outputs directly from `staging_output_record_batches.batches`+ /// to avoid small/empty batches. Non-filtered joins output directly from+ /// `joined_record_batches.joined_batches`pub output: BatchCoalescer,- /// The comparison result of current streamed row and buffered batches- pub current_ordering: Ordering,/// Manages the process of spilling and reading back intermediate datapub spill_manager: SpillManager,- /// Tracks the active stream when loading spilled buffered batches back in memory- pub spill_stream: Option<SendableRecordBatchStream>,/// Tracks the number of batches currently spilledpub spilled_batch_count: usize,+ /// Time spent doing the join's own work (including spill write and+ /// read-back). The clock is stopped while awaiting the child inputs or+ /// the consumer taking an emitted batch — see [`Self::stop_join_time`].+ pub join_time: Time,+ /// Start of the currently running `join_time` span; `None` while the+ /// clock is stopped.+ pub join_time_start: Option<Instant>,+// ========================================================================// CACHED COMPARATORS:// Pre-built comparators to avoid per-row type dispatch in hot loops.@@ -413,8 +375,9 @@ pub(super) struct MaterializingSortMergeJoinStream {pub reservation: MemoryReservation,/// Runtime envpub runtime_env: Arc<RuntimeEnv>,- /// A unique number for each batch- pub streamed_batch_counter: AtomicUsize,+ /// A unique id per streamed batch, tagging deferred-filter metadata so+ /// `get_corrected_filter_mask` can group output rows by input batch.+ pub streamed_batch_counter: usize,}/// Staging area for joined data before output@@ -560,264 +523,6 @@ impl JoinedRecordBatches {self.debug_assert_empty_consistency();}}-impl RecordBatchStream for MaterializingSortMergeJoinStream {- fn schema(&self) -> SchemaRef {- Arc::clone(&self.schema)- }-}--impl Stream for MaterializingSortMergeJoinStream {- type Item = Result<RecordBatch>;-- fn poll_next(- mut self: Pin<&mut Self>,- cx: &mut Context<'_>,- ) -> Poll<Option<Self::Item>> {- let join_time = self.join_metrics.join_time().clone();- let _timer = join_time.timer();- loop {- match &self.state {- SortMergeJoinState::Init => {- let streamed_exhausted =- self.streamed_state == StreamedState::Exhausted;- let buffered_exhausted =- self.buffered_state == BufferedState::Exhausted;- self.state = if streamed_exhausted && buffered_exhausted {- SortMergeJoinState::Exhausted- } else {- match self.current_ordering {- Ordering::Less | Ordering::Equal => {- if !streamed_exhausted {- // Batch deferred filtering: process_filtered_batches()- // only when >= batch_size rows have accumulated.- // Without this gate, unique keys cause per-row pipeline- // execution (concat + correct_mask + filter_by_type),- // which dominates runtime.- //- // Accumulated rows are bounded to ~2*batch_size:- // one batch_size worth from freeze_dequeuing_buffered()- // (when an input batch is fully consumed), plus up to- // batch_size pairs accumulating toward the next freeze.- // This does not reintroduce the unbounded buffering- // fixed by PR #20482. Exhausted state flushes remainder.- if needs_deferred_filtering(- &self.filter,- self.join_type,- ) {- let accumulated = self.num_unfrozen_pairs()- + self- .joined_record_batches- .filter_metadata- .filter_mask- .len();- if accumulated >= self.batch_size {- // Ensure required spilled batches are restored to memory- // before processing, as this path invokes freeze_all().- let needed = self.get_required_batch_indices(- self.buffered_data.batches.len(),- );- if let Err(e) = ready!(- self.poll_spilled_batches(cx, &needed)- ) {- return Poll::Ready(Some(Err(e)));- }- match self.process_filtered_batches()? {- Poll::Ready(Some(batch)) => {- return Poll::Ready(Some(Ok(batch)));- }- Poll::Ready(None) | Poll::Pending => {}- }- }- }-- self.streamed_joined = false;- self.streamed_state = StreamedState::Init;- }- }- Ordering::Greater => {- if !buffered_exhausted {- self.buffered_joined = false;- self.buffered_state = BufferedState::Init;- }- }- }- SortMergeJoinState::Polling- };- }- SortMergeJoinState::Polling => {- if ![StreamedState::Exhausted, StreamedState::Ready]- .contains(&self.streamed_state)- {- match self.poll_streamed_row(cx)? {- Poll::Ready(_) => {}- Poll::Pending => return Poll::Pending,- }- }-- if ![BufferedState::Exhausted, BufferedState::Ready]- .contains(&self.buffered_state)- {- match self.poll_buffered_batches(cx)? {- Poll::Ready(_) => {}- Poll::Pending => return Poll::Pending,- }- }- let streamed_exhausted =- self.streamed_state == StreamedState::Exhausted;- let buffered_exhausted =- self.buffered_state == BufferedState::Exhausted;- if streamed_exhausted && buffered_exhausted {- self.state = SortMergeJoinState::Exhausted;- continue;- }- self.current_ordering = self.compare_streamed_buffered()?;- self.state = SortMergeJoinState::JoinOutput;- }- SortMergeJoinState::EmitReadyThenInit => {- // If have data to emit, emit it and if no more, change to next-- // Verify metadata alignment before checking if we have batches to output- self.joined_record_batches- .filter_metadata- .debug_assert_metadata_aligned();-- // For filtered joins, skip output and let Init state handle it- if needs_deferred_filtering(&self.filter, self.join_type) {- self.state = SortMergeJoinState::Init;- continue;- }-- // For non-filtered joins, only output if we have a completed batch- // (opportunistic output when target batch size is reached)- if self- .joined_record_batches- .joined_batches- .has_completed_batch()- {- let record_batch = self- .joined_record_batches- .joined_batches- .next_completed_batch()- .expect("has_completed_batch was true");- (&record_batch)- .record_output(&self.join_metrics.baseline_metrics());- return Poll::Ready(Some(Ok(record_batch)));- }- self.state = SortMergeJoinState::Init;- }- SortMergeJoinState::JoinOutput => {- // If the batch size limit is reached, restore required spilled batches to memory and freeze.- // Guarding at the top of the loop safely handles re-entry from Poll::Pending.- if self.num_unfrozen_pairs() >= self.batch_size {- let needed = self- .get_required_batch_indices(self.buffered_data.batches.len());- ready!(self.poll_spilled_batches(cx, &needed))?;-- self.freeze_all()?;-- // Verify metadata alignment before checking if we have batches to output- self.joined_record_batches- .filter_metadata- .debug_assert_metadata_aligned();-- // For filtered joins, skip output and let Init state handle it- if needs_deferred_filtering(&self.filter, self.join_type) {- continue;- }-- // For non-filtered joins, only output if we have a completed batch- if self- .joined_record_batches- .joined_batches- .has_completed_batch()- {- let record_batch = self- .joined_record_batches- .joined_batches- .next_completed_batch()- .expect("has_completed_batch was true");- (&record_batch)- .record_output(&self.join_metrics.baseline_metrics());- return Poll::Ready(Some(Ok(record_batch)));- }-- // Otherwise keep buffering (don't output yet)- continue;- }-- self.join_partial()?;-- if self.num_unfrozen_pairs() < self.batch_size- && self.buffered_data.scanning_finished()- {- self.buffered_data.scanning_reset();- self.state = SortMergeJoinState::EmitReadyThenInit;- }- // Note: If join_partial() reached the batch size, the loop repeats to freeze the data.- }- SortMergeJoinState::Exhausted => {- let needed =- self.get_required_batch_indices(self.buffered_data.batches.len());- ready!(self.poll_spilled_batches(cx, &needed))?;-- self.freeze_all()?;-- // Verify metadata alignment before final output- self.joined_record_batches- .filter_metadata- .debug_assert_metadata_aligned();-- // For filtered joins, must concat and filter ALL data at once- if needs_deferred_filtering(&self.filter, self.join_type)- && !self.joined_record_batches.joined_batches.is_empty()- {- let record_batch = self.filter_joined_batch()?;- (&record_batch)- .record_output(&self.join_metrics.baseline_metrics());- return Poll::Ready(Some(Ok(record_batch)));- }-- // For non-filtered joins, finish buffered data first- if !self.joined_record_batches.joined_batches.is_empty() {- self.joined_record_batches- .joined_batches- .finish_buffered_batch()?;- }-- // Output one completed batch at a time (stay in Exhausted until empty)- if self- .joined_record_batches- .joined_batches- .has_completed_batch()- {- let record_batch = self- .joined_record_batches- .joined_batches- .next_completed_batch()- .expect("has_completed_batch was true");- (&record_batch)- .record_output(&self.join_metrics.baseline_metrics());- return Poll::Ready(Some(Ok(record_batch)));- }-- // Finally check self.output BatchCoalescer (used by filtered joins)- return if !self.output.is_empty() {- self.output.finish_buffered_batch()?;- let record_batch = self- .output- .next_completed_batch()- .expect("Failed to get last batch");- (&record_batch)- .record_output(&self.join_metrics.baseline_metrics());- Poll::Ready(Some(Ok(record_batch)))- } else {- Poll::Ready(None)- };- }- }- }- }-}impl MaterializingSortMergeJoinStream {#[expect(clippy::too_many_arguments)]@@ -836,7 +541,7 @@ impl MaterializingSortMergeJoinStream {reservation: MemoryReservation,spill_manager: SpillManager,runtime_env: Arc<RuntimeEnv>,- ) -> Result<Self> {+ ) -> Result<SendableRecordBatchStream> {let streamed_schema = streamed.schema();let buffered_schema = buffered.schema();debug_assert!(@@ -847,8 +552,8 @@ impl MaterializingSortMergeJoinStream {"MaterializingSortMergeJoinStream does not handle {join_type:?}; \semi/anti/mark joins use BitwiseSortMergeJoinStream");- Ok(Self {- state: SortMergeJoinState::Init,+ let join_time = join_metrics.join_time();+ let mut this = Self {sort_options,null_equality,schema: Arc::clone(&schema),@@ -858,13 +563,12 @@ impl MaterializingSortMergeJoinStream {buffered,streamed_batch: StreamedBatch::new_empty(streamed_schema),buffered_data: BufferedData::default(),- streamed_joined: false,- buffered_joined: false,- streamed_state: StreamedState::Init,- buffered_state: BufferedState::Init,- current_ordering: Ordering::Equal,+ buffered_group_matched: false,+ streamed_exhausted: false,+ buffered_exhausted: false,on_streamed,on_buffered,+ deferred_filtering: needs_deferred_filtering(&filter, join_type),filter,joined_record_batches: JoinedRecordBatches {joined_batches: BatchCoalescer::new(Arc::clone(&schema), batch_size)@@ -879,12 +583,299 @@ impl MaterializingSortMergeJoinStream {reservation,runtime_env,spill_manager,- spill_stream: None,spilled_batch_count: 0,+ join_time,+ join_time_start: None,streamed_buffered_cmp: None,buffered_equality_cmp: None,- streamed_batch_counter: AtomicUsize::new(0),- })+ streamed_batch_counter: 0,+ };++ let schema = Arc::clone(&this.schema);+ let baseline_metrics = this.join_metrics.baseline_metrics();++ let stream = async_try_stream(|mut emitter| async move {+ this.start_join_time();+ let result = this.join(&mut emitter).await;+ this.stop_join_time();+ result+ });+ // ObservedStream records the baseline metrics (output rows/batches,+ // end time).+ Ok(Box::pin(ObservedStream::new(+ Box::pin(RecordBatchStreamAdapter::new(schema, stream)),+ baseline_metrics,+ None,+ )))+ }++ /// Main loop: the textbook sort-merge join.+ ///+ /// Both inputs arrive sorted on the join keys. The streamed side is+ /// consumed one row at a time; the buffered side one key *group* (all+ /// contiguous rows sharing a key) at a time+ async fn join(+ &mut self,+ emitter: &mut TryEmitter<RecordBatch, DataFusionError>,+ ) -> Result<()> {+ // 1. Load the first streamed row and the first buffered key group.+ self.load_next_streamed_batch().await?;+ self.advance_buffered_group().await?;++ // 2. Merge-scan while either input still has rows.+ while !(self.streamed_exhausted && self.buffered_exhausted) {+ // Flush the deferred-filtering pipeline once a full batch of+ // rows accumulated (filtered outer joins output through it).+ if self.deferred_filtering+ && self.deferred_rows_accumulated() >= self.batch_size+ {+ self.emit_deferred_output(emitter).await?;+ }++ // 3. Compare the join keys at both cursors. An exhausted side+ // compares as the larger one, so the other side keeps+ // draining through its own arm.+ match self.compare_streamed_buffered()? {+ // 3a. The streamed row can never match: null-join it (outer+ // joins emit it; inner joins drop it), then advance.+ Ordering::Less => {+ self.null_join_streamed_row();+ if self.num_unfrozen_pairs() >= self.batch_size {+ self.freeze_and_emit(emitter).await?;+ }+ if !self.try_advance_streamed_row() {+ self.load_next_streamed_batch().await?;+ }+ }+ // 3b. The buffered group can never match again: null-join+ // it if nothing matched it (FULL join), then advance to+ // the next key group.+ Ordering::Greater => {+ self.null_join_buffered_group();+ if !self.try_advance_buffered_group()? {+ self.advance_buffered_group().await?;+ }+ }+ // 3c. Match: pair the streamed row with the whole group —+ // materializing ("freezing") mid-scan whenever a full+ // batch of pairs accumulates — then advance streamed.+ // The group stays for the next streamed row.+ Ordering::Equal => {+ while !self.pair_streamed_row_with_group() {+ self.freeze_and_emit(emitter).await?;+ }+ if !self.try_advance_streamed_row() {+ self.load_next_streamed_batch().await?;+ }+ }+ }++ // 4. Emit completed output batches (filtered joins emit+ // through the deferred-filtering pipeline above instead).+ if !self.deferred_filtering+ && self+ .joined_record_batches+ .joined_batches+ .has_completed_batch()+ {+ self.emit_completed_joined_batches(emitter).await;+ }+ }++ // 5. Flush everything that remains.+ self.on_children_exhausted(emitter).await+ }++ /// `Equal`: pair the current streamed row with every row of the+ /// buffered key group, and mark the group as matched.+ ///+ /// Returns false when a full batch of pairs has accumulated (the scan+ /// may or may not be complete): the caller must materialize+ /// (`freeze_and_emit`) and call again, which resumes the scan where it+ /// paused. Returns true when the group scan is complete and there is+ /// room for more pairs.+ fn pair_streamed_row_with_group(&mut self) -> bool {+ while !self.buffered_data.scanning_finished()+ && self.num_unfrozen_pairs() < self.batch_size+ {+ let scanning_idx = self.buffered_data.scanning_idx();+ self.streamed_batch.append_output_pair(+ Some(self.buffered_data.scanning_batch_idx),+ Some(scanning_idx),+ self.batch_size,+ );+ self.buffered_data.scanning_advance();+ }+ if self.num_unfrozen_pairs() >= self.batch_size {+ return false;+ }++ self.buffered_group_matched = true;+ self.buffered_data.scanning_reset();+ true+ }++ /// `Less` (outer joins): no buffered row matches the current streamed+ /// row — emit it joined to NULLs. Inner joins emit nothing.+ fn null_join_streamed_row(&mut self) {+ if matches!(+ self.join_type,+ JoinType::Left | JoinType::Right | JoinType::Full+ ) {+ let scanning_batch_idx = if self.buffered_data.scanning_finished() {+ None+ } else {+ Some(self.buffered_data.scanning_batch_idx)+ };+ self.streamed_batch.append_output_pair(+ scanning_batch_idx,+ None,+ self.batch_size,+ );+ }+ self.buffered_data.scanning_reset();+ }++ /// `Greater` (FULL join): the buffered group can never match a streamed+ /// row anymore — if nothing matched it, mark all its rows for+ /// null-joined output (produced when the group's batches are dequeued).+ fn null_join_buffered_group(&mut self) {+ if self.join_type == JoinType::Full && !self.buffered_group_matched {+ while !self.buffered_data.scanning_finished() {+ let scanning_idx = self.buffered_data.scanning_idx();+ self.buffered_data+ .scanning_batch_mut()+ .null_joined+ .push(scanning_idx);+ self.buffered_data.scanning_advance();+ }+ }+ self.buffered_data.scanning_reset();+ }++ /// Start (resume) the `join_time` clock.+ fn start_join_time(&mut self) {+ debug_assert!(self.join_time_start.is_none(), "join_time already running");+ self.join_time_start = Some(Instant::now());+ }++ /// Stop (pause) the `join_time` clock, accumulating the elapsed span.+ ///+ /// Called around awaits whose duration is not the join's own work: the+ /// child input streams' `next()` and `emitter.emit()` (where the+ /// consumer processes the batch). The join's own spill write and+ /// read-back are NOT excluded — that time is join work.+ fn stop_join_time(&mut self) {+ if let Some(start) = self.join_time_start.take() {+ self.join_time.add_elapsed(start);+ }+ }++ /// Number of rows currently waiting in the deferred-filtering pipeline.+ ///+ /// Typically bounded to ~2*batch_size: one batch_size worth from+ /// freeze_dequeuing_buffered() (when an input batch is fully consumed),+ /// plus up to batch_size pairs accumulating toward the next freeze. A+ /// single streamed row matching a very large key group can exceed that+ /// (its pairs freeze into the pipeline before the gate runs again — same+ /// as the pre-generator design). This does not reintroduce the unbounded+ /// buffering fixed by PR #20482; `on_children_exhausted` flushes the+ /// remainder.+ fn deferred_rows_accumulated(&self) -> usize {+ self.num_unfrozen_pairs()+ + self.joined_record_batches.filter_metadata.filter_mask.len()+ }++ /// Run the deferred-filtering pipeline over everything accumulated so+ /// far and emit its completed output, if any. Clears the accumulation+ /// it processed.+ ///+ /// The caller gates this on `deferred_rows_accumulated() >= batch_size`:+ /// running the pipeline per row instead (concat + correct_mask ++ /// filter_by_type) would dominate runtime for unique keys.+ async fn emit_deferred_output(+ &mut self,+ emitter: &mut TryEmitter<RecordBatch, DataFusionError>,+ ) -> Result<()> {+ // Ensure required spilled batches are restored to memory before+ // processing, as this path invokes freeze_all().+ self.restore_spilled_batches_for_freeze().await?;+ if let Some(batch) = self.process_filtered_batches()? {+ // While the emitted batch is in the consumer's hands the join+ // isn't doing any work.+ self.stop_join_time();+ emitter.emit(batch).await;+ self.start_join_time();+ }+ Ok(())+ }++ /// Restore every spilled buffered batch that the next freeze needs.+ async fn restore_spilled_batches_for_freeze(&mut self) -> Result<()> {+ let needed = self.get_required_batch_indices(self.buffered_data.batches.len());+ self.restore_spilled_batches(&needed).await+ }++ /// Emit all completed joined batches to the stream consumer.+ async fn emit_completed_joined_batches(+ &mut self,+ emitter: &mut TryEmitter<RecordBatch, DataFusionError>,+ ) {+ while let Some(record_batch) = self+ .joined_record_batches+ .joined_batches+ .next_completed_batch()+ {+ // While the emitted batch is in the consumer's hands the join+ // isn't doing any work.+ self.stop_join_time();+ emitter.emit(record_batch).await;+ self.start_join_time();+ }+ }++ /// Flush everything that remains once both inputs are exhausted.+ async fn on_children_exhausted(+ &mut self,+ emitter: &mut TryEmitter<RecordBatch, DataFusionError>,+ ) -> Result<()> {+ // Freeze the remaining pairs, restoring any spilled batches needed.+ self.restore_spilled_batches_for_freeze().await?;+ self.freeze_all()?;++ // Verify metadata alignment before final output+ self.joined_record_batches+ .filter_metadata+ .debug_assert_metadata_aligned();++ if self.deferred_filtering {+ // Filtered joins must concat and filter ALL remaining data at once+ if !self.joined_record_batches.joined_batches.is_empty() {+ let record_batch = self.filter_joined_batch()?;+ self.stop_join_time();+ emitter.emit(record_batch).await;+ self.start_join_time();+ }+ } else if !self.joined_record_batches.joined_batches.is_empty() {+ // For non-filtered joins, finish buffered data first, then emit+ // every completed batch.+ self.joined_record_batches+ .joined_batches+ .finish_buffered_batch()?;+ self.emit_completed_joined_batches(emitter).await;+ }++ // Drain the double-buffering coalescer used by filtered joins.+ if !self.output.is_empty() {+ self.output.finish_buffered_batch()?;+ while let Some(record_batch) = self.output.next_completed_batch() {+ self.stop_join_time();+ emitter.emit(record_batch).await;+ self.start_join_time();+ }+ }++ Ok(())}/// Build a comparator for streamed vs buffered head batch keys.@@ -927,9 +918,9 @@ impl MaterializingSortMergeJoinStream {/// Process accumulated batches for filtered joins///- /// Freezes unfrozen pairs, applies deferred filtering, and outputs if ready.- /// Returns Poll::Ready with a batch if one is available, otherwise Poll::Pending.- fn process_filtered_batches(&mut self) -> Poll<Option<Result<RecordBatch>>> {+ /// Freezes unfrozen pairs, applies deferred filtering, and returns a+ /// completed output batch if one is ready.+ fn process_filtered_batches(&mut self) -> Result<Option<RecordBatch>> {self.freeze_all()?;self.joined_record_batches@@ -947,12 +938,11 @@ impl MaterializingSortMergeJoinStream {.output.next_completed_batch().expect("Failed to get output batch");- (&record_batch).record_output(&self.join_metrics.baseline_metrics());- return Poll::Ready(Some(Ok(record_batch)));+ return Ok(Some(record_batch));}}- Poll::Pending+ Ok(None)}/// Identifies which buffered batches are needed for the upcoming freeze operation@@ -981,11 +971,10 @@ impl MaterializingSortMergeJoinStream {/// Asynchronously reads spilled batches back into memory./// Only processes the required indices to avoid OOMs.- fn poll_spilled_batches(+ async fn restore_spilled_batches(&mut self,- cx: &mut Context<'_>,required_indices: &[usize],- ) -> Poll<Result<()>> {+ ) -> Result<()> {for &idx in required_indices {// Guard against indices that might be out of bounds if the queue was clearedif idx >= self.buffered_data.batches.len() {@@ -995,15 +984,12 @@ impl MaterializingSortMergeJoinStream {let bb = &mut self.buffered_data.batches[idx];if let BufferedBatchState::Spilled(spill_file) = &bb.batch {- if self.spill_stream.is_none() {- let stream = self- .spill_manager- .read_spill_as_stream(Arc::clone(spill_file), None)?;- self.spill_stream = Some(stream);- }+ let mut spill_stream = self+ .spill_manager+ .read_spill_as_stream(Arc::clone(spill_file), None)?;- match ready!(self.spill_stream.as_mut().unwrap().poll_next_unpin(cx)) {- Some(Ok(batch)) => {+ match spill_stream.next().await.transpose()? {+ Some(batch) => {// Transition the batch back to InMemorybb.batch = BufferedBatchState::InMemory(batch);self.spilled_batch_count -= 1;@@ -1016,78 +1002,65 @@ impl MaterializingSortMergeJoinStream {self.join_metrics.peak_mem_used().set_max(self.reservation.size());-- self.spill_stream = None;- }- Some(Err(e)) => {- self.spill_stream = None;- return Poll::Ready(Err(e));}None => {- self.spill_stream = None;- return Poll::Ready(internal_err!("Spill file was empty"));+ return internal_err!("Spill file was empty");}}}}- Poll::Ready(Ok(()))++ Ok(())}- /// Poll next streamed row- fn poll_streamed_row(&mut self, cx: &mut Context) -> Poll<Option<Result<()>>> {+ /// Sync fast path of advancing the streamed cursor: move to the next row+ /// of the current batch. Returns false at the batch boundary, where the+ /// caller must load the next batch via+ /// [`Self::load_next_streamed_batch`].+ fn try_advance_streamed_row(&mut self) -> bool {+ if self.streamed_batch.idx + 1 < self.streamed_batch.batch.num_rows() {+ self.streamed_batch.idx += 1;+ return true;+ }+ false+ }++ /// Load the next streamed batch (freezing the finished one) and point+ /// the streamed cursor at its first row. Sets `streamed_exhausted` when+ /// the streamed input has no more rows.+ async fn load_next_streamed_batch(&mut self) -> Result<()> {loop {- match &self.streamed_state {- StreamedState::Init => {- if self.streamed_batch.idx + 1 < self.streamed_batch.batch.num_rows()- {- self.streamed_batch.idx += 1;- self.streamed_state = StreamedState::Ready;- return Poll::Ready(Some(Ok(())));- } else {- self.streamed_state = StreamedState::Polling;- }+ // Loading a new streamed batch freezes the current one, which+ // materializes buffered columns — restore any spilled buffered+ // batches it needs first.+ self.restore_spilled_batches_for_freeze().await?;++ // The child's execution time is its own, not join_time.+ self.stop_join_time();+ let item = self.streamed.next().await.transpose();+ self.start_join_time();+ match item? {+ None => {+ // Release the streamed input pipeline's resources.+ let streamed_schema = self.streamed.schema();+ self.streamed =+ Box::pin(EmptyRecordBatchStream::new(streamed_schema));+ self.streamed_exhausted = true;+ return Ok(());}- StreamedState::Polling => {- let needed =- self.get_required_batch_indices(self.buffered_data.batches.len());- if let Err(e) = ready!(self.poll_spilled_batches(cx, &needed)) {- return Poll::Ready(Some(Err(e)));- }-- match self.streamed.poll_next_unpin(cx)? {- Poll::Pending => {- return Poll::Pending;- }- Poll::Ready(None) => {- // Release the streamed input pipeline's resources.- let streamed_schema = self.streamed.schema();- self.streamed =- Box::pin(EmptyRecordBatchStream::new(streamed_schema));- self.streamed_state = StreamedState::Exhausted;- }- Poll::Ready(Some(batch)) => {- if batch.num_rows() > 0 {- self.freeze_streamed()?;- self.join_metrics.input_batches().add(1);- self.join_metrics.input_rows().add(batch.num_rows());- self.streamed_batch =- StreamedBatch::new(batch, &self.on_streamed);- self.rebuild_streamed_buffered_cmp()?;- // Every incoming streaming batch should have its unique id- // Check `JoinedRecordBatches.self.streamed_batch_counter` documentation- self.streamed_batch_counter- .fetch_add(1, std::sync::atomic::Ordering::SeqCst);- self.streamed_state = StreamedState::Ready;- }- }+ Some(batch) => {+ if batch.num_rows() > 0 {+ self.freeze_streamed()?;+ self.join_metrics.input_batches().add(1);+ self.join_metrics.input_rows().add(batch.num_rows());+ self.streamed_batch =+ StreamedBatch::new(batch, &self.on_streamed);+ self.rebuild_streamed_buffered_cmp()?;+ // Every incoming streamed batch gets a unique id.+ self.streamed_batch_counter += 1;+ return Ok(());}}- StreamedState::Ready => {- return Poll::Ready(Some(Ok(())));- }- StreamedState::Exhausted => {- return Poll::Ready(None);- }}}}@@ -1150,146 +1123,193 @@ impl MaterializingSortMergeJoinStream {Ok(())}- /// Poll next buffered batches- fn poll_buffered_batches(&mut self, cx: &mut Context) -> Poll<Option<Result<()>>> {- loop {- match &self.buffered_state {- BufferedState::Init => {- // pop previous buffered batches- let mut head_changed = false;- while !self.buffered_data.batches.is_empty() {- let head_batch = self.buffered_data.head_batch();- // If the head batch is fully processed, dequeue it and produce output of it.- if head_batch.range.end == head_batch.num_rows {- // load the spilled head batch before dequeuing- let needed = self.get_required_batch_indices(1);- if let Err(e) = ready!(self.poll_spilled_batches(cx, &needed))- {- return Poll::Ready(Some(Err(e)));- }+ /// Sync fast path of [`Self::advance_buffered_group`]: when the next+ /// group starts in the single remaining buffered batch and provably ends+ /// within it (the common case — a group only reaches a batch boundary+ /// once per batch), advance entirely synchronously. Returns false —+ /// leaving all state unchanged — when the async path must run instead.+ fn try_advance_buffered_group(&mut self) -> Result<bool> {+ if self.buffered_data.batches.len() != 1 {+ return Ok(false);+ }+ let head_batch = self.buffered_data.head_batch();+ if head_batch.range.end == head_batch.num_rows {+ // Fully consumed — needs dequeuing (and loading the next batch).+ return Ok(false);+ }- self.freeze_dequeuing_buffered()?;- if let Some(mut buffered_batch) =- self.buffered_data.batches.pop_front()- {- self.produce_buffered_not_matched(&mut buffered_batch)?;- self.free_reservation(&buffered_batch);- if matches!(- buffered_batch.batch,- BufferedBatchState::Spilled(_)- ) {- self.spilled_batch_count -= 1;- }- head_changed = true;- }- } else {- // If the head batch is not fully processed, break the loop.- // Streamed batch will be joined with the head batch in the next step.- break;- }- }- if head_changed {+ if self.buffered_equality_cmp.is_none() {+ self.rebuild_buffered_equality_cmp()?;+ }+ let cmp = self.buffered_equality_cmp.as_ref().unwrap();++ // Scan the next group's extent before committing any state, so a+ // bail-out (the group may span into the next batch) leaves+ // everything untouched for the async path.+ let batch = self.buffered_data.head_batch();+ let group_start = batch.range.end;+ let mut group_end = group_start + 1;+ while group_end < batch.num_rows && cmp.is_equal(group_start, group_end) {+ group_end += 1;+ }+ if group_end == batch.num_rows {+ return Ok(false);+ }++ let batch = self.buffered_data.tail_batch_mut();+ batch.range.start = group_start;+ batch.range.end = group_end;+ self.buffered_group_matched = false;+ Ok(true)+ }++ /// Advance the buffered side to the next key group: dequeue batches+ /// fully consumed by the previous group, then collect all contiguous+ /// rows sharing the next join key (the group may span multiple buffered+ /// batches). Sets `buffered_exhausted` when no group remains.+ async fn advance_buffered_group(&mut self) -> Result<()> {+ self.buffered_group_matched = false;+ self.dequeue_consumed_buffered_batches().await?;++ if self.buffered_data.batches.is_empty() {+ // Load the batch holding the first row of the next group.+ if !self.load_next_buffered_batch().await? {+ self.buffered_exhausted = true;+ return Ok(());+ }+ } else {+ // Seed the next group at the first unconsumed row of the+ // remaining batch.+ let tail_batch = self.buffered_data.tail_batch_mut();+ tail_batch.range.start = tail_batch.range.end;+ tail_batch.range.end += 1;+ }++ self.extend_buffered_group().await+ }++ /// Dequeue buffered batches fully consumed by the previous group,+ /// producing their pending output (e.g. Full-join null-joined rows).+ async fn dequeue_consumed_buffered_batches(&mut self) -> Result<()> {+ let mut head_changed = false;+ while !self.buffered_data.batches.is_empty() {+ let head_batch = self.buffered_data.head_batch();+ if head_batch.range.end != head_batch.num_rows {+ // The next group starts within the head batch: streamed rows+ // will be joined with the head batch in the next step.+ break;+ }+ // load the spilled head batch before dequeuing+ let needed = self.get_required_batch_indices(1);+ self.restore_spilled_batches(&needed).await?;++ self.freeze_dequeuing_buffered()?;+ if let Some(mut buffered_batch) = self.buffered_data.batches.pop_front() {+ self.produce_buffered_not_matched(&mut buffered_batch)?;+ self.free_reservation(&buffered_batch);+ if matches!(buffered_batch.batch, BufferedBatchState::Spilled(_)) {+ self.spilled_batch_count -= 1;+ }+ head_changed = true;+ }+ }+ if head_changed {+ self.streamed_buffered_cmp = None;+ self.buffered_equality_cmp = None;+ }+ Ok(())+ }++ /// Load the next non-empty buffered batch and seed a new group with its+ /// first row. Returns false when the buffered input is exhausted.+ async fn load_next_buffered_batch(&mut self) -> Result<bool> {+ loop {+ // The child's execution time is its own, not join_time.+ self.stop_join_time();+ let item = self.buffered.next().await.transpose();+ self.start_join_time();+ match item? {+ None => {+ // Release the buffered input pipeline's resources.+ let buffered_schema = self.buffered.schema();+ self.buffered =+ Box::pin(EmptyRecordBatchStream::new(buffered_schema));+ return Ok(false);+ }+ Some(batch) => {+ self.join_metrics.input_batches().add(1);+ self.join_metrics.input_rows().add(batch.num_rows());++ if batch.num_rows() > 0 {+ let buffered_batch =+ BufferedBatch::new(batch, 0..1, &self.on_buffered);+ self.allocate_reservation(buffered_batch)?;self.streamed_buffered_cmp = None;- self.buffered_equality_cmp = None;+ return Ok(true);}- if self.buffered_data.batches.is_empty() {- self.buffered_state = BufferedState::PollingFirst;+ }+ }+ }+ }++ /// Extend the current group with every following row that shares its+ /// key, loading more buffered batches as needed.+ async fn extend_buffered_group(&mut self) -> Result<()> {+ loop {+ if self.buffered_data.tail_batch().range.end+ < self.buffered_data.tail_batch().num_rows+ {+ if self.buffered_equality_cmp.is_none() {+ self.rebuild_buffered_equality_cmp()?;+ }+ while self.buffered_data.tail_batch().range.end+ < self.buffered_data.tail_batch().num_rows+ {+ if self.buffered_equality_cmp.as_ref().unwrap().is_equal(+ self.buffered_data.head_batch().range.start,+ self.buffered_data.tail_batch().range.end,+ ) {+ self.buffered_data.tail_batch_mut().range.end += 1;} else {- let tail_batch = self.buffered_data.tail_batch_mut();- tail_batch.range.start = tail_batch.range.end;- tail_batch.range.end += 1;- self.buffered_state = BufferedState::PollingRest;+ // Group complete within the current batch.+ return Ok(());}}- BufferedState::PollingFirst => match self.buffered.poll_next_unpin(cx)? {- Poll::Pending => {- return Poll::Pending;- }- Poll::Ready(None) => {+ } else {+ // The child's execution time is its own, not join_time.+ self.stop_join_time();+ let item = self.buffered.next().await.transpose();+ self.start_join_time();+ match item? {+ None => {+ // Group complete; the input is done but the group is+ // still valid — `buffered_exhausted` is only set once+ // it has been fully consumed and dequeued.// Release the buffered input pipeline's resources.let buffered_schema = self.buffered.schema();self.buffered =Box::pin(EmptyRecordBatchStream::new(buffered_schema));- self.buffered_state = BufferedState::Exhausted;- return Poll::Ready(None);+ return Ok(());}- Poll::Ready(Some(batch)) => {+ Some(batch) => {+ // Polling batches coming concurrently as multiple partitionsself.join_metrics.input_batches().add(1);self.join_metrics.input_rows().add(batch.num_rows());-if batch.num_rows() > 0 {let buffered_batch =- BufferedBatch::new(batch, 0..1, &self.on_buffered);-+ BufferedBatch::new(batch, 0..0, &self.on_buffered);self.allocate_reservation(buffered_batch)?;- self.streamed_buffered_cmp = None;- self.buffered_state = BufferedState::PollingRest;- }- }- },- BufferedState::PollingRest => {- if self.buffered_data.tail_batch().range.end- < self.buffered_data.tail_batch().num_rows- {- if self.buffered_equality_cmp.is_none() {- self.rebuild_buffered_equality_cmp()?;- }- while self.buffered_data.tail_batch().range.end- < self.buffered_data.tail_batch().num_rows- {- if self.buffered_equality_cmp.as_ref().unwrap().is_equal(- self.buffered_data.head_batch().range.start,- self.buffered_data.tail_batch().range.end,- ) {- self.buffered_data.tail_batch_mut().range.end += 1;- } else {- self.buffered_state = BufferedState::Ready;- return Poll::Ready(Some(Ok(())));- }- }- } else {- match self.buffered.poll_next_unpin(cx)? {- Poll::Pending => {- return Poll::Pending;- }- Poll::Ready(None) => {- // Release the buffered input pipeline's resources.- let buffered_schema = self.buffered.schema();- self.buffered = Box::pin(EmptyRecordBatchStream::new(- buffered_schema,- ));- self.buffered_state = BufferedState::Ready;- }- Poll::Ready(Some(batch)) => {- // Polling batches coming concurrently as multiple partitions- self.join_metrics.input_batches().add(1);- self.join_metrics.input_rows().add(batch.num_rows());- if batch.num_rows() > 0 {- let buffered_batch = BufferedBatch::new(- batch,- 0..0,- &self.on_buffered,- );- self.allocate_reservation(buffered_batch)?;- self.buffered_equality_cmp = None;- }- }+ self.buffered_equality_cmp = None;}}}- BufferedState::Ready => {- return Poll::Ready(Some(Ok(())));- }- BufferedState::Exhausted => {- return Poll::Ready(None);- }}}}/// Get comparison result of streamed row and buffered batchesfn compare_streamed_buffered(&mut self) -> Result<Ordering> {- if self.streamed_state == StreamedState::Exhausted {+ if self.streamed_exhausted {return Ok(Ordering::Greater);}if !self.buffered_data.has_buffered_rows() {@@ -1305,81 +1325,23 @@ impl MaterializingSortMergeJoinStream {))}- /// Produce join and fill output buffer until reaching target batch size- /// or the join is finished- fn join_partial(&mut self) -> Result<()> {- // Whether to join streamed rows- let mut join_streamed = false;- // Whether to join buffered rows- let mut join_buffered = false;-- // determine whether we need to join streamed/buffered rows- match self.current_ordering {- Ordering::Less => {- if matches!(- self.join_type,- JoinType::Left | JoinType::Right | JoinType::Full- ) {- join_streamed = !self.streamed_joined;- }- }- Ordering::Equal => {- join_streamed = true;- join_buffered = true;- }- Ordering::Greater => {- if self.join_type == JoinType::Full {- join_buffered = !self.buffered_joined;- };- }- }- if !join_streamed && !join_buffered {- // no joined data- self.buffered_data.scanning_finish();- return Ok(());- }-- if join_buffered {- // joining streamed/nulls and buffered- while !self.buffered_data.scanning_finished()- && self.num_unfrozen_pairs() < self.batch_size- {- let scanning_idx = self.buffered_data.scanning_idx();- if join_streamed {- // Join streamed row and buffered row- self.streamed_batch.append_output_pair(- Some(self.buffered_data.scanning_batch_idx),- Some(scanning_idx),- self.batch_size,- );- } else {- // Join nulls and buffered row for FULL join- self.buffered_data- .scanning_batch_mut()- .null_joined- .push(scanning_idx);- }- self.buffered_data.scanning_advance();+ /// Materialize ("freeze") the accumulated pairs — restoring any spilled+ /// batches they reference first — and emit completed output batches+ /// (filtered joins emit through the deferred-filtering gate instead).+ async fn freeze_and_emit(+ &mut self,+ emitter: &mut TryEmitter<RecordBatch, DataFusionError>,+ ) -> Result<()> {+ self.restore_spilled_batches_for_freeze().await?;+ self.freeze_all()?;- if self.buffered_data.scanning_finished() {- self.streamed_joined = join_streamed;- self.buffered_joined = true;- }- }- } else {- // joining streamed and nulls- let scanning_batch_idx = if self.buffered_data.scanning_finished() {- None- } else {- Some(self.buffered_data.scanning_batch_idx)- };- self.streamed_batch.append_output_pair(- scanning_batch_idx,- None,- self.batch_size,- );- self.buffered_data.scanning_finish();- self.streamed_joined = true;+ if !self.deferred_filtering+ && self+ .joined_record_batches+ .joined_batches+ .has_completed_batch()+ {+ self.emit_completed_joined_batches(emitter).await;}Ok(())}@@ -1513,7 +1475,7 @@ impl MaterializingSortMergeJoinStream {// but must flow through the same pipeline as matched rows to// preserve output ordering. Use null metadata as a sentinel so// get_corrected_filter_mask() passes them through unchanged.- if needs_deferred_filtering(&self.filter, self.join_type) {+ if self.deferred_filtering {self.joined_record_batches.push_batch_with_null_metadata(batch, self.join_type);} else {@@ -1616,12 +1578,12 @@ impl MaterializingSortMergeJoinStream {filter_result_mask.clone()};- if needs_deferred_filtering(&self.filter, self.join_type) {+ if self.deferred_filtering {self.joined_record_batches.push_batch_with_filter_metadata(output_batch,&combined_left_indices,&mask,- self.streamed_batch_counter.load(Relaxed),+ self.streamed_batch_counter,self.join_type,);} else {@@ -1968,9 +1930,9 @@ fn fetch_right_columns_from_batch_by_idxs(pub(super) struct BufferedData {/// Buffered batches with the same keypub batches: VecDeque<BufferedBatch>,- /// current scanning batch index used in join_partial()+ /// current scanning batch index used by the group-scan phasepub scanning_batch_idx: usize,- /// current scanning offset used in join_partial()+ /// current scanning offset used by the group-scan phasepub scanning_offset: usize,}@@ -2023,11 +1985,6 @@ impl BufferedData {pub fn scanning_finished(&self) -> bool {self.scanning_batch_idx == self.batches.len()}-- pub fn scanning_finish(&mut self) {- self.scanning_batch_idx = self.batches.len();- self.scanning_offset = 0;- }}/// Get join array refs of given batch and join columnsdiff --git a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rsindex 313818cf6c..3dbb50eba0 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs@@ -4315,6 +4315,147 @@ async fn join_time_excludes_consumer_wait() -> Result<()> {.await}+/// Three 2-row batches with unique matching keys, right-side column names.+fn join_time_batches_right() -> Vec<RecordBatch> {+ vec![+ build_table_i32(+ ("a2", &vec![0, 1]),+ ("b2", &vec![1, 2]),+ ("c2", &vec![7, 8]),+ ),+ build_table_i32(+ ("a2", &vec![2, 3]),+ ("b2", &vec![3, 4]),+ ("c2", &vec![7, 8]),+ ),+ build_table_i32(+ ("a2", &vec![4, 5]),+ ("b2", &vec![5, 6]),+ ("c2", &vec![7, 8]),+ ),+ ]+}++/// Build a no-filter Inner materializing join over the given input streams.+/// The small batch size makes the output surface as multiple batches, so a+/// slow consumer test sees multiple emits.+fn materializing_join_time_test_join(+ streamed: SendableRecordBatchStream,+ buffered: SendableRecordBatchStream,+) -> (SendableRecordBatchStream, ExecutionPlanMetricsSet) {+ use crate::joins::sort_merge_join::materializing_stream::MaterializingSortMergeJoinStream;+ use crate::joins::sort_merge_join::metrics::SortMergeJoinMetrics;++ let metrics = ExecutionPlanMetricsSet::new();+ let out_schema = Arc::new(Schema::new(+ streamed+ .schema()+ .fields()+ .iter()+ .chain(buffered.schema().fields().iter())+ .map(|f| f.as_ref().clone())+ .collect::<Vec<_>>(),+ ));+ let (reservation, spill_manager, runtime_env) =+ test_stream_resources(buffered.schema(), &metrics);+ let stream = MaterializingSortMergeJoinStream::try_new(+ out_schema,+ vec![SortOptions::default()],+ NullEquality::NullEqualsNothing,+ streamed,+ buffered,+ vec![Arc::new(Column::new("b1", 1)) as _],+ vec![Arc::new(Column::new("b2", 1)) as _],+ None,+ Inner,+ 2,+ SortMergeJoinMetrics::new(0, &metrics),+ reservation,+ spill_manager,+ runtime_env,+ )+ .unwrap();+ (stream, metrics)+}++/// join_time must not include time spent waiting for the streamed input.+#[tokio::test]+async fn materializing_join_time_excludes_streamed_input_wait() -> Result<()> {+ check_join_time_excluded(|delay| async move {+ let streamed = delayed_stream(join_time_batches(), delay);+ let buffered = delayed_stream(join_time_batches_right(), Duration::ZERO);+ let (stream, metrics) = materializing_join_time_test_join(streamed, buffered);++ let start = Instant::now();+ let batches = collect_stream(stream).await?;+ let wall = start.elapsed();++ let rows: usize = batches.iter().map(|b| b.num_rows()).sum();+ assert_eq!(rows, 6, "all rows should match");+ assert!(+ wall >= delay * 3,+ "streamed delays should dominate wall time, got {wall:?}"+ );+ Ok((join_time_of(&metrics), wall))+ })+ .await+}++/// join_time must not include time spent waiting for the buffered input.+#[tokio::test]+async fn materializing_join_time_excludes_buffered_input_wait() -> Result<()> {+ check_join_time_excluded(|delay| async move {+ let streamed = delayed_stream(join_time_batches(), Duration::ZERO);+ let buffered = delayed_stream(join_time_batches_right(), delay);+ let (stream, metrics) = materializing_join_time_test_join(streamed, buffered);++ let start = Instant::now();+ let batches = collect_stream(stream).await?;+ let wall = start.elapsed();++ let rows: usize = batches.iter().map(|b| b.num_rows()).sum();+ assert_eq!(rows, 6, "all rows should match");+ assert!(+ wall >= delay * 3,+ "buffered delays should dominate wall time, got {wall:?}"+ );+ Ok((join_time_of(&metrics), wall))+ })+ .await+}++/// join_time must not include time the consumer spends holding an emitted+/// batch (the generator is suspended inside `emitter.emit` meanwhile).+#[tokio::test]+async fn materializing_join_time_excludes_consumer_wait() -> Result<()> {+ check_join_time_excluded(|delay| async move {+ let streamed = delayed_stream(join_time_batches(), Duration::ZERO);+ let buffered = delayed_stream(join_time_batches_right(), Duration::ZERO);+ let (mut stream, metrics) = materializing_join_time_test_join(streamed, buffered);++ let start = Instant::now();+ let mut output_batches = 0u32;+ while let Some(batch) = stream.next().await {+ batch?;+ output_batches += 1;+ // Simulate a slow consumer between emitted batches.+ tokio::time::sleep(delay).await;+ }+ let wall = start.elapsed();++ assert!(+ output_batches >= 3,+ "expected multiple emitted batches, got {output_batches}"+ );+ assert!(+ wall >= delay * output_batches,+ "consumer delays should dominate wall time, got {wall:?}"+ );+ Ok((join_time_of(&metrics), wall))+ })+ .await+}+/// An inner key group spanning multiple inner batches must survive the inner/// input returning Pending mid-way: inner rows delivered before the Pending/// still take part in the filter evaluation.