Apache DataFusion SQL Query Engine
chore: refactor SortMergeJoin bitwise stream to generators and simplify to be textbook like as possible (#23761)
Raz Luvatonb0b9dae41095
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rsindex d1ca9707fe..3716ecde28 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs@@ -84,8 +84,9 @@//!//! Key groups can span batch boundaries on either side. The stream handles//! this by detecting when a group extends to the end of a batch, loading the-//! next batch, and continuing if the key matches. The [`PendingBoundary`] enum-//! preserves loop context across async `Poll::Pending` re-entries.+//! next batch, and continuing if the key matches. The generator-based stream+//! suspends in place at `await` points, so no explicit re-entry state is+//! needed.//!//! # Memory//!@@ -119,29 +120,31 @@//! factor than the pair-materialization approach.use std::cmp::Ordering;-use std::pin::Pin;use std::sync::Arc;-use std::task::{Context, Poll};+use crate::EmptyRecordBatchStream;use crate::joins::utils::{JoinFilter, JoinKeyComparator, compare_join_arrays};use crate::metrics::{- BaselineMetrics, Count, ExecutionPlanMetricsSet, Gauge, MetricBuilder,+ BaselineMetrics, Count, ExecutionPlanMetricsSet, Gauge, MetricBuilder, Time,};use crate::spill::spill_manager::SpillManager;-use crate::{EmptyRecordBatchStream, RecordBatchStream};+use crate::stream::{ObservedStream, RecordBatchStreamAdapter};use arrow::array::{Array, ArrayRef, BooleanArray, BooleanBufferBuilder, RecordBatch};use arrow::compute::{BatchCoalescer, SortOptions, filter_record_batch, not};use arrow::datatypes::SchemaRef;use arrow::util::bit_chunk_iterator::UnalignedBitChunk;use arrow::util::bit_util::apply_bitwise_binary_op;+use datafusion_common::instant::Instant;use datafusion_common::{- JoinSide, JoinType, NullEquality, Result, ScalarValue, internal_err,+ DataFusionError, JoinSide, JoinType, NullEquality, Result, ScalarValue, internal_err,};use datafusion_execution::memory_pool::MemoryReservation;-use datafusion_execution::{SendableRecordBatchStream, SpillFile};+use datafusion_execution::{+ SendableRecordBatchStream, SpillFile, TryEmitter, async_try_stream,+};use datafusion_physical_expr_common::physical_expr::PhysicalExprRef;-use futures::{Stream, StreamExt, ready};+use futures::StreamExt;/// Evaluates join key expressions against a batch, returning one array per key.fn evaluate_join_keys(@@ -194,26 +197,6 @@ fn find_key_group_end(cmp: &JoinKeyComparator, from: usize, len: usize) -> usizelo}-/// When an outer key group spans a batch boundary, the boundary loop emits-/// the current batch, then polls for the next. If that poll returns Pending,-/// `ready!` exits `poll_join` and we re-enter from the top on the next call.-/// Without this state, the new batch would be processed fresh by the-/// merge-scan — but inner already advanced past this key, so the matching-/// outer rows would be skipped via `Ordering::Less` and never marked.-///-/// This enum carries the last key (as single-row sliced arrays) from the-/// previous batch so we can check whether the next batch continues the same-/// key group. Stored as `Option<PendingBoundary>`: `None` means normal-/// processing.-#[derive(Debug)]-enum PendingBoundary {- /// Resuming a no-filter boundary loop.- NoFilter { saved_keys: Vec<ArrayRef> },- /// Resuming a filtered boundary loop. Inner key data remains in the- /// buffer (or spill file) for the resumed loop.- Filtered { saved_keys: Vec<ArrayRef> },-}-/// Sort-Merge join stream for Semi/Anti/Mark joins.////// Named "bitwise" because it tracks outer-row matches via a per-batch@@ -255,22 +238,6 @@ pub(crate) struct BitwiseSortMergeJoinStream {inner_key_buffer: Vec<RecordBatch>,inner_key_spill: Option<Arc<dyn SpillFile>>,- // Track the active spill_stream- spill_stream: Option<SendableRecordBatchStream>,- // Whether the active spill stream has produced any batches yet.- spill_stream_has_data: bool,- // Prevents wiping out the buffer if we yield while evaluating the filter- inner_group_buffered: bool,-- // True when buffer_inner_key_group returned Pending after partially- // filling inner_key_buffer. On re-entry, buffer_inner_key_group- // must skip clear() and resume from poll_next_inner_batch (the- // current inner_batch was already sliced and pushed before Pending).- buffering_inner_pending: bool,-- // Boundary re-entry state — see PendingBoundary doc comment.- pending_boundary: Option<PendingBoundary>,-// Join ON expressions, evaluated against each new batch to produce// the key arrays used for sorted key comparisons.on_outer: Vec<PhysicalExprRef>,@@ -286,12 +253,18 @@ pub(crate) struct BitwiseSortMergeJoinStream {coalescer: BatchCoalescer,schema: SchemaRef,- // Metrics- join_time: crate::metrics::Time,+ // Metrics — output rows/batches and end time are recorded by the+ // ObservedStream wrapper in try_new, not here.input_batches: Count,input_rows: Count,- baseline_metrics: BaselineMetrics,peak_mem_used: Gauge,+ /// Time spent doing the join's own work (including spill write and+ /// read-back). The clock is stopped while awaiting the child inputs or+ /// the consumer taking an emitted batch — see [`Self::stop_join_time`].+ join_time: Time,+ /// Start of the currently running `join_time` span; `None` while the+ /// clock is stopped.+ join_time_start: Option<Instant>,// Memory / spill — only the inner key buffer is tracked via reservation,// matching existing SMJ (which tracks only the buffered side). The outer@@ -308,14 +281,6 @@ pub(crate) struct BitwiseSortMergeJoinStream {outer_self_cmp: Option<JoinKeyComparator>,/// Comparator for inner self-comparison (find_key_group_end on inner)inner_self_cmp: Option<JoinKeyComparator>,-- // True once the current outer batch has been emitted. The Equal- // branch's inner loops call emit then `ready!(poll_next_outer_batch)`.- // If that poll returns Pending, poll_join re-enters from the top- // on the next poll — with outer_batch still Some and outer_offset- // past the end. The main loop's step 3 would re-emit without this- // guard. Cleared when poll_next_outer_batch loads a new batch.- batch_emitted: bool,}impl BitwiseSortMergeJoinStream {@@ -336,7 +301,7 @@ impl BitwiseSortMergeJoinStream {reservation: MemoryReservation,spill_manager: SpillManager,runtime_env: Arc<datafusion_execution::runtime_env::RuntimeEnv>,- ) -> Result<Self> {+ ) -> Result<SendableRecordBatchStream> {debug_assert!(matches!(join_type,@@ -362,7 +327,7 @@ impl BitwiseSortMergeJoinStream {let peak_mem_used =MetricBuilder::new(metrics).peak_memory_usage("peak_mem_used", partition);- Ok(Self {+ let mut state = Self {join_type,outer,inner,@@ -375,11 +340,6 @@ impl BitwiseSortMergeJoinStream {matched: BooleanBufferBuilder::new(0),inner_key_buffer: vec![],inner_key_spill: None,- spill_stream: None,- spill_stream_has_data: false,- inner_group_buffered: false,- buffering_inner_pending: false,- pending_boundary: None,on_outer,on_inner,filter,@@ -388,12 +348,12 @@ impl BitwiseSortMergeJoinStream {outer_is_left,coalescer: BatchCoalescer::new(Arc::clone(&schema), batch_size).with_biggest_coalesce_batch_size(Some(batch_size / 2)),- schema,- join_time,+ schema: Arc::clone(&schema),input_batches,input_rows,- baseline_metrics,peak_mem_used,+ join_time,+ join_time_start: None,reservation,spill_manager,runtime_env,@@ -401,8 +361,39 @@ impl BitwiseSortMergeJoinStream {outer_inner_cmp: None,outer_self_cmp: None,inner_self_cmp: None,- batch_emitted: false,- })+ };++ let stream = async_try_stream(|mut emitter| async move {+ state.start_join_time();+ let result = state.join(&mut emitter).await;+ state.stop_join_time();+ result+ });+ // ObservedStream records the baseline metrics (output rows/batches,+ // end time) exactly as the former hand-written poll_next did.+ Ok(Box::pin(ObservedStream::new(+ Box::pin(RecordBatchStreamAdapter::new(schema, stream)),+ baseline_metrics,+ None,+ )))+ }++ /// Start (resume) the `join_time` clock.+ fn start_join_time(&mut self) {+ debug_assert!(self.join_time_start.is_none(), "join_time already running");+ self.join_time_start = Some(Instant::now());+ }++ /// Stop (pause) the `join_time` clock, accumulating the elapsed span.+ ///+ /// Called around awaits whose duration is not the join's own work: the+ /// child input streams' `next()` and `emitter.emit()` (where the+ /// consumer processes the batch). The join's own spill read-back is NOT+ /// excluded — that time is join work.+ fn stop_join_time(&mut self) {+ if let Some(start) = self.join_time_start.take() {+ self.join_time.add_elapsed(start);+ }}/// Resize the memory reservation to match current tracked usage.@@ -475,23 +466,24 @@ impl BitwiseSortMergeJoinStream {fn clear_inner_key_group(&mut self) {self.inner_key_buffer.clear();self.inner_key_spill = None;- self.spill_stream = None;- self.spill_stream_has_data = false;- self.inner_group_buffered = false;self.inner_buffer_size = 0;}- /// Poll for the next outer batch. Returns true if a batch was loaded.- fn poll_next_outer_batch(&mut self, cx: &mut Context<'_>) -> Poll<Result<bool>> {+ /// Fetch the next outer batch. Returns true if a batch was loaded.+ async fn next_outer_batch(&mut self) -> Result<bool> {loop {- match ready!(self.outer.poll_next_unpin(cx)) {+ // The child's execution time is its own, not join_time.+ self.stop_join_time();+ let item = self.outer.next().await;+ self.start_join_time();+ match item {None => {// Release the outer input pipeline's resources.let outer_schema = self.outer.schema();self.outer = Box::pin(EmptyRecordBatchStream::new(outer_schema));- return Poll::Ready(Ok(false));+ return Ok(false);}- Some(Err(e)) => return Poll::Ready(Err(e)),+ Some(Err(e)) => return Err(e),Some(Ok(batch)) => {let batch_num_rows = batch.num_rows();self.input_batches.add(1);@@ -505,26 +497,29 @@ impl BitwiseSortMergeJoinStream {self.outer_key_arrays = keys;self.outer_inner_cmp = None;self.outer_self_cmp = None;- self.batch_emitted = false;self.matched = BooleanBufferBuilder::new(batch_num_rows);self.matched.append_n(batch_num_rows, false);- return Poll::Ready(Ok(true));+ return Ok(true);}}}}- /// Poll for the next inner batch. Returns true if a batch was loaded.- fn poll_next_inner_batch(&mut self, cx: &mut Context<'_>) -> Poll<Result<bool>> {+ /// Fetch the next inner batch. Returns true if a batch was loaded.+ async fn next_inner_batch(&mut self) -> Result<bool> {loop {- match ready!(self.inner.poll_next_unpin(cx)) {+ // The child's execution time is its own, not join_time.+ self.stop_join_time();+ let item = self.inner.next().await;+ self.start_join_time();+ match item {None => {// Release the inner input pipeline's resources.let inner_schema = self.inner.schema();self.inner = Box::pin(EmptyRecordBatchStream::new(inner_schema));- return Poll::Ready(Ok(false));+ return Ok(false);}- Some(Err(e)) => return Poll::Ready(Err(e)),+ Some(Err(e)) => return Err(e),Some(Ok(batch)) => {let batch_num_rows = batch.num_rows();self.input_batches.add(1);@@ -538,22 +533,17 @@ impl BitwiseSortMergeJoinStream {self.inner_key_arrays = keys;self.outer_inner_cmp = None;self.inner_self_cmp = None;- return Poll::Ready(Ok(true));+ return Ok(true);}}}}- /// Emit the current outer batch through the coalescer, applying the- /// matched bitset as a selection mask. No-op if already emitted- /// (see `batch_emitted` field).+ /// Push the current outer batch into the coalescer, applying the matched+ /// bitset as a selection mask. Consumes the batch (`outer_batch` becomes+ /// `None`).fn emit_outer_batch(&mut self) -> Result<()> {- if self.batch_emitted {- return Ok(());- }- self.batch_emitted = true;-- let batch = self.outer_batch.as_ref().unwrap();+ let batch = self.outer_batch.take().unwrap();// finish() converts the bit-packed builder directly to a// BooleanBuffer — no iteration or repacking needed.@@ -576,14 +566,14 @@ impl BitwiseSortMergeJoinStream {}JoinType::LeftSemi | JoinType::RightSemi => {let selection = BooleanArray::new(matched_buf, None);- let filtered = filter_record_batch(batch, &selection)?;+ let filtered = filter_record_batch(&batch, &selection)?;if filtered.num_rows() > 0 {self.coalescer.push_batch(filtered)?;}}JoinType::LeftAnti | JoinType::RightAnti => {let selection = not(&BooleanArray::new(matched_buf, None))?;- let filtered = filter_record_batch(batch, &selection)?;+ let filtered = filter_record_batch(&batch, &selection)?;if filtered.num_rows() > 0 {self.coalescer.push_batch(filtered)?;}@@ -593,165 +583,114 @@ impl BitwiseSortMergeJoinStream {Ok(())}- /// Process a key match between outer and inner sides (no filter).- /// Sets matched bits for all outer rows sharing the current key.- fn process_key_match_no_filter(&mut self) -> Result<()> {- let outer_batch = self.outer_batch.as_ref().unwrap();- let num_outer = outer_batch.num_rows();+ /// Mark all outer rows in the current key group as matched and advance+ /// the outer cursor past the group (within the current batch).+ fn mark_outer_key_group_matched(&mut self) -> Result<()> {+ let num_outer = self.outer_batch.as_ref().unwrap().num_rows();+ let from = self.outer_offset;+ let group_end = find_key_group_end(self.get_outer_self_cmp()?, from, num_outer);- self.get_outer_self_cmp()?;- let outer_group_end = find_key_group_end(- self.outer_self_cmp.as_ref().unwrap(),- self.outer_offset,- num_outer,- );-- for i in self.outer_offset..outer_group_end {+ for i in from..group_end {self.matched.set_bit(i, true);}- self.outer_offset = outer_group_end;+ self.outer_offset = group_end;Ok(())}- /// Advance inner past the current key group. Returns Ok(true) if inner+ /// Advance the inner cursor past the current key group. The group may+ /// span multiple inner batches. Sets `inner_batch` to `None` if inner/// is exhausted.- fn advance_inner_past_key_group(- &mut self,- cx: &mut Context<'_>,- ) -> Poll<Result<bool>> {+ async fn advance_inner_past_key_group(&mut self) -> Result<()> {loop {- let inner_batch = match &self.inner_batch {- Some(b) => b,- None => return Poll::Ready(Ok(true)),+ let Some(inner_batch) = &self.inner_batch else {+ return Ok(());};let num_inner = inner_batch.num_rows();-- self.get_inner_self_cmp()?;- let group_end = find_key_group_end(- self.inner_self_cmp.as_ref().unwrap(),- self.inner_offset,- num_inner,- );+ let from = self.inner_offset;+ let group_end =+ find_key_group_end(self.get_inner_self_cmp()?, from, num_inner);if group_end < num_inner {self.inner_offset = group_end;- return Poll::Ready(Ok(false));+ return Ok(());}- // Key group extends to end of batch — need to check next batch+ // Key group extends to the end of the batch — it may continue+ // into the next one; save the last key so we can check.let saved_inner_keys = slice_keys(&self.inner_key_arrays, num_inner - 1);- match ready!(self.poll_next_inner_batch(cx)) {- Err(e) => return Poll::Ready(Err(e)),- Ok(false) => {- return Poll::Ready(Ok(true));- }- Ok(true) => {- if keys_match(- &saved_inner_keys,- &self.inner_key_arrays,- &self.sort_options,- self.null_equality,- )? {- continue;- } else {- return Poll::Ready(Ok(false));- }- }+ if !self.next_inner_batch().await? {+ self.inner_batch = None;+ return Ok(());+ }+ if !keys_match(+ &saved_inner_keys,+ &self.inner_key_arrays,+ &self.sort_options,+ self.null_equality,+ )? {+ return Ok(());}}}- /// Buffer inner key group for filter evaluation. Collects all inner rows- /// with the current key across batch boundaries.- ///- /// If poll_next_inner_batch returns Pending, we save progress via- /// buffering_inner_pending. On re-entry (from the Equal branch in- /// poll_join), we skip clear() and the slice+push for the current- /// batch (which was already buffered before Pending), and go directly- /// to polling for the next inner batch.- fn buffer_inner_key_group(&mut self, cx: &mut Context<'_>) -> Poll<Result<bool>> {- // On re-entry after Pending: don't clear the partially-filled- // buffer. The current inner_batch was already sliced and pushed- // before Pending, so jump to polling for the next batch.- let mut resume_from_poll = false;- if self.buffering_inner_pending {- self.buffering_inner_pending = false;- resume_from_poll = true;- } else {- self.clear_inner_key_group();- }+ /// Buffer the inner key group for filter evaluation, advancing the inner+ /// cursor past the group. Collects all inner rows with the current key+ /// across batch boundaries. Sets `inner_batch` to `None` if inner is+ /// exhausted.+ async fn buffer_inner_key_group(&mut self) -> Result<()> {+ self.clear_inner_key_group();loop {- if self.inner_batch.is_none() {- return Poll::Ready(Ok(true));- }- let num_inner = self.inner_batch.as_ref().unwrap().num_rows();- self.get_inner_self_cmp()?;- let group_end = find_key_group_end(- self.inner_self_cmp.as_ref().unwrap(),- self.inner_offset,- num_inner,- );-- if !resume_from_poll {- let inner_batch = self.inner_batch.as_ref().unwrap();- let slice =- inner_batch.slice(self.inner_offset, group_end - self.inner_offset);- self.inner_buffer_size += slice.get_array_memory_size();- self.inner_key_buffer.push(slice);-- // Reserve memory for the newly buffered slice. If the pool- // is exhausted, spill the entire buffer to disk.- if self.try_resize_reservation().is_err() {- if self.runtime_env.disk_manager.tmp_files_enabled() {- self.spill_inner_key_buffer()?;- } else {- // Re-attempt to get the error message- self.try_resize_reservation().map_err(|e| {- datafusion_common::DataFusionError::Execution(format!(- "{e}. Disk spilling disabled."- ))- })?;- }+ let Some(inner_batch) = &self.inner_batch else {+ return Ok(());+ };+ let num_inner = inner_batch.num_rows();+ let from = self.inner_offset;+ let group_end =+ find_key_group_end(self.get_inner_self_cmp()?, from, num_inner);++ let inner_batch = self.inner_batch.as_ref().unwrap();+ let slice = inner_batch.slice(from, group_end - from);+ self.inner_buffer_size += slice.get_array_memory_size();+ self.inner_key_buffer.push(slice);++ // Reserve memory for the newly buffered slice. If the pool+ // is exhausted, spill the entire buffer to disk.+ if self.try_resize_reservation().is_err() {+ if self.runtime_env.disk_manager.tmp_files_enabled() {+ self.spill_inner_key_buffer()?;+ } else {+ // Re-attempt to get the error message+ self.try_resize_reservation().map_err(|e| {+ DataFusionError::Execution(format!(+ "{e}. Disk spilling disabled."+ ))+ })?;}+ }- if group_end < num_inner {- self.inner_offset = group_end;- return Poll::Ready(Ok(false));- }+ if group_end < num_inner {+ self.inner_offset = group_end;+ return Ok(());}- resume_from_poll = false;- // Key group extends to end of batch — check next+ // Key group extends to the end of the batch — it may continue+ // into the next one; save the last key so we can check.let saved_inner_keys = slice_keys(&self.inner_key_arrays, num_inner - 1);- // If poll returns Pending, the current batch is already- // in inner_key_buffer.- self.buffering_inner_pending = true;- match ready!(self.poll_next_inner_batch(cx)) {- Err(e) => {- self.buffering_inner_pending = false;- return Poll::Ready(Err(e));- }- Ok(false) => {- self.buffering_inner_pending = false;- return Poll::Ready(Ok(true));- }- Ok(true) => {- self.buffering_inner_pending = false;- if keys_match(- &saved_inner_keys,- &self.inner_key_arrays,- &self.sort_options,- self.null_equality,- )? {- continue;- } else {- return Poll::Ready(Ok(false));- }- }+ if !self.next_inner_batch().await? {+ self.inner_batch = None;+ return Ok(());+ }+ if !keys_match(+ &saved_inner_keys,+ &self.inner_key_arrays,+ &self.sort_options,+ self.null_equality,+ )? {+ return Ok(());}}}@@ -759,14 +698,8 @@ impl BitwiseSortMergeJoinStream {/// Process a key match with a filter. For each inner row in the buffered/// key group, evaluates the filter against the outer key group and ORs/// the results into the matched bitset using u64-chunked bitwise ops.- fn process_key_match_with_filter(- &mut self,- cx: &mut Context<'_>,- ) -> Poll<Result<()>> {- self.get_outer_self_cmp()?;- let filter = self.filter.as_ref().unwrap();- let outer_batch = self.outer_batch.as_ref().unwrap();- let num_outer = outer_batch.num_rows();+ async fn process_key_match_with_filter(&mut self) -> Result<()> {+ let num_outer = self.outer_batch.as_ref().unwrap().num_rows();// buffer_inner_key_group must be called before this functiondebug_assert!(@@ -782,60 +715,54 @@ impl BitwiseSortMergeJoinStream {"matched vector must be sized for the current outer batch");- let outer_group_end = find_key_group_end(- self.outer_self_cmp.as_ref().unwrap(),- self.outer_offset,- num_outer,- );- let outer_group_len = outer_group_end - self.outer_offset;- let outer_slice = outer_batch.slice(self.outer_offset, outer_group_len);+ let outer_group_start = self.outer_offset;+ let outer_group_end =+ find_key_group_end(self.get_outer_self_cmp()?, outer_group_start, num_outer);+ let outer_group_len = outer_group_end - outer_group_start;++ let filter = self.filter.as_ref().unwrap();+ let outer_batch = self.outer_batch.as_ref().unwrap();+ let outer_slice = outer_batch.slice(outer_group_start, outer_group_len);// Count already-matched bits using popcnt on u64 chunks (zero-copy).let mut matched_count = UnalignedBitChunk::new(self.matched.as_slice(),- self.outer_offset,+ outer_group_start,outer_group_len,).count_ones();// Process spilled inner batches first asynchronously.if matched_count < outer_group_len- && (self.inner_key_spill.is_some() || self.spill_stream.is_some())+ && let Some(spill_file) = &self.inner_key_spill{- if self.spill_stream.is_none()- && let Some(spill_file) = &self.inner_key_spill- {- let stream = self- .spill_manager- .read_spill_as_stream(Arc::clone(spill_file), None)?;- self.spill_stream = Some(stream);- }-+ let mut spill_stream = self+ .spill_manager+ .read_spill_as_stream(Arc::clone(spill_file), None)?;+ let mut spill_stream_has_data = false;++ // Note: the clock keeps running across the spill reads — the+ // spill file is the join's own data, so reading it back is+ // join work (unlike the child inputs' `next()`).while matched_count < outer_group_len {- let stream = self.spill_stream.as_mut().unwrap();- match ready!(stream.poll_next_unpin(cx)) {+ match spill_stream.next().await {Some(Ok(inner_slice)) => {- self.spill_stream_has_data = true;+ spill_stream_has_data = true;matched_count = eval_filter_for_inner_slice(self.outer_is_left,filter,&outer_slice,&inner_slice,&mut self.matched,- self.outer_offset,+ outer_group_start,outer_group_len,matched_count,)?;}- Some(Err(e)) => {- self.spill_stream = None;- self.spill_stream_has_data = false;- return Poll::Ready(Err(e));- }+ Some(Err(e)) => return Err(e),None => {- self.spill_stream = None;- if !self.spill_stream_has_data {- return Poll::Ready(internal_err!("Spill file was empty"));+ if !spill_stream_has_data {+ return internal_err!("Spill file was empty");}break;}@@ -855,7 +782,7 @@ impl BitwiseSortMergeJoinStream {&outer_slice,inner_slice,&mut self.matched,- self.outer_offset,+ outer_group_start,outer_group_len,matched_count,)?;@@ -867,357 +794,291 @@ impl BitwiseSortMergeJoinStream {self.outer_offset = outer_group_end;- self.spill_stream = None;- self.spill_stream_has_data = false;-- Poll::Ready(Ok(()))+ Ok(())}- /// Continue processing an outer key group that spans multiple outer- /// batches. Returns `true` if this outer batch was fully consumed- /// by the key group and the caller should load another.- fn resume_boundary(&mut self, cx: &mut Context<'_>) -> Poll<Result<bool>> {- debug_assert!(- self.outer_batch.is_some(),- "caller must load outer_batch first"- );- match self.pending_boundary.take() {- Some(PendingBoundary::NoFilter { saved_keys }) => {- let same_key = keys_match(- &saved_keys,- &self.outer_key_arrays,- &self.sort_options,- self.null_equality,- )?;- if same_key {- self.process_key_match_no_filter()?;- let num_outer = self.outer_batch.as_ref().unwrap().num_rows();- if self.outer_offset >= num_outer {- self.pending_boundary = Some(PendingBoundary::NoFilter {- saved_keys: slice_keys(&self.outer_key_arrays, num_outer - 1),- });- self.emit_outer_batch()?;- self.outer_batch = None;- return Poll::Ready(Ok(true));- }- }+ /// Evaluate the filter for the buffered inner key group against the+ /// outer key group. If the outer key group continues into subsequent+ /// outer batches, keep evaluating there too.+ async fn process_filtered_match_loop(&mut self) -> Result<()> {+ loop {+ self.process_key_match_with_filter().await?;++ let outer_batch = self.outer_batch.as_ref().unwrap();+ if self.outer_offset < outer_batch.num_rows() {+ break;}- Some(PendingBoundary::Filtered { saved_keys }) => {- debug_assert!(- !self.inner_key_buffer.is_empty() || self.inner_key_spill.is_some(),- "Filtered pending boundary entered but no inner key data exists"- );- let same_key = keys_match(- &saved_keys,- &self.outer_key_arrays,- &self.sort_options,- self.null_equality,- )?;- if same_key {- match self.process_key_match_with_filter(cx) {- Poll::Ready(Ok(())) => (),- Poll::Ready(Err(e)) => return Poll::Ready(Err(e)),- Poll::Pending => {- self.pending_boundary =- Some(PendingBoundary::Filtered { saved_keys });- return Poll::Pending;- }- }- let num_outer = self.outer_batch.as_ref().unwrap().num_rows();- if self.outer_offset >= num_outer {- self.pending_boundary = Some(PendingBoundary::Filtered {- saved_keys: slice_keys(&self.outer_key_arrays, num_outer - 1),- });- self.emit_outer_batch()?;- self.outer_batch = None;- return Poll::Ready(Ok(true));- }- }- self.clear_inner_key_group();++ // The outer key group may continue into the next outer batch;+ // save the last key so we can check.+ let saved_keys =+ slice_keys(&self.outer_key_arrays, outer_batch.num_rows() - 1);++ self.emit_outer_batch()?;++ if !self.next_outer_batch().await? {+ break;+ }+ if !keys_match(+ &saved_keys,+ &self.outer_key_arrays,+ &self.sort_options,+ self.null_equality,+ )? {+ break;}- None => {}}- Poll::Ready(Ok(false))++ self.clear_inner_key_group();+ Ok(())}- /// Helper to process an Equal match across potential outer batch boundaries.- fn process_filtered_match_loop(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {+ /// Mark the outer key group as matched. If the outer key group continues+ /// into subsequent outer batches, keep marking there too.+ async fn process_unfiltered_match_loop(&mut self) -> Result<()> {loop {- ready!(self.process_key_match_with_filter(cx))?;+ self.mark_outer_key_group_matched()?;let outer_batch = self.outer_batch.as_ref().unwrap();- if self.outer_offset >= outer_batch.num_rows() {- let saved_keys =- slice_keys(&self.outer_key_arrays, outer_batch.num_rows() - 1);+ if self.outer_offset < outer_batch.num_rows() {+ return Ok(());+ }- self.emit_outer_batch()?;- self.pending_boundary = Some(PendingBoundary::Filtered { saved_keys });+ // The outer key group may continue into the next outer batch;+ // save the last key so we can check.+ let saved_keys =+ slice_keys(&self.outer_key_arrays, outer_batch.num_rows() - 1);- // Clear stale batch before polling- self.outer_batch = None;+ self.emit_outer_batch()?;- match ready!(self.poll_next_outer_batch(cx)) {- Err(e) => return Poll::Ready(Err(e)),- Ok(false) => {- self.pending_boundary = None;- break;- }- Ok(true) => {- let Some(PendingBoundary::Filtered { saved_keys }) =- self.pending_boundary.take()- else {- unreachable!()- };- let same = keys_match(- &saved_keys,- &self.outer_key_arrays,- &self.sort_options,- self.null_equality,- )?;- if same {- continue;- }- break;- }- }- } else {- break;+ if !self.next_outer_batch().await? {+ return Ok(());+ }+ if !keys_match(+ &saved_keys,+ &self.outer_key_arrays,+ &self.sort_options,+ self.null_equality,+ )? {+ return Ok(());}}+ }- self.clear_inner_key_group(); // This resets inner_group_buffered to false- Poll::Ready(Ok(()))+ /// Keys at both cursors are equal: determine which outer rows in the key+ /// group have a match. Both key groups may span batch boundaries.+ async fn process_key_match(&mut self) -> Result<()> {+ if self.filter.is_some() {+ // Buffer the inner key group so each inner row can be evaluated+ // against the outer key group, OR-ing filter results into the+ // matched bitset.+ self.buffer_inner_key_group().await?;+ self.process_filtered_match_loop().await+ } else {+ // Without a filter, key equality alone means every outer row in+ // the group matches; the inner rows themselves are not needed.+ self.advance_inner_past_key_group().await?;+ self.process_unfiltered_match_loop().await+ }}- /// Main loop: drive the merge-scan to produce output batches.- fn poll_join(&mut self, cx: &mut Context<'_>) -> Poll<Result<Option<RecordBatch>>> {- let join_time = self.join_time.clone();- let _timer = join_time.timer();+ /// Compare the join keys at the outer and inner cursors, returning the+ /// ordering of the outer key relative to the inner key (e.g. `Greater`+ /// means outer key > inner key, per the sort options).+ fn compare_current_keys(&mut self) -> Result<Ordering> {+ let (outer_idx, inner_idx) = (self.outer_offset, self.inner_offset);+ Ok(self.get_outer_inner_cmp()?.compare(outer_idx, inner_idx))+ }- loop {- // 1. Ensure we have an outer batch- if self.outer_batch.is_none() {- match ready!(self.poll_next_outer_batch(cx)) {- Err(e) => return Poll::Ready(Err(e)),- Ok(false) => {- // Outer exhausted — flush coalescer- self.pending_boundary = None;- self.coalescer.finish_buffered_batch()?;- if let Some(batch) = self.coalescer.next_completed_batch() {- return Poll::Ready(Ok(Some(batch)));- }- return Poll::Ready(Ok(None));- }- Ok(true) => {} // Loaded batch, move on to checks- }- }+ /// Outer key is unmatched: advance the outer cursor past its key group+ /// (within the current batch). If the group continues into the next+ /// batch, those rows compare Less again and are skipped the same way.+ fn skip_outer_key_group(&mut self) -> Result<()> {+ let num_outer = self.outer_batch.as_ref().unwrap().num_rows();+ let from = self.outer_offset;+ self.outer_offset =+ find_key_group_end(self.get_outer_self_cmp()?, from, num_outer);+ Ok(())+ }- // Handles pausing while fetching a NEW outer batch.- if self.pending_boundary.is_some() && ready!(self.resume_boundary(cx))? {- continue;- }+ /// Sync fast path for `Ordering::Greater`: skip the inner key group when+ /// it ends within the current batch. Returns false — leaving all state+ /// unchanged — when the group reaches the batch boundary, in which case+ /// the caller must take [`Self::advance_inner_past_key_group`].+ fn try_skip_inner_key_group(&mut self) -> Result<bool> {+ let num_inner = self.inner_batch.as_ref().unwrap().num_rows();+ let from = self.inner_offset;+ let group_end = find_key_group_end(self.get_inner_self_cmp()?, from, num_inner);+ if group_end >= num_inner {+ return Ok(false);+ }+ self.inner_offset = group_end;+ Ok(true)+ }- // Handles pausing while reading the disk stream mid-batch.- if self.inner_group_buffered {- ready!(self.process_filtered_match_loop(cx))?;- continue;- }+ /// Sync fast path for `Ordering::Equal` without a filter: when both key+ /// groups end within their current batches (the common case — a group+ /// only reaches a batch boundary once per batch), mark the outer group+ /// matched and advance both cursors without any async machinery.+ /// Returns false — leaving all state unchanged — when a filter is+ /// present or either group reaches a batch boundary, in which case the+ /// caller must take [`Self::process_key_match`].+ fn try_process_key_match(&mut self) -> Result<bool> {+ if self.filter.is_some() {+ return Ok(false);+ }- // 2. Ensure we have an inner batch (unless inner is exhausted).- // Skip this when resuming a pending boundary — inner was already- // advanced past the key group before the boundary loop started.- if self.inner_batch.is_none() && self.pending_boundary.is_none() {- match ready!(self.poll_next_inner_batch(cx)) {- Err(e) => return Poll::Ready(Err(e)),- Ok(false) => {- // Inner exhausted — emit remaining outer batches.- // For semi: no more matches possible.- // For anti: all remaining outer rows are unmatched.- self.emit_outer_batch()?;- self.outer_batch = None;-- loop {- match ready!(self.poll_next_outer_batch(cx)) {- Err(e) => return Poll::Ready(Err(e)),- Ok(false) => break,- Ok(true) => {- self.emit_outer_batch()?;- self.outer_batch = None;- }- }- }+ let num_inner = self.inner_batch.as_ref().unwrap().num_rows();+ let inner_from = self.inner_offset;+ let inner_group_end =+ find_key_group_end(self.get_inner_self_cmp()?, inner_from, num_inner);+ if inner_group_end >= num_inner {+ return Ok(false);+ }- self.coalescer.finish_buffered_batch()?;- if let Some(batch) = self.coalescer.next_completed_batch() {- return Poll::Ready(Ok(Some(batch)));- }- return Poll::Ready(Ok(None));- }- Ok(true) => {}- }- }+ let num_outer = self.outer_batch.as_ref().unwrap().num_rows();+ let outer_from = self.outer_offset;+ let outer_group_end =+ find_key_group_end(self.get_outer_self_cmp()?, outer_from, num_outer);+ if outer_group_end >= num_outer {+ return Ok(false);+ }- // 3. Main merge-scan loop- let outer_batch = self.outer_batch.as_ref().unwrap();- let num_outer = outer_batch.num_rows();+ for i in outer_from..outer_group_end {+ self.matched.set_bit(i, true);+ }+ self.outer_offset = outer_group_end;+ self.inner_offset = inner_group_end;+ Ok(true)+ }- if self.outer_offset >= num_outer {- self.emit_outer_batch()?;- self.outer_batch = None;+ /// True when the outer cursor already points at an unprocessed row: the+ /// sync fast path of [`Self::advance_outer_row`]. Checked inline in the+ /// hot loop so the async helper (and its state machine) is only entered+ /// at batch boundaries — same pattern as `sorts/merge.rs`.+ fn has_current_outer_row(&self) -> bool {+ self.outer_batch+ .as_ref()+ .is_some_and(|batch| self.outer_offset < batch.num_rows())+ }- if let Some(batch) = self.coalescer.next_completed_batch() {- return Poll::Ready(Ok(Some(batch)));- }- continue;- }+ /// True when the inner cursor already points at an unprocessed row: the+ /// sync fast path of [`Self::advance_inner_row`].+ fn has_current_inner_row(&self) -> bool {+ self.inner_batch+ .as_ref()+ .is_some_and(|batch| self.inner_offset < batch.num_rows())+ }- let inner_batch = match &self.inner_batch {- Some(b) => b,- None => {+ /// Ensure the outer cursor points at an unprocessed row, emitting+ /// finished outer batches and loading new ones as needed. Returns false+ /// when outer is exhausted.+ async fn advance_outer_row(+ &mut self,+ emitter: &mut TryEmitter<RecordBatch, DataFusionError>,+ ) -> Result<bool> {+ loop {+ match &self.outer_batch {+ Some(batch) if self.outer_offset < batch.num_rows() => {+ return Ok(true);+ }+ Some(_) => {+ // Current batch fully scanned — emit it and load the next.self.emit_outer_batch()?;- self.outer_batch = None;- continue;+ self.emit_completed_batches(emitter).await;}- };- let num_inner = inner_batch.num_rows();-- if self.inner_offset >= num_inner {- match ready!(self.poll_next_inner_batch(cx)) {- Err(e) => return Poll::Ready(Err(e)),- Ok(false) => {- self.inner_batch = None;- continue;+ None => {+ if !self.next_outer_batch().await? {+ return Ok(false);}- Ok(true) => continue,}}+ }+ }- // 4. Compare keys at current positions- self.get_outer_inner_cmp()?;- let cmp = self- .outer_inner_cmp- .as_ref()- .unwrap()- .compare(self.outer_offset, self.inner_offset);-- match cmp {- Ordering::Less => {- self.get_outer_self_cmp()?;- let group_end = find_key_group_end(- self.outer_self_cmp.as_ref().unwrap(),- self.outer_offset,- num_outer,- );- self.outer_offset = group_end;- }+ /// Ensure the inner cursor points at an unprocessed row, loading new+ /// inner batches as needed. Returns false when inner is exhausted.+ async fn advance_inner_row(&mut self) -> Result<bool> {+ loop {+ if let Some(batch) = &self.inner_batch+ && self.inner_offset < batch.num_rows()+ {+ return Ok(true);+ }+ if !self.next_inner_batch().await? {+ self.inner_batch = None;+ return Ok(false);+ }+ }+ }++ /// Inner is exhausted, so no further matches are possible: emit the+ /// current outer batch and all remaining ones with their current matched+ /// bits (semi drops unmatched rows, anti emits them, mark emits them+ /// with mark=false).+ async fn drain_outer(&mut self) -> Result<()> {+ self.emit_outer_batch()?;+ while self.next_outer_batch().await? {+ self.emit_outer_batch()?;+ }+ Ok(())+ }++ /// Emit all completed coalescer batches to the stream consumer.+ async fn emit_completed_batches(+ &mut self,+ emitter: &mut TryEmitter<RecordBatch, DataFusionError>,+ ) {+ while let Some(batch) = self.coalescer.next_completed_batch() {+ // While the emitted batch is in the consumer's hands the join+ // isn't doing any work.+ self.stop_join_time();+ emitter.emit(batch).await;+ self.start_join_time();+ }+ }++ /// Main loop: a classic merge-scan over the two sorted inputs, emitting+ /// output batches as they complete.+ async fn join(+ &mut self,+ emitter: &mut TryEmitter<RecordBatch, DataFusionError>,+ ) -> Result<()> {+ // The `has_current_*` / `has_completed_batch` fast paths keep async+ // state machinery out of the per-key-group hot path; the awaiting+ // helpers are only entered at batch boundaries.+ while self.has_current_outer_row() || self.advance_outer_row(emitter).await? {+ if !(self.has_current_inner_row() || self.advance_inner_row().await?) {+ self.drain_outer().await?;+ break;+ }++ // Each arm handles the common case synchronously (`try_*`); the+ // async continuations only run when a key group reaches a batch+ // boundary or a filter must be evaluated.+ match self.compare_current_keys()? {+ Ordering::Less => self.skip_outer_key_group()?,Ordering::Greater => {- self.get_inner_self_cmp()?;- let group_end = find_key_group_end(- self.inner_self_cmp.as_ref().unwrap(),- self.inner_offset,- num_inner,- );- if group_end >= num_inner {- let saved_keys =- slice_keys(&self.inner_key_arrays, num_inner - 1);- match ready!(self.poll_next_inner_batch(cx)) {- Err(e) => return Poll::Ready(Err(e)),- Ok(false) => {- self.inner_batch = None;- continue;- }- Ok(true) => {- if keys_match(- &saved_keys,- &self.inner_key_arrays,- &self.sort_options,- self.null_equality,- )? {- match ready!(self.advance_inner_past_key_group(cx)) {- Err(e) => return Poll::Ready(Err(e)),- Ok(_) => continue,- }- }- continue;- }- }- } else {- self.inner_offset = group_end;+ if !self.try_skip_inner_key_group()? {+ self.advance_inner_past_key_group().await?;}}Ordering::Equal => {- if self.filter.is_some() {- debug_assert!(!self.inner_group_buffered);- // Buffer inner key group (may span batches)- match ready!(self.buffer_inner_key_group(cx)) {- Err(e) => return Poll::Ready(Err(e)),- Ok(_inner_exhausted) => {- self.inner_group_buffered = true;- }- }- // Process outer rows against buffered inner group- // (may need to handle outer batch boundary)- ready!(self.process_filtered_match_loop(cx))?;- } else {- // No filter: advance inner past key group, then- // mark all outer rows with this key as matched.- match ready!(self.advance_inner_past_key_group(cx)) {- Err(e) => return Poll::Ready(Err(e)),- Ok(_inner_exhausted) => {}- }-- loop {- self.process_key_match_no_filter()?;-- let num_outer = self.outer_batch.as_ref().unwrap().num_rows();- if self.outer_offset >= num_outer {- let saved_keys =- slice_keys(&self.outer_key_arrays, num_outer - 1);-- self.emit_outer_batch()?;- self.pending_boundary =- Some(PendingBoundary::NoFilter { saved_keys });- // Clear stale batch before polling- self.outer_batch = None;-- match ready!(self.poll_next_outer_batch(cx)) {- Err(e) => return Poll::Ready(Err(e)),- Ok(false) => {- self.pending_boundary = None;- break;- }- Ok(true) => {- let Some(PendingBoundary::NoFilter {- saved_keys,- }) = self.pending_boundary.take()- else {- unreachable!()- };- let same_key = keys_match(- &saved_keys,- &self.outer_key_arrays,- &self.sort_options,- self.null_equality,- )?;- if same_key {- continue;- }- break;- }- }- } else {- break;- }- }+ if !self.try_process_key_match()? {+ self.process_key_match().await?;}}}- // Check for completed coalescer batch- if let Some(batch) = self.coalescer.next_completed_batch() {- return Poll::Ready(Ok(Some(batch)));+ if self.coalescer.has_completed_batch() {+ self.emit_completed_batches(emitter).await;}}++ // Flush whatever is still buffered in the coalescer.+ self.coalescer.finish_buffered_batch()?;+ self.emit_completed_batches(emitter).await;+ Ok(())}}@@ -1370,7 +1231,7 @@ fn evaluate_filter_for_inner_row(.as_any().downcast_ref::<BooleanArray>().ok_or_else(|| {- datafusion_common::DataFusionError::Internal(+ DataFusionError::Internal("Filter expression did not return BooleanArray".to_string(),)})?;@@ -1381,21 +1242,3 @@ fn evaluate_filter_for_inner_row(Ok(bool_arr.clone())}}--impl Stream for BitwiseSortMergeJoinStream {- type Item = Result<RecordBatch>;-- fn poll_next(- mut self: Pin<&mut Self>,- cx: &mut Context<'_>,- ) -> Poll<Option<Self::Item>> {- let poll = self.poll_join(cx).map(|result| result.transpose());- self.baseline_metrics.record_poll(poll)- }-}--impl RecordBatchStream for BitwiseSortMergeJoinStream {- fn schema(&self) -> SchemaRef {- Arc::clone(&self.schema)- }-}diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rsindex 00cac069aa..3b597323b2 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs@@ -528,7 +528,7 @@ impl ExecutionPlan for SortMergeJoinExec {| JoinType::LeftMark| JoinType::RightMark) {- Ok(Box::pin(BitwiseSortMergeJoinStream::try_new(+ BitwiseSortMergeJoinStream::try_new(Arc::clone(&self.schema),self.sort_options.clone(),self.null_equality,@@ -544,7 +544,7 @@ impl ExecutionPlan for SortMergeJoinExec {reservation,spill_manager,context.runtime_env(),- )?))+ )} else {Ok(Box::pin(MaterializingSortMergeJoinStream::try_new(Arc::clone(&self.schema),diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rsindex 64dadcb123..313818cf6c 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs@@ -27,6 +27,7 @@use std::pin::Pin;use std::sync::Arc;use std::task::{Context, Poll};+use std::time::Duration;use super::bitwise_stream::BitwiseSortMergeJoinStream;use crate::joins::utils::{ColumnIndex, JoinFilter, JoinOn};@@ -51,6 +52,7 @@ use arrow_ord::sort::SortColumn;use arrow_schema::SchemaRef;use bytes::Bytes;use datafusion_common::JoinType::*;+use datafusion_common::instant::Instant;use datafusion_common::{JoinSide, internal_err,test_util::{batches_to_sort_string, batches_to_string},@@ -3814,7 +3816,7 @@ async fn consume_stream_until_finish_barrier_reached(let mut after_finish_barrier_reached = vec![];let mut background_task = JoinSet::new();- let mut start_time_since_last_ready = datafusion_common::instant::Instant::now();+ let mut start_time_since_last_ready = Instant::now();loop {let next_item = output_stream.next();@@ -3834,7 +3836,7 @@ async fn consume_stream_until_finish_barrier_reached(} else {output_batched.push(batch);}- start_time_since_last_ready = datafusion_common::instant::Instant::now();+ start_time_since_last_ready = Instant::now();}Poll::Ready(Some(Err(e))) => return Err(e),Poll::Ready(None) if !switch_to_finish_barrier => {@@ -3861,9 +3863,7 @@ async fn consume_stream_until_finish_barrier_reached(}// Make sure the test doesn't run forever- if start_time_since_last_ready.elapsed()- > std::time::Duration::from_secs(5)- {+ if start_time_since_last_ready.elapsed() > Duration::from_secs(5) {return internal_err!("Stream should have emitted data by now, but it's still pending. Output batches so far: {}",output_batched.len()@@ -4032,7 +4032,7 @@ fn columns(schema: &Schema) -> Vec<String> {// ==================== BitwiseSortMergeJoinStream direct tests ====================//// These tests construct a BitwiseSortMergeJoinStream directly (bypassing exec)-// to exercise async re-entry and spill edge cases using PendingStream.+// to exercise waiting on inputs and spill edge cases using PendingStream./// Create test memory/spill resources for stream-level tests.fn test_stream_resources(@@ -4112,18 +4112,212 @@ impl RecordBatchStream for PendingStream {}/// Helper: collect all output from a BitwiseSortMergeJoinStream.-async fn collect_stream(stream: BitwiseSortMergeJoinStream) -> Result<Vec<RecordBatch>> {- common::collect(Box::pin(stream)).await+async fn collect_stream(stream: SendableRecordBatchStream) -> Result<Vec<RecordBatch>> {+ common::collect(stream).await+}++// ==================== join_time metric tests ====================+//+// These verify that `join_time` measures only the join's own work: waiting+// for either child input or for the consumer to take an emitted batch must+// not be counted.++/// Stream that sleeps `delay` before yielding each batch, to simulate a+/// slow input.+fn delayed_stream(+ batches: Vec<RecordBatch>,+ delay: Duration,+) -> SendableRecordBatchStream {+ let schema = batches[0].schema();+ Box::pin(crate::stream::RecordBatchStreamAdapter::new(+ schema,+ futures::stream::iter(batches.into_iter().map(Ok)).then(move |item| async move {+ tokio::time::sleep(delay).await;+ item+ }),+ ))+}++/// Three 2-row batches with unique matching keys.+fn join_time_batches() -> Vec<RecordBatch> {+ vec![+ build_table_i32(+ ("a1", &vec![0, 1]),+ ("b1", &vec![1, 2]),+ ("c1", &vec![7, 8]),+ ),+ build_table_i32(+ ("a1", &vec![2, 3]),+ ("b1", &vec![3, 4]),+ ("c1", &vec![7, 8]),+ ),+ build_table_i32(+ ("a1", &vec![4, 5]),+ ("b1", &vec![5, 6]),+ ("c1", &vec![7, 8]),+ ),+ ]+}++/// Build a no-filter LeftSemi bitwise stream over the given input streams.+/// The small batch size makes each outer batch surface as its own output+/// batch, so a slow consumer test sees multiple emits.+fn join_time_test_join(+ outer: SendableRecordBatchStream,+ inner: SendableRecordBatchStream,+) -> (SendableRecordBatchStream, ExecutionPlanMetricsSet) {+ let metrics = ExecutionPlanMetricsSet::new();+ let outer_schema = outer.schema();+ let (reservation, spill_manager, runtime_env) =+ test_stream_resources(inner.schema(), &metrics);+ let stream = BitwiseSortMergeJoinStream::try_new(+ outer_schema,+ vec![SortOptions::default()],+ NullEquality::NullEqualsNothing,+ outer,+ inner,+ vec![Arc::new(Column::new("b1", 1)) as PhysicalExprRef],+ vec![Arc::new(Column::new("b1", 1)) as PhysicalExprRef],+ None,+ LeftSemi,+ 2,+ 0,+ &metrics,+ reservation,+ spill_manager,+ runtime_env,+ )+ .unwrap();+ (stream, metrics)}-/// Reproduces the buffer_inner_key_group re-entry bug:+fn join_time_of(metrics: &ExecutionPlanMetricsSet) -> Duration {+ Duration::from_nanos(+ metrics+ .clone_inner()+ .sum_by_name("join_time")+ .map(|m| m.as_usize())+ .unwrap_or(0) as u64,+ )+}++/// Run a join with the given injected `delay`, retrying with 4x the delay+/// (up to 3 attempts) when `join_time < delay` fails.///-/// When buffer_inner_key_group buffers inner rows across batch boundaries-/// and poll_next_inner_batch returns Pending mid-way, the ready! macro-/// exits poll_join. On re-entry, the merge-scan reaches Equal again and-/// calls buffer_inner_key_group a second time -- which starts with-/// clear(), destroying the partially collected inner rows. Previously-/// consumed batches are gone, so re-buffering misses them.+/// This de-flakes the check without masking real bugs: a genuine exclusion+/// bug makes `join_time` absorb the injected waits, so it scales with the+/// delay and fails at every escalation level. Only a fixed-size disturbance+/// (e.g. the OS preempting the test thread while the join_time clock is+/// running) is filtered out, since it cannot grow 4x with the delay.+///+/// `run` returns `(join_time, wall)` for one join execution. Deterministic+/// invariants (row counts, wall-time lower bounds) stay as asserts inside+/// `run` — deliberately: a panic there fails the test immediately without+/// retrying, since those cannot flake and escalation would only mask a real+/// bug. Likewise `Err` from `run` (join execution failure) propagates+/// immediately. Only the preemption-sensitive `join_time` check is retried.+async fn check_join_time_excluded<F, Fut>(mut run: F) -> Result<()>+where+ F: FnMut(Duration) -> Fut,+ Fut: Future<Output = Result<(Duration, Duration)>>,+{+ let mut delay = Duration::from_millis(50);+ for attempt in 0..3 {+ let (join_time, wall) = run(delay).await?;+ if join_time < delay {+ return Ok(());+ }+ assert!(+ attempt < 2,+ "join_time ({join_time:?}) should be well below the injected \+ delay ({delay:?}) even after escalating retries; wall {wall:?}"+ );+ delay *= 4;+ }+ unreachable!()+}++/// join_time must not include time spent waiting for the outer input.+#[tokio::test]+async fn join_time_excludes_outer_input_wait() -> Result<()> {+ check_join_time_excluded(|delay| async move {+ let outer = delayed_stream(join_time_batches(), delay);+ let inner = delayed_stream(join_time_batches(), Duration::ZERO);+ let (stream, metrics) = join_time_test_join(outer, inner);++ let start = Instant::now();+ let batches = collect_stream(stream).await?;+ let wall = start.elapsed();++ let rows: usize = batches.iter().map(|b| b.num_rows()).sum();+ assert_eq!(rows, 6, "all outer rows should match");+ assert!(+ wall >= delay * 3,+ "outer delays should dominate wall time, got {wall:?}"+ );+ Ok((join_time_of(&metrics), wall))+ })+ .await+}++/// join_time must not include time spent waiting for the inner input.+#[tokio::test]+async fn join_time_excludes_inner_input_wait() -> Result<()> {+ check_join_time_excluded(|delay| async move {+ let outer = delayed_stream(join_time_batches(), Duration::ZERO);+ let inner = delayed_stream(join_time_batches(), delay);+ let (stream, metrics) = join_time_test_join(outer, inner);++ let start = Instant::now();+ let batches = collect_stream(stream).await?;+ let wall = start.elapsed();++ let rows: usize = batches.iter().map(|b| b.num_rows()).sum();+ assert_eq!(rows, 6, "all outer rows should match");+ assert!(+ wall >= delay * 3,+ "inner delays should dominate wall time, got {wall:?}"+ );+ Ok((join_time_of(&metrics), wall))+ })+ .await+}++/// join_time must not include time the consumer spends holding an emitted+/// batch (the generator is suspended inside `emitter.emit` meanwhile).+#[tokio::test]+async fn join_time_excludes_consumer_wait() -> Result<()> {+ check_join_time_excluded(|delay| async move {+ let outer = delayed_stream(join_time_batches(), Duration::ZERO);+ let inner = delayed_stream(join_time_batches(), Duration::ZERO);+ let (mut stream, metrics) = join_time_test_join(outer, inner);++ let start = Instant::now();+ let mut output_batches = 0u32;+ while let Some(batch) = stream.next().await {+ batch?;+ output_batches += 1;+ // Simulate a slow consumer between emitted batches.+ tokio::time::sleep(delay).await;+ }+ let wall = start.elapsed();++ assert!(+ output_batches >= 3,+ "expected multiple emitted batches, got {output_batches}"+ );+ assert!(+ wall >= delay * output_batches,+ "consumer delays should dominate wall time, got {wall:?}"+ );+ Ok((join_time_of(&metrics), wall))+ })+ .await+}++/// An inner key group spanning multiple inner batches must survive the inner+/// input returning Pending mid-way: inner rows delivered before the Pending+/// still take part in the filter evaluation.////// Setup:/// - Inner: 3 single-row batches, all with key=1, filter values c2=[10, 20, 30]@@ -4131,8 +4325,7 @@ async fn collect_stream(stream: BitwiseSortMergeJoinStream) -> Result<Vec<Record/// - Filter: c1 == c2 (only first inner row c2=10 matches)/// - Pending injected before 3rd inner batch///-/// Without the bug: outer row emitted (match via c2=10)-/// With the bug: outer row missing (c2=10 batch lost on re-entry)+/// Expected: outer row emitted (match via c2=10)#[tokio::test]async fn filter_buffer_pending_loses_inner_rows() -> Result<()> {let left_schema = Arc::new(Schema::new(vec![@@ -4249,22 +4442,17 @@ async fn filter_buffer_pending_loses_inner_rows() -> Result<()> {Ok(())}-/// Reproduces the no-filter boundary Pending re-entry bug:-///-/// When an outer key group spans a batch boundary, the no-filter path-/// emits the current batch, then polls for the next outer batch. If-/// poll returns Pending, poll_join exits. On re-entry, without the-/// PendingBoundary fix, the new batch is processed fresh by the-/// merge-scan. Since inner already advanced past this key, the outer-/// rows with the matching key are skipped via Ordering::Less.+/// A matched outer key group spanning a batch boundary must survive the outer+/// input returning Pending at that boundary: the rows continuing the key group+/// still count as matched, even though the inner side has already advanced+/// past the key.////// Setup:/// - Outer: 2 single-row batches, both with key=1 (key group spans boundary)/// - Inner: 1 row with key=1/// - Pending injected on outer before 2nd batch///-/// Without fix: only first outer row emitted (second lost on re-entry)-/// With fix: both outer rows emitted+/// Expected: both outer rows emitted#[tokio::test]async fn no_filter_boundary_pending_loses_outer_rows() -> Result<()> {let left_schema = Arc::new(Schema::new(vec![@@ -4353,9 +4541,8 @@ async fn no_filter_boundary_pending_loses_outer_rows() -> Result<()> {////// The outer input has an unmatched prefix row followed by a matching key/// group that continues in the next batch. Both rows with key=1 should be-/// treated as matched. Returning `Pending` before the second batch forces-/// `poll_join` to return and later resume from its top-level state, rather-/// than continuing the same in-progress boundary loop.+/// treated as matched. Returning `Pending` before the second batch makes the+/// join wait for the continuation while the key group is still open.#[tokio::test]async fn no_filter_boundary_pending_with_unmatched_prefix() -> Result<()> {let left_schema = Arc::new(Schema::new(vec![@@ -4447,8 +4634,8 @@ async fn no_filter_boundary_pending_with_unmatched_prefix() -> Result<()> {Ok(())}-/// Tests the filtered boundary Pending re-entry: outer key group spans-/// batches with a filter, and poll_next_outer_batch returns Pending.+/// Same as the no-filter boundary case, with a filter: the outer key group+/// spans batches and the outer input returns Pending at the boundary.////// Setup:/// - Outer: 2 single-row batches, both key=1, c1=[10, 20]@@ -4656,6 +4843,21 @@ async fn bitwise_spill_with_filter() -> Result<()> {metrics.spilled_rows().unwrap() > 0,"expected spilled_rows > 0 for {join_type:?}, batch_size={batch_size}");+ let join_time = metrics+ .sum_by_name("join_time")+ .map(|m| m.as_usize())+ .unwrap_or(0);+ assert!(+ join_time > 0,+ "expected join_time > 0 for {join_type:?}, batch_size={batch_size}"+ );+ let output_rows = metrics.output_rows().unwrap_or(0);+ let collected_rows: usize = spilled_result.iter().map(|b| b.num_rows()).sum();+ assert_eq!(+ output_rows, collected_rows,+ "output_rows metric should match collected rows for \+ {join_type:?}, batch_size={batch_size}"+ );// Run without spilling and compare resultslet task_ctx_no_spill = Arc::new(@@ -4690,22 +4892,19 @@ async fn bitwise_spill_with_filter() -> Result<()> {Ok(())}-/// Reproduces a bug where `resume_boundary` for the Filtered pending case-/// only checks `inner_key_buffer.is_empty()` but ignores `inner_key_spill`.-/// After spilling, the in-memory buffer is cleared while the spill file-/// holds the data. If the outer key group spans a batch boundary, the-/// second outer batch's rows are never evaluated against the inner group.+/// Once the inner key group has spilled, an outer key group spanning a batch+/// boundary must still be evaluated against the spilled inner rows — the+/// second outer batch's rows must not be treated as having no inner group to+/// match against.////// Setup:/// - Outer: 2 single-row batches, both key=1, c1=[10, 10]/// - Inner: 1 batch with many rows all key=1 (enough to trigger spill)/// - Filter: c1 == c2 (matches when c2=10)/// - Memory limit: tiny (100 bytes) to force spilling-/// - Pending before 2nd outer batch to trigger boundary re-entry+/// - Pending before 2nd outer batch, while the key group is still open////// Expected: both outer rows match (semi=2 rows, anti=0 rows)-/// Bug: second outer row is skipped because resume_boundary sees empty-/// inner_key_buffer and skips re-evaluation.#[tokio::test]async fn spill_filtered_boundary_loses_outer_rows() -> Result<()> {let left_schema = Arc::new(Schema::new(vec![@@ -5253,8 +5452,8 @@ async fn materializing_spill_pending_stream() -> Result<()> {"expected spill_count > 0 for {join_type:?}");- // Compare against a no-spill run to make sure the Pending- // re-entry path didn't corrupt or drop any data.+ // Compare against a no-spill run to make sure waiting on the+ // spill reads didn't corrupt or drop any data.let task_ctx_no_spill = Arc::new(TaskContext::default());let join_no_spill = join_with_options(Arc::clone(&left),@@ -5277,9 +5476,9 @@ async fn materializing_spill_pending_stream() -> Result<()> {}/// Bitwise-side (Semi/Anti) coverage: identical to `bitwise_spill_with_filter`,-/// but every spill read goes through `PendingSpillFile`, forcing-/// `process_key_match_with_filter`'s spilled-batch loop to actually hit and-/// resume from `Poll::Pending`.+/// but every spill read goes through `PendingSpillFile`, so reading the+/// spilled inner rows back must actually hit and recover from `Poll::Pending`+/// mid-read.#[tokio::test]async fn bitwise_spill_pending_stream() -> Result<()> {let left = build_table(