Apache DataFusion SQL Query Engine
refactor: Update SortMergeJoin to use async spill abstractions (#22230)
pantShreyfa271ce8fdb2
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rsindex ad7312426b..99aef6ed82 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs@@ -119,8 +119,6 @@//! factor than the pair-materialization approach.use std::cmp::Ordering;-use std::fs::File;-use std::io::BufReader;use std::pin::Pin;use std::sync::Arc;use std::task::{Context, Poll};@@ -134,7 +132,6 @@ use crate::{EmptyRecordBatchStream, RecordBatchStream};use arrow::array::{Array, ArrayRef, BooleanArray, BooleanBufferBuilder, RecordBatch};use arrow::compute::{BatchCoalescer, SortOptions, filter_record_batch, not};use arrow::datatypes::SchemaRef;-use arrow::ipc::reader::StreamReader;use arrow::util::bit_chunk_iterator::UnalignedBitChunk;use arrow::util::bit_util::apply_bitwise_binary_op;use datafusion_common::{@@ -259,6 +256,13 @@ pub(crate) struct BitwiseSortMergeJoinStream {inner_key_buffer: Vec<RecordBatch>,inner_key_spill: Option<RefCountedTempFile>,+ // Track the active spill_stream+ spill_stream: Option<SendableRecordBatchStream>,+ // Whether the active spill stream has produced any batches yet.+ spill_stream_has_data: bool,+ // Prevents wiping out the buffer if we yield while evaluating the filter+ inner_group_buffered: bool,+// True when buffer_inner_key_group returned Pending after partially// filling inner_key_buffer. On re-entry, buffer_inner_key_group// must skip clear() and resume from poll_next_inner_batch (the@@ -371,6 +375,9 @@ impl BitwiseSortMergeJoinStream {matched: BooleanBufferBuilder::new(0),inner_key_buffer: vec![],inner_key_spill: None,+ spill_stream: None,+ spill_stream_has_data: false,+ inner_group_buffered: false,buffering_inner_pending: false,pending_boundary: None,on_outer,@@ -468,6 +475,9 @@ impl BitwiseSortMergeJoinStream {fn clear_inner_key_group(&mut self) {self.inner_key_buffer.clear();self.inner_key_spill = None;+ self.spill_stream = None;+ self.spill_stream_has_data = false;+ self.inner_group_buffered = false;self.inner_buffer_size = 0;}@@ -749,7 +759,10 @@ impl BitwiseSortMergeJoinStream {/// Process a key match with a filter. For each inner row in the buffered/// key group, evaluates the filter against the outer key group and ORs/// the results into the matched bitset using u64-chunked bitwise ops.- fn process_key_match_with_filter(&mut self) -> Result<()> {+ fn process_key_match_with_filter(+ &mut self,+ cx: &mut Context<'_>,+ ) -> Poll<Result<()>> {self.get_outer_self_cmp()?;let filter = self.filter.as_ref().unwrap();let outer_batch = self.outer_batch.as_ref().unwrap();@@ -785,24 +798,47 @@ impl BitwiseSortMergeJoinStream {).count_ones();- // Process spilled inner batches first (read back from disk).- if let Some(spill_file) = &self.inner_key_spill {- let file = BufReader::new(File::open(spill_file.path())?);- let reader = StreamReader::try_new(file, None)?;- for batch_result in reader {- let inner_slice = batch_result?;- matched_count = eval_filter_for_inner_slice(- self.outer_is_left,- filter,- &outer_slice,- &inner_slice,- &mut self.matched,- self.outer_offset,- outer_group_len,- matched_count,- )?;- if matched_count == outer_group_len {- break;+ // Process spilled inner batches first asynchronously.+ if matched_count < outer_group_len+ && (self.inner_key_spill.is_some() || self.spill_stream.is_some())+ {+ if self.spill_stream.is_none()+ && let Some(spill_file) = &self.inner_key_spill+ {+ let stream = self+ .spill_manager+ .read_spill_as_stream(spill_file.clone(), None)?;+ self.spill_stream = Some(stream);+ }++ while matched_count < outer_group_len {+ let stream = self.spill_stream.as_mut().unwrap();+ match ready!(stream.poll_next_unpin(cx)) {+ Some(Ok(inner_slice)) => {+ self.spill_stream_has_data = true;+ matched_count = eval_filter_for_inner_slice(+ self.outer_is_left,+ filter,+ &outer_slice,+ &inner_slice,+ &mut self.matched,+ self.outer_offset,+ outer_group_len,+ matched_count,+ )?;+ }+ Some(Err(e)) => {+ self.spill_stream = None;+ self.spill_stream_has_data = false;+ return Poll::Ready(Err(e));+ }+ None => {+ self.spill_stream = None;+ if !self.spill_stream_has_data {+ return Poll::Ready(internal_err!("Spill file was empty"));+ }+ break;+ }}}}@@ -830,13 +866,17 @@ impl BitwiseSortMergeJoinStream {}self.outer_offset = outer_group_end;- Ok(())++ self.spill_stream = None;+ self.spill_stream_has_data = false;++ Poll::Ready(Ok(()))}/// Continue processing an outer key group that spans multiple outer/// batches. Returns `true` if this outer batch was fully consumed/// by the key group and the caller should load another.- fn resume_boundary(&mut self) -> Result<bool> {+ fn resume_boundary(&mut self, cx: &mut Context<'_>) -> Poll<Result<bool>> {debug_assert!(self.outer_batch.is_some(),"caller must load outer_batch first"@@ -858,7 +898,7 @@ impl BitwiseSortMergeJoinStream {});self.emit_outer_batch()?;self.outer_batch = None;- return Ok(true);+ return Poll::Ready(Ok(true));}}}@@ -874,7 +914,15 @@ impl BitwiseSortMergeJoinStream {self.null_equality,)?;if same_key {- self.process_key_match_with_filter()?;+ match self.process_key_match_with_filter(cx) {+ Poll::Ready(Ok(())) => (),+ Poll::Ready(Err(e)) => return Poll::Ready(Err(e)),+ Poll::Pending => {+ self.pending_boundary =+ Some(PendingBoundary::Filtered { saved_keys });+ return Poll::Pending;+ }+ }let num_outer = self.outer_batch.as_ref().unwrap().num_rows();if self.outer_offset >= num_outer {self.pending_boundary = Some(PendingBoundary::Filtered {@@ -882,14 +930,63 @@ impl BitwiseSortMergeJoinStream {});self.emit_outer_batch()?;self.outer_batch = None;- return Ok(true);+ return Poll::Ready(Ok(true));}}self.clear_inner_key_group();}None => {}}- Ok(false)+ Poll::Ready(Ok(false))+ }++ /// Helper to process an Equal match across potential outer batch boundaries.+ fn process_filtered_match_loop(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {+ loop {+ ready!(self.process_key_match_with_filter(cx))?;++ let outer_batch = self.outer_batch.as_ref().unwrap();+ if self.outer_offset >= outer_batch.num_rows() {+ let saved_keys =+ slice_keys(&self.outer_key_arrays, outer_batch.num_rows() - 1);++ self.emit_outer_batch()?;+ self.pending_boundary = Some(PendingBoundary::Filtered { saved_keys });++ // Clear stale batch before polling+ self.outer_batch = None;++ match ready!(self.poll_next_outer_batch(cx)) {+ Err(e) => return Poll::Ready(Err(e)),+ Ok(false) => {+ self.pending_boundary = None;+ break;+ }+ Ok(true) => {+ let Some(PendingBoundary::Filtered { saved_keys }) =+ self.pending_boundary.take()+ else {+ unreachable!()+ };+ let same = keys_match(+ &saved_keys,+ &self.outer_key_arrays,+ &self.sort_options,+ self.null_equality,+ )?;+ if same {+ continue;+ }+ break;+ }+ }+ } else {+ break;+ }+ }++ self.clear_inner_key_group(); // This resets inner_group_buffered to false+ Poll::Ready(Ok(()))}/// Main loop: drive the merge-scan to produce output batches.@@ -911,14 +1008,21 @@ impl BitwiseSortMergeJoinStream {}return Poll::Ready(Ok(None));}- Ok(true) => {- if self.resume_boundary()? {- continue;- }- }+ Ok(true) => {} // Loaded batch, move on to checks}}+ // Handles pausing while fetching a NEW outer batch.+ if self.pending_boundary.is_some() && ready!(self.resume_boundary(cx))? {+ continue;+ }++ // Handles pausing while reading the disk stream mid-batch.+ if self.inner_group_buffered {+ ready!(self.process_filtered_match_loop(cx))?;+ continue;+ }+// 2. Ensure we have an inner batch (unless inner is exhausted).// Skip this when resuming a pending boundary — inner was already// advanced past the key group before the boundary loop started.@@ -1043,65 +1147,17 @@ impl BitwiseSortMergeJoinStream {}Ordering::Equal => {if self.filter.is_some() {+ debug_assert!(!self.inner_group_buffered);// Buffer inner key group (may span batches)match ready!(self.buffer_inner_key_group(cx)) {Err(e) => return Poll::Ready(Err(e)),- Ok(_inner_exhausted) => {}+ Ok(_inner_exhausted) => {+ self.inner_group_buffered = true;+ }}-// Process outer rows against buffered inner group// (may need to handle outer batch boundary)- loop {- self.process_key_match_with_filter()?;-- let outer_batch = self.outer_batch.as_ref().unwrap();- if self.outer_offset >= outer_batch.num_rows() {- let saved_keys = slice_keys(- &self.outer_key_arrays,- outer_batch.num_rows() - 1,- );-- self.emit_outer_batch()?;- debug_assert!(- !self.inner_key_buffer.is_empty()- || self.inner_key_spill.is_some(),- "Filtered pending boundary requires inner key data in buffer or spill"- );- self.pending_boundary =- Some(PendingBoundary::Filtered { saved_keys });-- match ready!(self.poll_next_outer_batch(cx)) {- Err(e) => return Poll::Ready(Err(e)),- Ok(false) => {- self.pending_boundary = None;- self.outer_batch = None;- break;- }- Ok(true) => {- let Some(PendingBoundary::Filtered {- saved_keys,- }) = self.pending_boundary.take()- else {- unreachable!()- };- let same = keys_match(- &saved_keys,- &self.outer_key_arrays,- &self.sort_options,- self.null_equality,- )?;- if same {- continue;- }- break;- }- }- } else {- break;- }- }-- self.clear_inner_key_group();+ ready!(self.process_filtered_match_loop(cx))?;} else {// No filter: advance inner past key group, then// mark all outer rows with this key as matched.diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rsindex 9bcc749c23..f1a18aac76 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs@@ -23,8 +23,7 @@use std::cmp::Ordering;use std::collections::{HashMap, VecDeque};-use std::fs::File;-use std::io::BufReader;+use std::fmt::Debug;use std::mem::size_of;use std::ops::Range;use std::pin::Pin;@@ -47,10 +46,9 @@ use crate::{PhysicalExpr, RecordBatchStream, SendableRecordBatchStream};use arrow::array::{types::UInt64Type, *};use arrow::compute::{self, BatchCoalescer, SortOptions, concat_batches, filter_record_batch, interleave,- take, take_arrays,+ take_arrays,};use arrow::datatypes::SchemaRef;-use arrow::ipc::reader::StreamReader;use datafusion_common::cast::as_uint64_array;use datafusion_common::{JoinType, NullEquality, Result, exec_err, internal_err};use datafusion_execution::disk_manager::RefCountedTempFile;@@ -58,7 +56,7 @@ use datafusion_execution::memory_pool::MemoryReservation;use datafusion_execution::runtime_env::RuntimeEnv;use datafusion_physical_expr_common::physical_expr::PhysicalExprRef;-use futures::{Stream, StreamExt};+use futures::{Stream, StreamExt, ready};/// State of SMJ stream#[derive(Debug, PartialEq, Eq)]@@ -382,6 +380,11 @@ pub(super) struct MaterializingSortMergeJoinStream {/// Manages the process of spilling and reading back intermediate datapub spill_manager: SpillManager,+ /// Tracks the active stream when loading spilled buffered batches back in memory+ pub spill_stream: Option<SendableRecordBatchStream>,+ /// Tracks the number of batches currently spilled+ pub spilled_batch_count: usize,+// ========================================================================// CACHED COMPARATORS:// Pre-built comparators to avoid per-row type dispatch in hot loops.@@ -599,6 +602,16 @@ impl Stream for MaterializingSortMergeJoinStream {.filter_mask.len();if accumulated >= self.batch_size {+ // Ensure required spilled batches are restored to memory+ // before processing, as this path invokes freeze_all().+ let needed = self.get_required_batch_indices(+ self.buffered_data.batches.len(),+ );+ if let Err(e) = ready!(+ self.poll_spilled_batches(cx, &needed)+ ) {+ return Poll::Ready(Some(Err(e)));+ }match self.process_filtered_batches()? {Poll::Ready(Some(batch)) => {return Poll::Ready(Some(Ok(batch)));@@ -684,14 +697,13 @@ impl Stream for MaterializingSortMergeJoinStream {self.state = SortMergeJoinState::Init;}SortMergeJoinState::JoinOutput => {- self.join_partial()?;+ // If the batch size limit is reached, restore required spilled batches to memory and freeze.+ // Guarding at the top of the loop safely handles re-entry from Poll::Pending.+ if self.num_unfrozen_pairs() >= self.batch_size {+ let needed = self+ .get_required_batch_indices(self.buffered_data.batches.len());+ ready!(self.poll_spilled_batches(cx, &needed))?;- if self.num_unfrozen_pairs() < self.batch_size {- if self.buffered_data.scanning_finished() {- self.buffered_data.scanning_reset();- self.state = SortMergeJoinState::EmitReadyThenInit;- }- } else {self.freeze_all()?;// Verify metadata alignment before checking if we have batches to output@@ -705,7 +717,6 @@ impl Stream for MaterializingSortMergeJoinStream {}// For non-filtered joins, only output if we have a completed batch- // (opportunistic output when target batch size is reached)if self.joined_record_batches.joined_batches@@ -720,10 +731,26 @@ impl Stream for MaterializingSortMergeJoinStream {.record_output(&self.join_metrics.baseline_metrics());return Poll::Ready(Some(Ok(record_batch)));}+// Otherwise keep buffering (don't output yet)+ continue;+ }++ self.join_partial()?;++ if self.num_unfrozen_pairs() < self.batch_size+ && self.buffered_data.scanning_finished()+ {+ self.buffered_data.scanning_reset();+ self.state = SortMergeJoinState::EmitReadyThenInit;}+ // Note: If join_partial() reached the batch size, the loop repeats to freeze the data.}SortMergeJoinState::Exhausted => {+ let needed =+ self.get_required_batch_indices(self.buffered_data.batches.len());+ ready!(self.poll_spilled_batches(cx, &needed))?;+self.freeze_all()?;// Verify metadata alignment before final output@@ -843,6 +870,8 @@ impl MaterializingSortMergeJoinStream {reservation,runtime_env,spill_manager,+ spill_stream: None,+ spilled_batch_count: 0,streamed_buffered_cmp: None,buffered_equality_cmp: None,streamed_batch_counter: AtomicUsize::new(0),@@ -917,6 +946,84 @@ impl MaterializingSortMergeJoinStream {Poll::Pending}+ /// Identifies which buffered batches are needed for the upcoming freeze operation+ fn get_required_batch_indices(&self, buffered_freeze_count: usize) -> Vec<usize> {+ let mut needed = vec![];+ // Avoid scanning if no spilled batches exist+ if self.spilled_batch_count == 0 {+ return needed;+ }+ // We need all batches that matched with streamed rows+ for chunk in &self.streamed_batch.output_indices {+ if let Some(idx) = chunk.buffered_batch_idx {+ needed.push(idx);+ }+ }++ // Full Joins need to emit null-joined rows, so we need batches up to freeze_count+ if self.join_type == JoinType::Full {+ needed.extend(0..buffered_freeze_count);+ }++ needed.sort_unstable();+ needed.dedup();+ needed+ }++ /// Asynchronously reads spilled batches back into memory.+ /// Only processes the required indices to avoid OOMs.+ fn poll_spilled_batches(+ &mut self,+ cx: &mut Context<'_>,+ required_indices: &[usize],+ ) -> Poll<Result<()>> {+ for &idx in required_indices {+ // Guard against indices that might be out of bounds if the queue was cleared+ if idx >= self.buffered_data.batches.len() {+ continue;+ }++ let bb = &mut self.buffered_data.batches[idx];++ if let BufferedBatchState::Spilled(spill_file) = &bb.batch {+ if self.spill_stream.is_none() {+ let stream = self+ .spill_manager+ .read_spill_as_stream(spill_file.clone(), None)?;+ self.spill_stream = Some(stream);+ }++ match ready!(self.spill_stream.as_mut().unwrap().poll_next_unpin(cx)) {+ Some(Ok(batch)) => {+ // Transition the batch back to InMemory+ bb.batch = BufferedBatchState::InMemory(batch);+ self.spilled_batch_count -= 1;+ // The batch is back in memory, so we must account for its size.+ let newly_allocated =+ bb.size_estimation.saturating_sub(bb.reserved_amount);+ self.reservation.grow(newly_allocated);+ bb.reserved_amount = bb.size_estimation;++ self.join_metrics+ .peak_mem_used()+ .set_max(self.reservation.size());++ self.spill_stream = None;+ }+ Some(Err(e)) => {+ self.spill_stream = None;+ return Poll::Ready(Err(e));+ }+ None => {+ self.spill_stream = None;+ return Poll::Ready(internal_err!("Spill file was empty"));+ }+ }+ }+ }+ Poll::Ready(Ok(()))+ }+/// Poll next streamed rowfn poll_streamed_row(&mut self, cx: &mut Context) -> Poll<Option<Result<()>>> {loop {@@ -931,33 +1038,41 @@ impl MaterializingSortMergeJoinStream {self.streamed_state = StreamedState::Polling;}}- StreamedState::Polling => match self.streamed.poll_next_unpin(cx)? {- Poll::Pending => {- return Poll::Pending;- }- Poll::Ready(None) => {- // Release the streamed input pipeline's resources.- let streamed_schema = self.streamed.schema();- self.streamed =- Box::pin(EmptyRecordBatchStream::new(streamed_schema));- self.streamed_state = StreamedState::Exhausted;+ StreamedState::Polling => {+ let needed =+ self.get_required_batch_indices(self.buffered_data.batches.len());+ if let Err(e) = ready!(self.poll_spilled_batches(cx, &needed)) {+ return Poll::Ready(Some(Err(e)));}- Poll::Ready(Some(batch)) => {- if batch.num_rows() > 0 {- self.freeze_streamed()?;- self.join_metrics.input_batches().add(1);- self.join_metrics.input_rows().add(batch.num_rows());- self.streamed_batch =- StreamedBatch::new(batch, &self.on_streamed);- self.rebuild_streamed_buffered_cmp()?;- // Every incoming streaming batch should have its unique id- // Check `JoinedRecordBatches.self.streamed_batch_counter` documentation- self.streamed_batch_counter- .fetch_add(1, std::sync::atomic::Ordering::SeqCst);- self.streamed_state = StreamedState::Ready;++ match self.streamed.poll_next_unpin(cx)? {+ Poll::Pending => {+ return Poll::Pending;+ }+ Poll::Ready(None) => {+ // Release the streamed input pipeline's resources.+ let streamed_schema = self.streamed.schema();+ self.streamed =+ Box::pin(EmptyRecordBatchStream::new(streamed_schema));+ self.streamed_state = StreamedState::Exhausted;+ }+ Poll::Ready(Some(batch)) => {+ if batch.num_rows() > 0 {+ self.freeze_streamed()?;+ self.join_metrics.input_batches().add(1);+ self.join_metrics.input_rows().add(batch.num_rows());+ self.streamed_batch =+ StreamedBatch::new(batch, &self.on_streamed);+ self.rebuild_streamed_buffered_cmp()?;+ // Every incoming streaming batch should have its unique id+ // Check `JoinedRecordBatches.self.streamed_batch_counter` documentation+ self.streamed_batch_counter+ .fetch_add(1, std::sync::atomic::Ordering::SeqCst);+ self.streamed_state = StreamedState::Ready;+ }}}- },+ }StreamedState::Ready => {return Poll::Ready(Some(Ok(())));}@@ -997,6 +1112,7 @@ impl MaterializingSortMergeJoinStream {.unwrap(); // Operation only return None if no batches are spilled, here we ensure that at least one batch is spilledbuffered_batch.batch = BufferedBatchState::Spilled(spill_file);+ self.spilled_batch_count += 1;// Join key arrays remain in memory after the batch is// spilled — the comparator needs them for key boundary@@ -1036,12 +1152,25 @@ impl MaterializingSortMergeJoinStream {let head_batch = self.buffered_data.head_batch();// If the head batch is fully processed, dequeue it and produce output of it.if head_batch.range.end == head_batch.num_rows {+ // load the spilled head batch before dequeuing+ let needed = self.get_required_batch_indices(1);+ if let Err(e) = ready!(self.poll_spilled_batches(cx, &needed))+ {+ return Poll::Ready(Some(Err(e)));+ }+self.freeze_dequeuing_buffered()?;if let Some(mut buffered_batch) =self.buffered_data.batches.pop_front(){self.produce_buffered_not_matched(&mut buffered_batch)?;self.free_reservation(&buffered_batch);+ if matches!(+ buffered_batch.batch,+ BufferedBatchState::Spilled(_)+ ) {+ self.spilled_batch_count -= 1;+ }head_changed = true;}} else {@@ -1556,18 +1685,6 @@ impl MaterializingSortMergeJoinStream {as_uint64_array(&compute::concat(&refs)?)?.clone()};- let spill_reservation = self.reservation.new_empty();- if matches!(- &self.buffered_data.batches[first_batch_idx].batch,- BufferedBatchState::Spilled(_)- ) {- spill_reservation- .grow(self.buffered_data.batches[first_batch_idx].size_estimation);- self.join_metrics- .peak_mem_used()- .set_max(self.reservation.size() + spill_reservation.size());- }-return fetch_right_columns_by_idxs(&self.buffered_data,first_batch_idx,@@ -1603,29 +1720,20 @@ impl MaterializingSortMergeJoinStream {let num_right_cols = self.buffered_schema.fields().len();// Read each source batch once (spilled batches require disk I/O).- // Track memory for each spilled batch at the point of deserialization- // so the pool reflects actual usage as it grows.- let spill_reservation = self.reservation.new_empty();- let mut source_data: Vec<Option<RecordBatch>> =- Vec::with_capacity(source_batches.len());- for &idx in &source_batches {- let bb = &self.buffered_data.batches[idx];- match &bb.batch {- BufferedBatchState::InMemory(batch) => {- source_data.push(Some(batch.clone()));- }- BufferedBatchState::Spilled(spill_file) => {- spill_reservation.grow(bb.size_estimation);- self.join_metrics- .peak_mem_used()- .set_max(self.reservation.size() + spill_reservation.size());-- let file = BufReader::new(File::open(spill_file.path())?);- let reader = StreamReader::try_new(file, None)?;- source_data.push(reader.into_iter().next().transpose()?);+ let source_data_result: Result<Vec<RecordBatch>> = source_batches+ .iter()+ .map(|&idx| {+ let bb = &self.buffered_data.batches[idx];+ match &bb.batch {+ BufferedBatchState::InMemory(batch) => Ok(batch.clone()),+ BufferedBatchState::Spilled(_) => {+ internal_err!("Buffered batch should have been unspilled before fetching columns")+ }}- }- }+ })+ .collect();++ let source_data = source_data_result?;let mut right_columns = Vec::with_capacity(num_right_cols);for col_idx in 0..num_right_cols {@@ -1637,14 +1745,7 @@ impl MaterializingSortMergeJoinStream {source_arrays.push(null_array.as_ref());for data in &source_data {- match data {- Some(batch) => source_arrays.push(batch.column(col_idx).as_ref()),- None => {- return internal_err!(- "Failed to read spilled buffered batch during interleave"- );- }- }+ source_arrays.push(data.column(col_idx).as_ref());}right_columns.push(interleave(&source_arrays, &interleave_indices)?);}@@ -1838,32 +1939,17 @@ fn fetch_right_columns_from_batch_by_idxs(buffered_indices: &UInt64Array,) -> Result<Vec<ArrayRef>> {match &buffered_batch.batch {- // In memory batch- // In memory batchBufferedBatchState::InMemory(batch) => {- // When indices form a contiguous range (common in SMJ since the- // buffered side is scanned sequentially), use zero-copy slice.if let Some(range) = is_contiguous_range(buffered_indices) {Ok(batch.slice(range.start, range.len()).columns().to_vec())} else {Ok(take_arrays(batch.columns(), buffered_indices, None)?)}}- // If the batch was spilled to disk, less likely- BufferedBatchState::Spilled(spill_file) => {- let mut buffered_cols: Vec<ArrayRef> =- Vec::with_capacity(buffered_indices.len());-- let file = BufReader::new(File::open(spill_file.path())?);- let reader = StreamReader::try_new(file, None)?;-- for batch in reader {- batch?.columns().iter().for_each(|column| {- buffered_cols.extend(take(column, &buffered_indices, None))- });- }-- Ok(buffered_cols)+ BufferedBatchState::Spilled(_) => {+ internal_err!(+ "Buffered batch should have been unspilled before fetching columns"+ )}}}diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rsindex b1fdf3ddab..0347299dd0 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs@@ -2459,6 +2459,24 @@ async fn overallocation_multi_batch_spill() -> Result<()> {assert!(join.metrics().unwrap().spilled_bytes().unwrap() > 0);assert!(join.metrics().unwrap().spilled_rows().unwrap() > 0);+ // For Full joins, get_required_batch_indices extends 0..batches.len(), so+ // poll_spilled_batches can restore all spilled batches at once via infallible+ // grow(). Verify accounting tracked the transient spike and cleaned up.+ let peak_mem = join+ .metrics()+ .and_then(|m| m.sum_by_name("peak_mem_used"))+ .map(|m| m.as_usize())+ .unwrap_or(0);+ assert!(+ peak_mem > 0,+ "peak_mem_used should be > 0 for {join_type:?} batch_size={batch_size}"+ );+ assert_eq!(+ runtime.memory_pool.reserved(),+ 0,+ "memory should be fully released after {join_type:?} completes+ (batch_size={batch_size}): infallible grow during restore must be balanced"+ );// Run the test with no spill configuration aslet task_ctx_no_spill =TaskContext::default().with_session_config(session_config.clone());