Apache DataFusion SQL Query Engine
mem: Cleanup resources of done streams immediately (#22064)
Emily Matheys12bd5b076584
diff --git a/datafusion/physical-plan/src/aggregates/no_grouping.rs b/datafusion/physical-plan/src/aggregates/no_grouping.rsindex a7dd7c9a66..ac7727b459 100644--- a/datafusion/physical-plan/src/aggregates/no_grouping.rs+++ b/datafusion/physical-plan/src/aggregates/no_grouping.rs@@ -23,6 +23,7 @@ use crate::aggregates::{finalize_aggregation,};use crate::metrics::{BaselineMetrics, RecordOutput};+use crate::stream::EmptyRecordBatchStream;use crate::{RecordBatchStream, SendableRecordBatchStream};use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;@@ -370,6 +371,9 @@ impl AggregateStream {Some(Err(e)) => Err(e),None => {this.finished = true;+ // Release the input pipeline's resources before finalization.+ let input_schema = this.input.schema();+ this.input = Box::pin(EmptyRecordBatchStream::new(input_schema));let timer = this.baseline_metrics.elapsed_compute().timer();let result =finalize_aggregation(&mut this.accumulators, &this.mode)diff --git a/datafusion/physical-plan/src/aggregates/row_hash.rs b/datafusion/physical-plan/src/aggregates/row_hash.rsindex a65aaf9134..a55cf09c79 100644--- a/datafusion/physical-plan/src/aggregates/row_hash.rs+++ b/datafusion/physical-plan/src/aggregates/row_hash.rs@@ -33,6 +33,7 @@ use crate::aggregates::{use crate::metrics::{BaselineMetrics, MetricBuilder, MetricCategory, RecordOutput};use crate::sorts::streaming_merge::{SortedSpillFile, StreamingMergeBuilder};use crate::spill::spill_manager::{GetSlicedSize, SpillManager};+use crate::stream::EmptyRecordBatchStream;use crate::{PhysicalExpr, aggregates, metrics};use crate::{RecordBatchStream, SendableRecordBatchStream};@@ -837,6 +838,10 @@ impl Stream for GroupedHashAggregateStream {self.group_values.len())));}+ // Release the input pipeline's resources.+ let input_schema = self.input.schema();+ self.input =+ Box::pin(EmptyRecordBatchStream::new(input_schema));self.exec_state = ExecutionState::Done;}}@@ -1320,6 +1325,11 @@ impl GroupedHashAggregateStream {fn set_input_done_and_produce_output(&mut self) -> Result<()> {self.input_done = true;self.group_ordering.input_done();+ // Release the original input pipeline's resources now that we're done+ // reading from it. In the spill branch below, `self.input` is replaced+ // again with a stream that merges spill files.+ let input_schema = self.input.schema();+ self.input = Box::pin(EmptyRecordBatchStream::new(input_schema));let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();let timer = elapsed_compute.timer();self.exec_state = if self.spill_state.spills.is_empty() {diff --git a/datafusion/physical-plan/src/aggregates/topk_stream.rs b/datafusion/physical-plan/src/aggregates/topk_stream.rsindex 4aa566ccfc..9128844f1d 100644--- a/datafusion/physical-plan/src/aggregates/topk_stream.rs+++ b/datafusion/physical-plan/src/aggregates/topk_stream.rs@@ -26,6 +26,7 @@ use crate::aggregates::{evaluate_many,};use crate::metrics::BaselineMetrics;+use crate::stream::EmptyRecordBatchStream;use crate::{RecordBatchStream, SendableRecordBatchStream};use arrow::array::{Array, ArrayRef, RecordBatch};use arrow::datatypes::SchemaRef;@@ -205,6 +206,9 @@ impl Stream for GroupedTopKAggregateStream {}// inner is done, emit all rows and switch to producing outputNone => {+ // Release the input pipeline's resources before emitting.+ let input_schema = self.input.schema();+ self.input = Box::pin(EmptyRecordBatchStream::new(input_schema));if self.priority_map.is_empty() {trace!("partition {} emit None", self.partition);return Poll::Ready(None);diff --git a/datafusion/physical-plan/src/analyze.rs b/datafusion/physical-plan/src/analyze.rsindex 491a0872a2..ea3abf439e 100644--- a/datafusion/physical-plan/src/analyze.rs+++ b/datafusion/physical-plan/src/analyze.rs@@ -214,6 +214,7 @@ impl ExecutionPlan for AnalyzeExec {while let Some(batch) = input_stream.next().await.transpose()? {total_rows += batch.num_rows();}+ drop(input_stream);let duration = Instant::now() - start;create_output_batch(diff --git a/datafusion/physical-plan/src/async_func.rs b/datafusion/physical-plan/src/async_func.rsindex 76a68bf570..8ad4ecb096 100644--- a/datafusion/physical-plan/src/async_func.rs+++ b/datafusion/physical-plan/src/async_func.rs@@ -17,7 +17,7 @@use crate::coalesce::LimitedBatchCoalescer;use crate::metrics::{ExecutionPlanMetricsSet, MetricsSet};-use crate::stream::RecordBatchStreamAdapter;+use crate::stream::{EmptyRecordBatchStream, RecordBatchStreamAdapter};use crate::{DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties,check_if_same_properties,@@ -290,6 +290,10 @@ impl Stream for CoalesceInputStream {}None => {completed = true;+ // Release the input pipeline's resources.+ let input_schema = self.input_stream.schema();+ self.input_stream =+ Box::pin(EmptyRecordBatchStream::new(input_schema));if let Err(err) = self.batch_coalescer.finish() {return Poll::Ready(Some(Err(err)));}diff --git a/datafusion/physical-plan/src/coalesce_batches.rs b/datafusion/physical-plan/src/coalesce_batches.rsindex 2bf046f03b..34cd770260 100644--- a/datafusion/physical-plan/src/coalesce_batches.rs+++ b/datafusion/physical-plan/src/coalesce_batches.rs@@ -24,6 +24,7 @@ use std::task::{Context, Poll};use super::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet};use super::{DisplayAs, ExecutionPlanProperties, PlanProperties, Statistics};use crate::projection::ProjectionExec;+use crate::stream::EmptyRecordBatchStream;use crate::{DisplayFormatType, ExecutionPlan, RecordBatchStream, SendableRecordBatchStream,check_if_same_properties,@@ -343,6 +344,8 @@ impl CoalesceBatchesStream {None => {// Input stream is exhausted, finalize any remaining batchesself.completed = true;+ self.input =+ Box::pin(EmptyRecordBatchStream::new(self.coalescer.schema()));self.coalescer.finish()?;}Some(Ok(batch)) => {@@ -353,6 +356,9 @@ impl CoalesceBatchesStream {PushBatchStatus::LimitReached => {// limit was reached, so stop earlyself.completed = true;+ self.input = Box::pin(EmptyRecordBatchStream::new(+ self.coalescer.schema(),+ ));self.coalescer.finish()?;}}diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rsindex 1119d1b240..c485e181f3 100644--- a/datafusion/physical-plan/src/filter.rs+++ b/datafusion/physical-plan/src/filter.rs@@ -42,6 +42,7 @@ use crate::projection::{EmbeddedProjection, ProjectionExec, ProjectionExpr, make_with_child,try_embed_projection, update_expr,};+use crate::stream::EmptyRecordBatchStream;use crate::{DisplayFormatType, ExecutionPlan,metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet, RatioMetrics},@@ -1030,6 +1031,9 @@ impl Stream for FilterExecStream {match ready!(self.input.poll_next_unpin(cx)) {None => {self.batch_coalescer.finish()?;+ // Release the input pipeline's resources.+ let input_schema = self.input.schema();+ self.input = Box::pin(EmptyRecordBatchStream::new(input_schema));// continue draining the coalescer}Some(Ok(batch)) => {@@ -1070,6 +1074,10 @@ impl Stream for FilterExecStream {PushBatchStatus::LimitReached => {// limit was reached, so stop earlyself.batch_coalescer.finish()?;+ // Release the input pipeline's resources.+ let input_schema = self.input.schema();+ self.input =+ Box::pin(EmptyRecordBatchStream::new(input_schema));// continue draining the coalescer}}diff --git a/datafusion/physical-plan/src/joins/cross_join.rs b/datafusion/physical-plan/src/joins/cross_join.rsindex 3027fb130f..ab66955dc6 100644--- a/datafusion/physical-plan/src/joins/cross_join.rs+++ b/datafusion/physical-plan/src/joins/cross_join.rs@@ -31,6 +31,7 @@ use crate::projection::{ProjectionExec, join_allows_pushdown, join_table_borders, new_join_children,physical_to_column_exprs,};+use crate::stream::EmptyRecordBatchStream;use crate::{ColumnStatistics, DisplayAs, DisplayFormatType, Distribution, ExecutionPlan,ExecutionPlanProperties, PlanProperties, RecordBatchStream,@@ -645,7 +646,12 @@ impl<T: BatchTransformer> CrossJoinStream<T> {let right_data = match ready!(self.right.poll_next_unpin(cx)) {Some(Ok(right_data)) => right_data,Some(Err(e)) => return Poll::Ready(Err(e)),- None => return Poll::Ready(Ok(StatefulStreamResult::Ready(None))),+ None => {+ // Release the right (probe) input pipeline's resources.+ let right_schema = self.right.schema();+ self.right = Box::pin(EmptyRecordBatchStream::new(right_schema));+ return Poll::Ready(Ok(StatefulStreamResult::Ready(None)));+ }};self.join_metrics.input_batches.add(1);self.join_metrics.input_rows.add(right_data.num_rows());diff --git a/datafusion/physical-plan/src/joins/hash_join/stream.rs b/datafusion/physical-plan/src/joins/hash_join/stream.rsindex 9885fb5c5c..040470c9be 100644--- a/datafusion/physical-plan/src/joins/hash_join/stream.rs+++ b/datafusion/physical-plan/src/joins/hash_join/stream.rs@@ -35,6 +35,7 @@ use crate::joins::hash_join::shared_bounds::{use crate::joins::utils::{OnceFut, equal_rows_arr, get_final_indices_from_shared_bitmap,};+use crate::stream::EmptyRecordBatchStream;use crate::{RecordBatchStream, SendableRecordBatchStream, handle_state,hash_utils::create_hashes,@@ -587,6 +588,11 @@ impl HashJoinStream {) -> Poll<Result<StatefulStreamResult<Option<RecordBatch>>>> {match ready!(self.right.poll_next_unpin(cx)) {None => {+ // Release the probe-side input pipeline's resources. The schema+ // is preserved so callers that still query `self.right.schema()`+ // (e.g. for unmatched-build emission) keep working.+ let right_schema = self.right.schema();+ self.right = Box::pin(EmptyRecordBatchStream::new(right_schema));self.state = HashJoinStreamState::ExhaustedProbeSide;}Some(Ok(batch)) => {diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rsindex db8c75b4a5..feaf344200 100644--- a/datafusion/physical-plan/src/joins/nested_loop_join.rs+++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs@@ -1611,6 +1611,12 @@ impl NestedLoopJoinStream {}}+ // If the left stream is fully exhausted, release its resources so the+ // upstream pipeline can be torn down before we move on to probing.+ if self.left_exhausted {+ active.left_stream = None;+ }+if active.pending_batches.is_empty() {// No data at all — go directly to Doneself.left_exhausted = true;diff --git a/datafusion/physical-plan/src/joins/piecewise_merge_join/classic_join.rs b/datafusion/physical-plan/src/joins/piecewise_merge_join/classic_join.rsindex da0d21f046..36a043cc7d 100644--- a/datafusion/physical-plan/src/joins/piecewise_merge_join/classic_join.rs+++ b/datafusion/physical-plan/src/joins/piecewise_merge_join/classic_join.rs@@ -39,6 +39,7 @@ use crate::joins::piecewise_merge_join::exec::{BufferedSide, BufferedSideReadyStuse crate::joins::piecewise_merge_join::utils::need_produce_result_in_final;use crate::joins::utils::{BuildProbeJoinMetrics, StatefulStreamResult};use crate::joins::utils::{JoinKeyComparator, get_final_indices_from_shared_bitmap};+use crate::stream::EmptyRecordBatchStream;pub(super) enum PiecewiseMergeJoinStreamState {WaitBufferedSide,@@ -212,6 +213,9 @@ impl ClassicPWMJStream {) -> Poll<Result<StatefulStreamResult<Option<RecordBatch>>>> {match ready!(self.streamed.poll_next_unpin(cx)) {None => {+ // Release the streamed input pipeline's resources.+ let streamed_schema = self.streamed.schema();+ self.streamed = Box::pin(EmptyRecordBatchStream::new(streamed_schema));if self.buffered_side.try_as_ready_mut()?diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rsindex 3b409c98b2..ad7312426b 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs@@ -125,12 +125,12 @@ use std::pin::Pin;use std::sync::Arc;use std::task::{Context, Poll};-use crate::RecordBatchStream;use crate::joins::utils::{JoinFilter, JoinKeyComparator, compare_join_arrays};use crate::metrics::{BaselineMetrics, Count, ExecutionPlanMetricsSet, Gauge, MetricBuilder,};use crate::spill::spill_manager::SpillManager;+use crate::{EmptyRecordBatchStream, RecordBatchStream};use arrow::array::{Array, ArrayRef, BooleanArray, BooleanBufferBuilder, RecordBatch};use arrow::compute::{BatchCoalescer, SortOptions, filter_record_batch, not};use arrow::datatypes::SchemaRef;@@ -475,7 +475,12 @@ impl BitwiseSortMergeJoinStream {fn poll_next_outer_batch(&mut self, cx: &mut Context<'_>) -> Poll<Result<bool>> {loop {match ready!(self.outer.poll_next_unpin(cx)) {- None => return Poll::Ready(Ok(false)),+ None => {+ // Release the outer input pipeline's resources.+ let outer_schema = self.outer.schema();+ self.outer = Box::pin(EmptyRecordBatchStream::new(outer_schema));+ return Poll::Ready(Ok(false));+ }Some(Err(e)) => return Poll::Ready(Err(e)),Some(Ok(batch)) => {let batch_num_rows = batch.num_rows();@@ -503,7 +508,12 @@ impl BitwiseSortMergeJoinStream {fn poll_next_inner_batch(&mut self, cx: &mut Context<'_>) -> Poll<Result<bool>> {loop {match ready!(self.inner.poll_next_unpin(cx)) {- None => return Poll::Ready(Ok(false)),+ None => {+ // Release the inner input pipeline's resources.+ let inner_schema = self.inner.schema();+ self.inner = Box::pin(EmptyRecordBatchStream::new(inner_schema));+ return Poll::Ready(Ok(false));+ }Some(Err(e)) => return Poll::Ready(Err(e)),Some(Ok(batch)) => {let batch_num_rows = batch.num_rows();diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rsindex 069e94d0a9..5d23046ec7 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs@@ -41,6 +41,7 @@ use crate::joins::sort_merge_join::metrics::SortMergeJoinMetrics;use crate::joins::utils::{JoinFilter, JoinKeyComparator};use crate::metrics::RecordOutput;use crate::spill::spill_manager::SpillManager;+use crate::stream::EmptyRecordBatchStream;use crate::{PhysicalExpr, RecordBatchStream, SendableRecordBatchStream};use arrow::array::{types::UInt64Type, *};@@ -935,6 +936,10 @@ impl MaterializingSortMergeJoinStream {return Poll::Pending;}Poll::Ready(None) => {+ // Release the streamed input pipeline's resources.+ let streamed_schema = self.streamed.schema();+ self.streamed =+ Box::pin(EmptyRecordBatchStream::new(streamed_schema));self.streamed_state = StreamedState::Exhausted;}Poll::Ready(Some(batch)) => {@@ -1063,6 +1068,10 @@ impl MaterializingSortMergeJoinStream {return Poll::Pending;}Poll::Ready(None) => {+ // Release the buffered input pipeline's resources.+ let buffered_schema = self.buffered.schema();+ self.buffered =+ Box::pin(EmptyRecordBatchStream::new(buffered_schema));self.buffered_state = BufferedState::Exhausted;return Poll::Ready(None);}@@ -1106,6 +1115,11 @@ impl MaterializingSortMergeJoinStream {return Poll::Pending;}Poll::Ready(None) => {+ // Release the buffered input pipeline's resources.+ let buffered_schema = self.buffered.schema();+ self.buffered = Box::pin(EmptyRecordBatchStream::new(+ buffered_schema,+ ));self.buffered_state = BufferedState::Ready;}Poll::Ready(Some(batch)) => {diff --git a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs b/datafusion/physical-plan/src/joins/symmetric_hash_join.rsindex 11e036434e..34af88ea40 100644--- a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs+++ b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs@@ -50,6 +50,7 @@ use crate::projection::{ProjectionExec, join_allows_pushdown, join_table_borders, new_join_children,physical_to_column_exprs, update_join_filter, update_join_on,};+use crate::stream::EmptyRecordBatchStream;use crate::{DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, ExecutionPlanProperties,PlanProperties, RecordBatchStream, SendableRecordBatchStream,@@ -1406,6 +1407,19 @@ impl<T: BatchTransformer> SymmetricHashJoinStream<T> {}}}++ /// Release the right input pipeline's resources.+ fn cleanup_depleted_right_stream(&mut self) {+ let right_schema = self.right_stream.schema();+ self.right_stream = Box::pin(EmptyRecordBatchStream::new(right_schema));+ }++ /// Release the left input pipeline's resources.+ fn cleanup_depleted_left_stream(&mut self) {+ let left_schema = self.left_stream.schema();+ self.left_stream = Box::pin(EmptyRecordBatchStream::new(left_schema));+ }+/// Asynchronously pulls the next batch from the right stream.////// This default implementation checks for the next value in the right stream.@@ -1429,6 +1443,7 @@ impl<T: BatchTransformer> SymmetricHashJoinStream<T> {}Some(Err(e)) => Poll::Ready(Err(e)),None => {+ self.cleanup_depleted_right_stream();self.set_state(SHJStreamState::RightExhausted);Poll::Ready(Ok(StatefulStreamResult::Continue))}@@ -1458,6 +1473,7 @@ impl<T: BatchTransformer> SymmetricHashJoinStream<T> {}Some(Err(e)) => Poll::Ready(Err(e)),None => {+ self.cleanup_depleted_left_stream();self.set_state(SHJStreamState::LeftExhausted);Poll::Ready(Ok(StatefulStreamResult::Continue))}@@ -1487,6 +1503,7 @@ impl<T: BatchTransformer> SymmetricHashJoinStream<T> {}Some(Err(e)) => Poll::Ready(Err(e)),None => {+ self.cleanup_depleted_left_stream();self.set_state(SHJStreamState::BothExhausted {final_result: false,});@@ -1518,6 +1535,7 @@ impl<T: BatchTransformer> SymmetricHashJoinStream<T> {}Some(Err(e)) => Poll::Ready(Err(e)),None => {+ self.cleanup_depleted_right_stream();self.set_state(SHJStreamState::BothExhausted {final_result: false,});diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rsindex b4af6e2c09..873ba8a5ee 100644--- a/datafusion/physical-plan/src/repartition/mod.rs+++ b/datafusion/physical-plan/src/repartition/mod.rs@@ -38,7 +38,7 @@ use crate::projection::{ProjectionExec, all_columns, make_with_child, update_expuse crate::sorts::streaming_merge::StreamingMergeBuilder;use crate::spill::spill_manager::SpillManager;use crate::spill::spill_pool::{self, SpillPoolWriter};-use crate::stream::RecordBatchStreamAdapter;+use crate::stream::{EmptyRecordBatchStream, RecordBatchStreamAdapter};use crate::{DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, Statistics,check_if_same_properties,@@ -1757,7 +1757,11 @@ impl PerPartitionStream {return Poll::Ready(Some(Err(e)));}Poll::Ready(None) => {- // Spill stream ended, keep draining the memory channel+ // Spill stream ended — release its resources before+ // we go back to draining the memory channel.+ let spill_schema = self.spill_stream.schema();+ self.spill_stream =+ Box::pin(EmptyRecordBatchStream::new(spill_schema));self.state = StreamState::ReadingMemory;}Poll::Pending => {diff --git a/datafusion/physical-plan/src/sorts/partial_sort.rs b/datafusion/physical-plan/src/sorts/partial_sort.rsindex 28b8745235..abd9ebb142 100644--- a/datafusion/physical-plan/src/sorts/partial_sort.rs+++ b/datafusion/physical-plan/src/sorts/partial_sort.rs@@ -58,6 +58,7 @@ use std::task::{Context, Poll};use crate::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet};use crate::sorts::sort::sort_batch;+use crate::stream::EmptyRecordBatchStream;use crate::{DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, ExecutionPlanProperties,Partitioning, PlanProperties, SendableRecordBatchStream, Statistics,@@ -403,6 +404,9 @@ impl PartialSortStream {// Check if we've already reached the fetch limitif self.fetch == Some(0) {self.is_closed = true;+ // Release the input pipeline's resources.+ let input_schema = self.input.schema();+ self.input = Box::pin(EmptyRecordBatchStream::new(input_schema));return Poll::Ready(None);}@@ -436,6 +440,9 @@ impl PartialSortStream {Some(Err(e)) => return Poll::Ready(Some(Err(e))),None => {self.is_closed = true;+ // Release the input pipeline's resources before sorting.+ let input_schema = self.input.schema();+ self.input = Box::pin(EmptyRecordBatchStream::new(input_schema));// Once input is consumed, sort the rest of the inserted batcheslet remaining_batch = self.sort_in_mem_batch()?;return if remaining_batch.num_rows() > 0 {diff --git a/datafusion/physical-plan/src/sorts/partitioned_topk.rs b/datafusion/physical-plan/src/sorts/partitioned_topk.rsindex f5d47f503b..f4c2585ea7 100644--- a/datafusion/physical-plan/src/sorts/partitioned_topk.rs+++ b/datafusion/physical-plan/src/sorts/partitioned_topk.rs@@ -495,6 +495,8 @@ async fn do_partitioned_topk(topk.insert_batch(sub_batch)?;}}+ // Release the input pipeline now that accumulation is complete.+ drop(input);// ---------- Emit phase ----------// Sort partition keys so output is ordered by (partition_keys, order_keys).diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rsindex 6c02af8dec..044580a86f 100644--- a/datafusion/physical-plan/src/sorts/sort.rs+++ b/datafusion/physical-plan/src/sorts/sort.rs@@ -1240,6 +1240,7 @@ impl ExecutionPlan for SortExec {break;}}+ drop(input);topk.emit()}).try_flatten(),@@ -1264,6 +1265,7 @@ impl ExecutionPlan for SortExec {let batch = batch?;sorter.insert_batch(batch).await?;}+ drop(input);sorter.sort().await}).try_flatten(),diff --git a/datafusion/physical-plan/src/spill/replayable_spill_input.rs b/datafusion/physical-plan/src/spill/replayable_spill_input.rsindex ddef15a639..fea998d268 100644--- a/datafusion/physical-plan/src/spill/replayable_spill_input.rs+++ b/datafusion/physical-plan/src/spill/replayable_spill_input.rs@@ -282,6 +282,9 @@ impl Stream for ReplayableSpillStream {}// The stream is exhausted, give the inner state ownership back to `ReplayableStreamSource`Poll::Ready(None) => {+ // Release the input pipeline's resources.+ let inner_schema = this.inner.schema();+ this.inner = Box::pin(EmptyRecordBatchStream::new(inner_schema));if let Some(spill_file) = this.spill_file.as_mut() {match spill_file.finish() {Ok(file) => {diff --git a/datafusion/physical-plan/src/stream.rs b/datafusion/physical-plan/src/stream.rsindex 9139a6dd04..9d0b964886 100644--- a/datafusion/physical-plan/src/stream.rs+++ b/datafusion/physical-plan/src/stream.rs@@ -411,8 +411,11 @@ pin_project! {pub struct RecordBatchStreamAdapter<S> {schema: SchemaRef,+ // Wrapped in Option so we can drop the inner stream as soon as it+ // returns `None`, releasing any upstream pipeline resources before the+ // adapter itself is dropped.#[pin]- stream: S,+ stream: Option<S>,}}@@ -441,7 +444,10 @@ impl<S> RecordBatchStreamAdapter<S> {/// // .../// ```pub fn new(schema: SchemaRef, stream: S) -> Self {- Self { schema, stream }+ Self {+ schema,+ stream: Some(stream),+ }}}@@ -460,11 +466,29 @@ wheretype Item = Result<RecordBatch>;fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {- self.project().stream.poll_next(cx)+ let mut this = self.project();+ let Some(inner) = this.stream.as_mut().as_pin_mut() else {+ return Poll::Ready(None);+ };+ let item = ready!(inner.poll_next(cx));+ if item.is_none() {+ // Drop the inner stream in place to release its resources.+ // SAFETY: the inner stream is dropped without moving it out of+ // its pinned location; assigning `None` only runs the inner+ // value's destructor in place, which is permitted for pinned+ // values.+ unsafe {+ *this.stream.as_mut().get_unchecked_mut() = None;+ }+ }+ Poll::Ready(item)}fn size_hint(&self) -> (usize, Option<usize>) {- self.stream.size_hint()+ match self.stream.as_ref() {+ Some(stream) => stream.size_hint(),+ None => (0, Some(0)),+ }}}@@ -538,6 +562,7 @@ impl ObservedStream {let Some(fetch) = self.fetch else { return poll };if self.produced >= fetch {+ self.release_inner();return Poll::Ready(None);}@@ -545,12 +570,22 @@ impl ObservedStream {if self.produced + batch.num_rows() > fetch {let batch = batch.slice(0, fetch.saturating_sub(self.produced));self.produced += batch.num_rows();+ if self.produced >= fetch {+ self.release_inner();+ }return Poll::Ready(Some(Ok(batch)));};self.produced += batch.num_rows()}poll}++ /// Replace the inner stream with an [`EmptyRecordBatchStream`], dropping+ /// the original stream so its upstream pipeline can be torn down.+ fn release_inner(&mut self) {+ let schema = self.inner.schema();+ self.inner = Box::pin(EmptyRecordBatchStream::new(schema));+ }}impl RecordBatchStream for ObservedStream {@@ -678,7 +713,12 @@ impl BatchSplitStream {}}Some(Err(e)) => Poll::Ready(Some(Err(e))),- None => Poll::Ready(None),+ None => {+ // Release the input pipeline's resources.+ let input_schema = self.input.schema();+ self.input = Box::pin(EmptyRecordBatchStream::new(input_schema));+ Poll::Ready(None)+ }}}}@@ -751,6 +791,9 @@ impl Stream for ReservationStream {None => {// Stream is done so free the reservation completelyself.reservation.free();+ // Release the input pipeline's resources.+ let inner_schema = self.inner.schema();+ self.inner = Box::pin(EmptyRecordBatchStream::new(inner_schema));Poll::Ready(None)}}diff --git a/datafusion/physical-plan/src/unnest.rs b/datafusion/physical-plan/src/unnest.rsindex c774ff09af..3a4b9d7232 100644--- a/datafusion/physical-plan/src/unnest.rs+++ b/datafusion/physical-plan/src/unnest.rs@@ -26,6 +26,7 @@ use super::metrics::{MetricsSet, RecordOutput,};use super::{DisplayAs, ExecutionPlanProperties, PlanProperties};+use crate::stream::EmptyRecordBatchStream;use crate::{DisplayFormatType, Distribution, ExecutionPlan, RecordBatchStream,SendableRecordBatchStream, check_if_same_properties,@@ -382,6 +383,7 @@ impl UnnestStream {debug_assert!(result_batch.num_rows() > 0);Some(Ok(result_batch))}+ // If the stream is depleted or returned an error, log the finish message:other => {trace!("Processed {} probe-side input batches containing {} rows and \@@ -392,6 +394,14 @@ impl UnnestStream {self.metrics.baseline_metrics.output_rows(),self.metrics.baseline_metrics.elapsed_compute(),);++ // In the non-error case, i.e., input is simply depleted:+ if other.is_none() {+ // Release the input pipeline's resources.+ let input_schema = self.input.schema();+ self.input = Box::pin(EmptyRecordBatchStream::new(input_schema));+ }+other}});diff --git a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rsindex 14f8ce5e95..f442bcea94 100644--- a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs+++ b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs@@ -28,6 +28,7 @@ use std::task::{Context, Poll};use super::utils::create_schema;use crate::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet};+use crate::stream::EmptyRecordBatchStream;use crate::windows::{calc_requirements, get_ordered_partition_by_indices, get_partition_by_sort_exprs,window_equivalence_properties,@@ -1083,6 +1084,10 @@ impl BoundedWindowAggStream {let _timer = elapsed_compute.timer();self.finished = true;+ // Release the input pipeline's resources before computing the+ // final aggregates.+ let input_schema = self.input.schema();+ self.input = Box::pin(EmptyRecordBatchStream::new(input_schema));for (_, partition_batch_state) in self.partition_buffers.iter_mut() {partition_batch_state.is_end = true;}diff --git a/datafusion/physical-plan/src/windows/window_agg_exec.rs b/datafusion/physical-plan/src/windows/window_agg_exec.rsindex 5098c84034..9e8fc8a6eb 100644--- a/datafusion/physical-plan/src/windows/window_agg_exec.rs+++ b/datafusion/physical-plan/src/windows/window_agg_exec.rs@@ -24,6 +24,7 @@ use std::task::{Context, Poll};use super::utils::create_schema;use crate::execution_plan::{CardinalityEffect, EmissionType};use crate::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet};+use crate::stream::EmptyRecordBatchStream;use crate::windows::{calc_requirements, get_ordered_partition_by_indices, get_partition_by_sort_exprs,window_equivalence_properties,@@ -444,6 +445,10 @@ impl WindowAggStream {}Some(Err(e)) => Err(e),None => {+ // Release the input pipeline's resources before computing+ // the final aggregates.+ let input_schema = self.input.schema();+ self.input = Box::pin(EmptyRecordBatchStream::new(input_schema));let Some(result) = self.compute_aggregates()? else {return Poll::Ready(None);};