Skip to content

Apache DataFusion SQL Query Engine

refactor: remove redundant sort-run reservation stream (#26930)

Geoffrey Claude8acb40dcb7a1
diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs
index 17e66ef339..96e673e404 100644
--- a/datafusion/physical-plan/src/sorts/sort.rs
+++ b/datafusion/physical-plan/src/sorts/sort.rs
@@ -35,7 +35,6 @@ use crate::spill::get_record_batch_memory_size;
use crate::spill::in_progress_spill_file::InProgressSpillFile;
use crate::spill::spill_manager::{GetSlicedSize, SpillManager};
use crate::statistics::{ChildStats, StatisticsArgs, with_per_partition_fetch};
-use crate::stream::ReservationStream;
use crate::stream::{ObservedStream, RecordBatchStreamAdapter};
use crate::topk::TopK;
use crate::topk::TopKDynamicFilters;
@@ -1053,17 +1052,6 @@ impl ExternalSorter {
.try_resize(total_sorted_size - workspace.size())
.map_err(Self::err_with_oom_context)?;
- if workspace.size() == 0 {
- return Ok(Box::pin(ReservationStream::new(
- Arc::clone(&schema),
- Box::pin(RecordBatchStreamAdapter::new(
- Arc::clone(&schema),
- futures::stream::iter(sorted_batches.into_iter().map(Ok)),
- )),
- reservation,
- )) as SendableRecordBatchStream);
- }
-
// Return borrowed workspace first so the merge's cursors can reuse
// it immediately. Both reservations also release on stream drop.
let batches = sorted_batches.into_iter().map(move |batch| {
diff --git a/datafusion/physical-plan/src/stream.rs b/datafusion/physical-plan/src/stream.rs
index 28870c4ef3..87f5feaaa9 100644
--- a/datafusion/physical-plan/src/stream.rs
+++ b/datafusion/physical-plan/src/stream.rs
@@ -14,13 +14,11 @@ use super::metrics::ExecutionPlanMetricsSet;
use super::metrics::{BaselineMetrics, SplitMetrics};
use super::{ExecutionPlan, RecordBatchStream, SendableRecordBatchStream};
use crate::displayable;
-use crate::spill::get_record_batch_memory_size;
use arrow::{datatypes::SchemaRef, record_batch::RecordBatch};
use datafusion_common::{Result, exec_err};
use datafusion_common_runtime::JoinSet;
use datafusion_execution::TaskContext;
-use datafusion_execution::memory_pool::MemoryReservation;
use futures::ready;
use futures::stream::BoxStream;
@@ -729,73 +727,6 @@ impl RecordBatchStream for BatchSplitStream {
}
}
-/// A stream that holds a memory reservation for its lifetime,
-/// shrinking the reservation as batches are consumed.
-/// The original reservation must have its batch sizes calculated using [`get_record_batch_memory_size`]
-/// On error, the reservation is *NOT* freed, until the stream is dropped.
-pub(crate) struct ReservationStream {
- schema: SchemaRef,
- inner: SendableRecordBatchStream,
- reservation: MemoryReservation,
-}
-
-impl ReservationStream {
- pub(crate) fn new(
- schema: SchemaRef,
- inner: SendableRecordBatchStream,
- reservation: MemoryReservation,
- ) -> Self {
- Self {
- schema,
- inner,
- reservation,
- }
- }
-}
-
-impl Stream for ReservationStream {
- type Item = Result<RecordBatch>;
-
- fn poll_next(
- mut self: Pin<&mut Self>,
- cx: &mut Context<'_>,
- ) -> Poll<Option<Self::Item>> {
- let res = self.inner.poll_next_unpin(cx);
-
- match res {
- Poll::Ready(res) => {
- match res {
- Some(Ok(batch)) => {
- self.reservation
- .shrink(get_record_batch_memory_size(&batch));
- Poll::Ready(Some(Ok(batch)))
- }
- Some(Err(err)) => Poll::Ready(Some(Err(err))),
- None => {
- // Stream is done so free the reservation completely
- self.reservation.free();
- // Release the input pipeline's resources.
- let inner_schema = self.inner.schema();
- self.inner = Box::pin(EmptyRecordBatchStream::new(inner_schema));
- Poll::Ready(None)
- }
- }
- }
- Poll::Pending => Poll::Pending,
- }
- }
-
- fn size_hint(&self) -> (usize, Option<usize>) {
- self.inner.size_hint()
- }
-}
-
-impl RecordBatchStream for ReservationStream {
- fn schema(&self) -> SchemaRef {
- Arc::clone(&self.schema)
- }
-}
-
#[cfg(test)]
mod test {
use super::*;
@@ -1023,123 +954,4 @@ mod test {
"Should have received exactly two empty batches"
);
}
-
- #[tokio::test]
- async fn test_reservation_stream_shrinks_on_poll() {
- use arrow::array::Int32Array;
- use datafusion_execution::memory_pool::MemoryConsumer;
- use datafusion_execution::runtime_env::RuntimeEnvBuilder;
-
- let runtime = RuntimeEnvBuilder::new()
- .with_memory_limit(10 * 1024 * 1024, 1.0)
- .build_arc()
- .unwrap();
-
- let reservation = MemoryConsumer::new("test").register(&runtime.memory_pool);
-
- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
-
- // Create batches
- let batch1 = RecordBatch::try_new(
- Arc::clone(&schema),
- vec![Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]))],
- )
- .unwrap();
- let batch2 = RecordBatch::try_new(
- Arc::clone(&schema),
- vec![Arc::new(Int32Array::from(vec![6, 7, 8, 9, 10]))],
- )
- .unwrap();
-
- let batch1_size = get_record_batch_memory_size(&batch1);
- let batch2_size = get_record_batch_memory_size(&batch2);
-
- // Reserve memory upfront
- reservation.try_grow(batch1_size + batch2_size).unwrap();
- let initial_reserved = runtime.memory_pool.reserved();
- assert_eq!(initial_reserved, batch1_size + batch2_size);
-
- // Create stream with batches
- let stream = futures::stream::iter(vec![Ok(batch1), Ok(batch2)]);
- let inner = Box::pin(RecordBatchStreamAdapter::new(Arc::clone(&schema), stream))
- as SendableRecordBatchStream;
-
- let mut res_stream =
- ReservationStream::new(Arc::clone(&schema), inner, reservation);
-
- // Poll first batch
- let result1 = res_stream.next().await;
- assert!(result1.is_some());
-
- // Memory should be reduced by batch1_size
- let after_first = runtime.memory_pool.reserved();
- assert_eq!(after_first, batch2_size);
-
- // Poll second batch
- let result2 = res_stream.next().await;
- assert!(result2.is_some());
-
- // Memory should be reduced by batch2_size
- let after_second = runtime.memory_pool.reserved();
- assert_eq!(after_second, 0);
-
- // Poll None (end of stream)
- let result3 = res_stream.next().await;
- assert!(result3.is_none());
-
- // Memory should still be 0
- assert_eq!(runtime.memory_pool.reserved(), 0);
- }
-
- #[tokio::test]
- async fn test_reservation_stream_error_handling() {
- use datafusion_execution::memory_pool::MemoryConsumer;
- use datafusion_execution::runtime_env::RuntimeEnvBuilder;
-
- let runtime = RuntimeEnvBuilder::new()
- .with_memory_limit(10 * 1024 * 1024, 1.0)
- .build_arc()
- .unwrap();
-
- let reservation = MemoryConsumer::new("test").register(&runtime.memory_pool);
-
- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
-
- reservation.try_grow(1000).unwrap();
- let initial = runtime.memory_pool.reserved();
- assert_eq!(initial, 1000);
-
- // Create a stream that errors
- let stream = futures::stream::iter(vec![exec_err!("Test error")]);
- let inner = Box::pin(RecordBatchStreamAdapter::new(Arc::clone(&schema), stream))
- as SendableRecordBatchStream;
-
- let mut res_stream =
- ReservationStream::new(Arc::clone(&schema), inner, reservation);
-
- // Get the error
- let result = res_stream.next().await;
- assert!(result.is_some());
- assert!(result.unwrap().is_err());
-
- // Verify reservation is NOT automatically freed on error
- // The reservation is only freed when poll_next returns Poll::Ready(None)
- // After an error, the stream may continue to hold the reservation
- // until it's explicitly dropped or polled to None
- let after_error = runtime.memory_pool.reserved();
- assert_eq!(
- after_error, 1000,
- "Reservation should still be held after error"
- );
-
- // Drop the stream to free the reservation
- drop(res_stream);
-
- // Now memory should be freed
- assert_eq!(
- runtime.memory_pool.reserved(),
- 0,
- "Memory should be freed when stream is dropped"
- );
- }
}