Skip to content

Apache DataFusion SQL Query Engine

fix: avoid quadratic row-buffer accounting in sort merges (#26366)

Geoffrey Claude3d9037cb0fdc
diff --git a/datafusion/core/benches/sort.rs b/datafusion/core/benches/sort.rs
index ac4be5b8b2..5dcdfb3eac 100644
--- a/datafusion/core/benches/sort.rs
+++ b/datafusion/core/benches/sort.rs
@@ -1065,5 +1065,64 @@ fn with_extra_columns(batches: Vec<RecordBatch>, n: usize) -> Vec<RecordBatch> {
.collect()
}
-criterion_group!(benches, criterion_benchmark, sort_axis_benchmark);
+fn small_batches_benchmark(c: &mut Criterion) {
+ let mut group = c.benchmark_group("small_batches");
+ group.sample_size(10);
+ group.sampling_mode(criterion::SamplingMode::Flat);
+ group.bench_function("two_keys", |b| {
+ let runtime = Runtime::new().unwrap();
+ let ctx = SessionContext::new_with_config(
+ SessionConfig::new()
+ .with_target_partitions(1)
+ .with_batch_size(32),
+ );
+
+ // Store small input batches before timing, then restore normal output batches.
+ runtime.block_on(async {
+ ctx.sql(
+ "CREATE TABLE sort_input AS \
+ SELECT value % 997 AS sort_key, value FROM range(2000000)",
+ )
+ .await
+ .unwrap()
+ .collect()
+ .await
+ .unwrap();
+ ctx.sql("SET datafusion.execution.batch_size = 8192")
+ .await
+ .unwrap()
+ .collect()
+ .await
+ .unwrap();
+ });
+
+ // Two keys exercise row-format merging of the small sorted runs.
+ b.iter(|| {
+ runtime.block_on(async {
+ let mut stream = ctx
+ .sql(
+ "SELECT sort_key, value FROM sort_input ORDER BY sort_key, value",
+ )
+ .await
+ .unwrap()
+ .execute_stream()
+ .await
+ .unwrap();
+ let mut row_count = 0;
+ while let Some(batch) = stream.next().await {
+ row_count += batch.unwrap().num_rows();
+ }
+ row_count
+ })
+ });
+ });
+ group.finish();
+}
+
+criterion_group!(
+ benches,
+ criterion_benchmark,
+ sort_axis_benchmark,
+ small_batches_benchmark
+);
criterion_main!(benches);
diff --git a/datafusion/physical-plan/src/sorts/stream.rs b/datafusion/physical-plan/src/sorts/stream.rs
index d7bc628225..b1708beb51 100644
--- a/datafusion/physical-plan/src/sorts/stream.rs
+++ b/datafusion/physical-plan/src/sorts/stream.rs
@@ -103,6 +103,9 @@ impl FusedStreams {
#[derive(Debug)]
struct ReusableRows {
inner: Vec<Option<Arc<Rows>>>,
+ /// Each stream's charge stays reserved while its buffer is taken for reuse.
+ /// The sum of these sizes equals `reservation.size()`.
+ reserved_sizes: Vec<usize>,
reservation: MemoryReservation,
}
@@ -130,23 +133,27 @@ impl ReusableRows {
/// reservation is free for the same reason. A pool that cannot cover the buffer
/// fails the query here, as it did before the buffer was cached at all.
fn save(&mut self, stream_idx: usize, rows: &Arc<Rows>) -> Result<()> {
- self.inner[stream_idx] = Some(Arc::clone(rows));
- let retained = self.retained_size();
+ let size = rows.size();
+ let retained = self.reservation.size() - self.reserved_sizes[stream_idx] + size;
debug_assert!(retained >= self.reservation.size());
if let Err(e) = self.reservation.try_resize(retained) {
- self.inner[stream_idx] = None;
- self.reservation.try_resize(self.retained_size())?;
+ self.release(stream_idx);
return Err(e);
}
+ self.inner[stream_idx] = Some(Arc::clone(rows));
+ self.reserved_sizes[stream_idx] = size;
Ok(())
}
// drop whatever a finished stream was holding
fn release(&mut self, stream_idx: usize) {
debug_assert!(self.reservation.size() >= self.retained_size());
- if let Some(rows) = self.inner[stream_idx].take() {
- self.reservation.shrink(rows.size());
- }
+ self.inner[stream_idx] = None;
+ let reserved = mem::take(&mut self.reserved_sizes[stream_idx]);
+ self.reservation.resize(self.reservation.size() - reserved);
+ // Verify the incremental accounting against a full recomputation, avoiding
+ // the cost of scanning all buffers in release builds.
+ debug_assert_eq!(self.reservation.size(), self.retained_size());
}
fn retained_size(&self) -> usize {
@@ -200,6 +207,7 @@ impl RowCursorStream {
streams: FusedStreams(streams),
rows: ReusableRows {
inner: vec![None; stream_count],
+ reserved_sizes: vec![0; stream_count],
reservation: rows_reservation,
},
})
@@ -442,7 +450,7 @@ mod tests {
use datafusion_common::DataFusionError;
use datafusion_execution::RecordBatchStream;
use datafusion_execution::memory_pool::{
- GreedyMemoryPool, MemoryConsumer, MemoryPool,
+ GreedyMemoryPool, MemoryConsumer, MemoryLimit, MemoryPool,
};
use datafusion_physical_expr::expressions::col;
use futures::Stream;
@@ -907,4 +915,36 @@ mod tests {
Ok(())
}
+
+ // The buffer has already been taken out of the cache when its reservation fails
+ // to grow. Cleanup must still release its previous charge.
+ #[test]
+ fn failed_buffer_growth_releases_only_its_stream_reservation() -> Result<()> {
+ let small = (64, 8);
+ let large = (2048, 128);
+ let mut harness =
+ RowCursorHarness::with_shapes(vec![vec![small, large], vec![small]])?;
+
+ // Cache stream 1's buffer and record the reservation that must survive.
+ assert!(harness.poll_and_drop_cursor(1)?);
+ let expected_remaining = harness.reserved();
+ assert!(harness.poll_and_drop_cursor(0)?);
+
+ // Leave no space for stream 0's larger batch.
+ let MemoryLimit::Finite(limit) = harness.pool.memory_limit() else {
+ unreachable!("the harness uses a bounded memory pool");
+ };
+ let competitor = MemoryConsumer::new("competitor").register(&harness.pool);
+ competitor.try_grow(limit - harness.reserved())?;
+
+ assert!(matches!(
+ harness.poll_cursor(0),
+ Err(DataFusionError::ResourcesExhausted(_))
+ ));
+ drop(competitor);
+
+ // Only stream 1's buffer and the shared converter remain reserved.
+ assert_eq!(harness.reserved(), expected_remaining);
+ Ok(())
+ }
}