Apache DataFusion SQL Query Engine
fix: avoid quadratic row-buffer accounting in sort merges (#26366)
Geoffrey Claude3d9037cb0fdc
diff --git a/datafusion/core/benches/sort.rs b/datafusion/core/benches/sort.rsindex ac4be5b8b2..5dcdfb3eac 100644--- a/datafusion/core/benches/sort.rs+++ b/datafusion/core/benches/sort.rs@@ -1065,5 +1065,64 @@ fn with_extra_columns(batches: Vec<RecordBatch>, n: usize) -> Vec<RecordBatch> {.collect()}-criterion_group!(benches, criterion_benchmark, sort_axis_benchmark);+fn small_batches_benchmark(c: &mut Criterion) {+ let mut group = c.benchmark_group("small_batches");+ group.sample_size(10);+ group.sampling_mode(criterion::SamplingMode::Flat);+ group.bench_function("two_keys", |b| {+ let runtime = Runtime::new().unwrap();+ let ctx = SessionContext::new_with_config(+ SessionConfig::new()+ .with_target_partitions(1)+ .with_batch_size(32),+ );++ // Store small input batches before timing, then restore normal output batches.+ runtime.block_on(async {+ ctx.sql(+ "CREATE TABLE sort_input AS \+ SELECT value % 997 AS sort_key, value FROM range(2000000)",+ )+ .await+ .unwrap()+ .collect()+ .await+ .unwrap();+ ctx.sql("SET datafusion.execution.batch_size = 8192")+ .await+ .unwrap()+ .collect()+ .await+ .unwrap();+ });++ // Two keys exercise row-format merging of the small sorted runs.+ b.iter(|| {+ runtime.block_on(async {+ let mut stream = ctx+ .sql(+ "SELECT sort_key, value FROM sort_input ORDER BY sort_key, value",+ )+ .await+ .unwrap()+ .execute_stream()+ .await+ .unwrap();+ let mut row_count = 0;+ while let Some(batch) = stream.next().await {+ row_count += batch.unwrap().num_rows();+ }+ row_count+ })+ });+ });+ group.finish();+}++criterion_group!(+ benches,+ criterion_benchmark,+ sort_axis_benchmark,+ small_batches_benchmark+);criterion_main!(benches);diff --git a/datafusion/physical-plan/src/sorts/stream.rs b/datafusion/physical-plan/src/sorts/stream.rsindex d7bc628225..b1708beb51 100644--- a/datafusion/physical-plan/src/sorts/stream.rs+++ b/datafusion/physical-plan/src/sorts/stream.rs@@ -103,6 +103,9 @@ impl FusedStreams {#[derive(Debug)]struct ReusableRows {inner: Vec<Option<Arc<Rows>>>,+ /// Each stream's charge stays reserved while its buffer is taken for reuse.+ /// The sum of these sizes equals `reservation.size()`.+ reserved_sizes: Vec<usize>,reservation: MemoryReservation,}@@ -130,23 +133,27 @@ impl ReusableRows {/// reservation is free for the same reason. A pool that cannot cover the buffer/// fails the query here, as it did before the buffer was cached at all.fn save(&mut self, stream_idx: usize, rows: &Arc<Rows>) -> Result<()> {- self.inner[stream_idx] = Some(Arc::clone(rows));- let retained = self.retained_size();+ let size = rows.size();+ let retained = self.reservation.size() - self.reserved_sizes[stream_idx] + size;debug_assert!(retained >= self.reservation.size());if let Err(e) = self.reservation.try_resize(retained) {- self.inner[stream_idx] = None;- self.reservation.try_resize(self.retained_size())?;+ self.release(stream_idx);return Err(e);}+ self.inner[stream_idx] = Some(Arc::clone(rows));+ self.reserved_sizes[stream_idx] = size;Ok(())}// drop whatever a finished stream was holdingfn release(&mut self, stream_idx: usize) {debug_assert!(self.reservation.size() >= self.retained_size());- if let Some(rows) = self.inner[stream_idx].take() {- self.reservation.shrink(rows.size());- }+ self.inner[stream_idx] = None;+ let reserved = mem::take(&mut self.reserved_sizes[stream_idx]);+ self.reservation.resize(self.reservation.size() - reserved);+ // Verify the incremental accounting against a full recomputation, avoiding+ // the cost of scanning all buffers in release builds.+ debug_assert_eq!(self.reservation.size(), self.retained_size());}fn retained_size(&self) -> usize {@@ -200,6 +207,7 @@ impl RowCursorStream {streams: FusedStreams(streams),rows: ReusableRows {inner: vec![None; stream_count],+ reserved_sizes: vec![0; stream_count],reservation: rows_reservation,},})@@ -442,7 +450,7 @@ mod tests {use datafusion_common::DataFusionError;use datafusion_execution::RecordBatchStream;use datafusion_execution::memory_pool::{- GreedyMemoryPool, MemoryConsumer, MemoryPool,+ GreedyMemoryPool, MemoryConsumer, MemoryLimit, MemoryPool,};use datafusion_physical_expr::expressions::col;use futures::Stream;@@ -907,4 +915,36 @@ mod tests {Ok(())}++ // The buffer has already been taken out of the cache when its reservation fails+ // to grow. Cleanup must still release its previous charge.+ #[test]+ fn failed_buffer_growth_releases_only_its_stream_reservation() -> Result<()> {+ let small = (64, 8);+ let large = (2048, 128);+ let mut harness =+ RowCursorHarness::with_shapes(vec![vec![small, large], vec![small]])?;++ // Cache stream 1's buffer and record the reservation that must survive.+ assert!(harness.poll_and_drop_cursor(1)?);+ let expected_remaining = harness.reserved();+ assert!(harness.poll_and_drop_cursor(0)?);++ // Leave no space for stream 0's larger batch.+ let MemoryLimit::Finite(limit) = harness.pool.memory_limit() else {+ unreachable!("the harness uses a bounded memory pool");+ };+ let competitor = MemoryConsumer::new("competitor").register(&harness.pool);+ competitor.try_grow(limit - harness.reserved())?;++ assert!(matches!(+ harness.poll_cursor(0),+ Err(DataFusionError::ResourcesExhausted(_))+ ));+ drop(competitor);++ // Only stream 1's buffer and the shared converter remain reserved.+ assert_eq!(harness.reserved(), expected_remaining);+ Ok(())+ }}