Apache DataFusion SQL Query Engine
fix: bound parallel batch sorting in SortExec (#26242)
Geoffrey Claude676c003146e2
diff --git a/datafusion/core/tests/fuzz_cases/sort_query_fuzz.rs b/datafusion/core/tests/fuzz_cases/sort_query_fuzz.rsindex 149630d910..b3aec96b23 100644--- a/datafusion/core/tests/fuzz_cases/sort_query_fuzz.rs+++ b/datafusion/core/tests/fuzz_cases/sort_query_fuzz.rs@@ -9,6 +9,7 @@ use std::fmt::Write as _;use std::sync::Arc;use arrow::array::RecordBatch;+use arrow::compute::concat_batches;use arrow_schema::SchemaRef;use datafusion::datasource::MemTable;use datafusion::prelude::{SessionConfig, SessionContext};@@ -24,8 +25,6 @@ use rand::Rng;use rand::prelude::IndexedRandom;use rand::{SeedableRng, rngs::StdRng};-use crate::fuzz_cases::aggregation_fuzzer::check_equality_of_batches;-use super::aggregation_fuzzer::ColumnDescr;use super::record_batch_generator::{RecordBatchGenerator, get_supported_types_columns};@@ -186,7 +185,7 @@ impl SortQueryFuzzer {let init_seed = self.runner_rng.random();for query_i in 0..self.queries_per_round {let query_seed = self.runner_rng.random();- let mut expected_results: Option<Vec<RecordBatch>> = None; // use first config's result as the expected result+ let mut expected_results: Option<RecordBatch> = None; // use first config's result as the expected resultfor config_i in 0..self.config_variations_per_query {if self.should_stop_due_to_time_limit(start_time, round, query_i) {return Ok(());@@ -204,18 +203,16 @@ impl SortQueryFuzzer {let results = self.test_gen- .fuzzer_run(init_seed, query_seed, config_seed)+ .fuzzer_run(init_seed, query_seed, config_seed, config_i == 0).await?;println!("\n"); // Separator between tested runsif expected_results.is_none() {expected_results = Some(results);} else if let Some(ref expected) = expected_results {- // `fuzzer_run` might append `LIMIT k` to either the- // expected or actual query. The number of results is- // checked inside `fuzzer_run()`. Here we only check- // that the first k rows of each result are consistent.- check_equality_of_batches(expected, &results).unwrap();+ // Every projected column is an ORDER BY key, so tied+ // rows have identical output and can be compared in order.+ assert_eq!(expected, &results);} else {unreachable!();}@@ -467,6 +464,7 @@ impl SortFuzzerTestGenerator {&self,rng_seed: u64,with_memory_limit: bool,+ sort_in_place_threshold_override: Option<usize>,) -> Result<SessionContext> {let mut rng = StdRng::seed_from_u64(rng_seed);let init_state = self.dataset_state.as_ref().unwrap();@@ -495,6 +493,8 @@ impl SortFuzzerTestGenerator {let dataset_size = self.dataset_state.as_ref().unwrap().dataset_size;rng.random_range(0..=dataset_size * 2_usize)};+ let sort_in_place_threshold_bytes =+ sort_in_place_threshold_override.unwrap_or(sort_in_place_threshold_bytes);// Set up strings for printinglet memory_limit_str = if with_memory_limit {@@ -526,6 +526,8 @@ impl SortFuzzerTestGenerator {let config = SessionConfig::new().with_target_partitions(num_partitions).with_batch_size(init_state.approx_batch_num_rows / 2)+ // Keep small input batches separate when comparing the two sort paths.+ .with_coalesce_batches(sort_in_place_threshold_override.is_none()).with_sort_spill_reservation_bytes(sort_spill_reservation_bytes).with_sort_in_place_threshold_bytes(sort_in_place_threshold_bytes);@@ -556,7 +558,8 @@ impl SortFuzzerTestGenerator {dataset_seed: u64,query_seed: u64,config_seed: u64,- ) -> Result<Vec<RecordBatch>> {+ compare_sort_paths: bool,+ ) -> Result<RecordBatch> {self.init_partitioned_staggered_batches(dataset_seed);let (query_str, limit) = self.generate_random_query(query_seed);println!(" Query:");@@ -569,12 +572,13 @@ impl SortFuzzerTestGenerator {// 2. Memory limiting is enabled in the test generator configlet with_mem_limit = !query_str.contains("LIMIT") && self.set_memory_limit;- let ctx = self.generate_random_config(config_seed, with_mem_limit)?;+ let ctx = self.generate_random_config(config_seed, with_mem_limit, None)?;let df = ctx.sql(&query_str).await.unwrap();- let results = df.collect().await.unwrap();+ let schema = Arc::clone(df.schema().inner());+ let results = concat_batches(&schema, &df.collect().await?)?;// ==== Check the result size is consistent with the limit ====- let result_num_rows = results.iter().map(|batch| batch.num_rows()).sum::<usize>();+ let result_num_rows = results.num_rows();let dataset_size = self.dataset_state.as_ref().unwrap().dataset_size;if let Some(limit) = limit {@@ -582,6 +586,21 @@ impl SortFuzzerTestGenerator {assert_eq!(result_num_rows, expected_num_rows);}+ if compare_sort_paths && limit.is_none() && !with_mem_limit {+ // Run the same query inline and with bounded parallel preparation.+ // An unbounded memory pool allows the inline path to concatenate all runs.+ for threshold in [usize::MAX, 0] {+ let ctx =+ self.generate_random_config(config_seed, false, Some(threshold))?;+ let actual = ctx.sql(&query_str).await?.collect().await?;+ assert_eq!(+ results,+ concat_batches(&schema, &actual)?,+ "sort_in_place_threshold_bytes={threshold}"+ );+ }+ }+Ok(results)}}@@ -603,8 +622,8 @@ mod test {gen_seed,);- let res1 = test_generator.fuzzer_run(1, 2, 3).await.unwrap();- let res2 = test_generator.fuzzer_run(1, 2, 3).await.unwrap();- check_equality_of_batches(&res1, &res2).unwrap();+ let res1 = test_generator.fuzzer_run(1, 2, 3, false).await.unwrap();+ let res2 = test_generator.fuzzer_run(1, 2, 3, false).await.unwrap();+ assert_eq!(res1, res2);}}diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rsindex 90f9488be2..8663515c07 100644--- a/datafusion/physical-plan/src/sorts/sort.rs+++ b/datafusion/physical-plan/src/sorts/sort.rs@@ -8,11 +8,11 @@use std::fmt;use std::fmt::{Debug, Formatter};-use std::sync::Arc;+use std::future::Future;+use std::sync::{Arc, OnceLock};use parking_lot::RwLock;-use crate::common::spawn_buffered;use crate::execution_plan::{Boundedness, CardinalityEffect, EmissionType, has_same_children_properties,replace_children_if_necessary,@@ -25,7 +25,8 @@ use crate::filter_pushdown::{};use crate::limit::LimitStream;use crate::metrics::{- BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet, SpillMetrics,+ BaselineMetrics, ElapsedComputeFutureExt, ExecutionPlanMetricsSet, MetricsSet,+ SpillMetrics,};use crate::projection::{ProjectionExec, make_with_child, update_ordering};use crate::sorts::IncrementalSortIterator;@@ -49,11 +50,14 @@ use arrow::compute::{concat_batches, lexsort_to_indices, take_arrays};use arrow::datatypes::{DataType, SchemaRef};use datafusion_common::config::SpillCompression;use datafusion_common::tree_node::TreeNodeRecursion;-use datafusion_common::utils::memory::RecordBatchMemoryCounter;+use datafusion_common::utils::{+ get_available_parallelism, memory::RecordBatchMemoryCounter,+};use datafusion_common::{DataFusionError, Result, assert_or_internal_err, internal_datafusion_err,unwrap_or_internal_err,};+use datafusion_common_runtime::SpawnedTask;use datafusion_execution::TaskContext;use datafusion_execution::memory_pool::{MemoryConsumer, MemoryPool, MemoryReservation, MergeMemoryPool,@@ -63,8 +67,10 @@ use datafusion_physical_expr::LexOrdering;use datafusion_physical_expr::PhysicalExpr;use datafusion_physical_expr::expressions::{DynamicFilterPhysicalExpr, lit};-use futures::{StreamExt, TryStreamExt};+use futures::{Stream, StreamExt, TryStreamExt};use log::{debug, trace};+use tokio::runtime::{Handle, RuntimeFlavor};+use tokio::sync::Semaphore;#[cfg(test)]mod spill_tests;@@ -85,6 +91,43 @@ impl ExternalSorterMetrics {}}+/// Limits concurrent batch sorting across the partitions of one [`SortExec`].+#[derive(Debug, Clone)]+struct SortConcurrency {+ /// Each partition can use the full concurrency budget, but queues at most this+ /// many jobs so it cannot put all its runs ahead of other partitions.+ per_partition_limit: usize,+ /// Shared budget: a job holds a permit until its run has been sorted.+ permits: Arc<Semaphore>,+}++impl SortConcurrency {+ /// Returns the shared limit on multithreaded Tokio runtimes, initialized+ /// on first use.+ fn for_current_runtime(cell: &OnceLock<Self>) -> Option<Self> {+ let handle = Handle::try_current().ok()?;+ if handle.runtime_flavor() != RuntimeFlavor::MultiThread {+ return None;+ }+ Some(+ cell.get_or_init(|| {+ // These jobs are CPU-bound. Cap concurrency by the worker count+ // to avoid queuing jobs that cannot run, and by estimated CPU+ // parallelism to avoid oversubscribing CPUs on larger runtimes.+ let limit = handle+ .metrics()+ .num_workers()+ .min(get_available_parallelism());+ Self {+ per_partition_limit: limit,+ permits: Arc::new(Semaphore::new(limit)),+ }+ })+ .clone(),+ )+ }+}+/// Sorts an arbitrary sized, unsorted, stream of [`RecordBatch`]es to/// a total order. Depending on the input size and memory manager/// configuration, writes intermediate results to disk ("spills")@@ -266,6 +309,9 @@ struct ExternalSorter {/// How much memory to reserve for performing in-memory sort/merges/// prior to spilling.sort_spill_reservation_bytes: usize,+ /// Shared by the owning SortExec's partitions. Initialized on the runtime+ /// that polls the sort, which may differ from the one calling `execute`.+ sort_concurrency: Arc<OnceLock<SortConcurrency>>,}impl ExternalSorter {@@ -326,6 +372,7 @@ impl ExternalSorter {batch_size,sort_spill_reservation_bytes,sort_in_place_threshold_bytes,+ sort_concurrency: Arc::new(OnceLock::new()),})}@@ -390,7 +437,7 @@ impl ExternalSorter {// workspace so another sorter can start while this stream is alive.self.merge_pool.release_unused();self.merge_reservation.free();- self.in_mem_sort_stream(true, true)+ self.in_mem_sort_stream(true, true).await}}@@ -510,11 +557,13 @@ impl ExternalSorter {// make spilling fail if another task consumes it or our share shrinks.self.merge_reservation.free();- let mut sorted_stream = self.in_mem_sort_stream(- false,- // No coalescing on the spill path: it raises per-run peak memory.- false,- )?;+ let mut sorted_stream = self+ .in_mem_sort_stream(+ false,+ // No coalescing on the spill path: it raises per-run peak memory.+ false,+ )+ .await?;// After `in_mem_sort_stream()` is constructed, all `in_mem_batches` is taken// to construct a globally sorted stream.assert_or_internal_err!(@@ -633,7 +682,7 @@ impl ExternalSorter {/// ```/// `coalesce_runs` merges buffered batches into fewer, larger sorted runs to/// reduce merge fan-in. Disabled on the spill path to keep peak memory low.- fn in_mem_sort_stream(+ async fn in_mem_sort_stream(&mut self,is_output_stream: bool,coalesce_runs: bool,@@ -650,13 +699,11 @@ impl ExternalSorter {// The elapsed compute timer is updated when the value is dropped.// There is no need for an explicit call to drop.let elapsed_compute = self.metrics.baseline.elapsed_compute().clone();- let _timer = elapsed_compute.timer();+ let mut timer = elapsed_compute.timer();- // Please pay attention that any operation inside of `in_mem_sort_stream` will- // not perform any memory reservation. This is for avoiding the need of handling- // reservation failure and spilling in the middle of the sort/merge. The memory- // space for batches produced by the resulting stream will be reserved by the- // consumer of the stream.+ // Sorting can resize reservations and borrow spill workspace.+ // Prepared output stays reserved until each batch is read from its run stream.+ // The consumer of the merged stream reserves its final output.if self.in_mem_batches.len() == 1 {let batch = self.in_mem_batches.swap_remove(0);@@ -694,11 +741,20 @@ impl ExternalSorter {batches};- let streams = self- .sort_run_streams(runs)?- .into_iter()- .map(|stream| spawn_buffered(stream, 1))- .collect();+ let concurrency = SortConcurrency::for_current_runtime(&self.sort_concurrency);+ let streams = if let Some(concurrency) = concurrency {+ // Time this task's setup and collection work, excluding waits for jobs.+ timer.stop();+ let streams = self+ .prepare_sort_runs(runs, &concurrency)+ .with_elapsed_compute(elapsed_compute.clone())+ .await?;+ timer.restart();+ streams+ } else {+ // Outside a multithreaded runtime, sort each run when the merge polls it.+ self.sort_run_streams(runs)?+ };StreamingMergeBuilder::new().with_streams(streams)@@ -715,6 +771,87 @@ impl ExternalSorter {.build()}+ /// Sort runs concurrently, returning their streams in input order.+ async fn prepare_sort_runs(+ &mut self,+ runs: Vec<RecordBatch>,+ concurrency: &SortConcurrency,+ ) -> Result<Vec<SendableRecordBatchStream>> {+ let run_count = runs.len();+ let (sizes, shared_input_size) = self.sort_run_sizes(&runs)?;+ let shared_input =+ shared_input_size.map(|size| Arc::new(self.reservation.split(size)));+ let jobs =+ runs.into_iter()+ .zip(sizes)+ .enumerate()+ .map(move |(index, (batch, size))| {+ let reservation = self.reservation.split(size);+ let sort = self.sort_run(batch, reservation, shared_input.clone())?;+ let permits = Arc::clone(&concurrency.permits);+ Ok::<_, DataFusionError>(async move {+ let stream = Self::prepare_sort_run(sort, permits).await?;+ Ok((index, stream))+ })+ });+ // Ordered buffering would keep completed runs behind a slow earlier run,+ // filling the window and preventing new jobs from starting.+ let jobs = futures::stream::iter(jobs)+ .try_buffer_unordered(concurrency.per_partition_limit);+ Self::collect_prepared_runs(run_count, jobs).await+ }++ /// Restore input order so job completion order does not affect the merge's output.+ async fn collect_prepared_runs(+ run_count: usize,+ mut jobs: impl Stream<Item = Result<(usize, SendableRecordBatchStream)>> + Unpin,+ ) -> Result<Vec<SendableRecordBatchStream>> {+ let mut streams: Vec<_> = (0..run_count).map(|_| None).collect();+ while let Some((index, stream)) = jobs.try_next().await? {+ streams[index] = Some(stream);+ }+ streams+ .into_iter()+ .collect::<Option<Vec<_>>>()+ .ok_or_else(|| internal_datafusion_err!("Missing prepared sort run"))+ }++ /// Sort one run under a shared permit, retaining its output reservations.+ ///+ /// The sort must finish independently of other runs and partitions so its+ /// permit can be returned before the merge starts consuming output.+ async fn prepare_sort_run(+ sort: impl Future<Output = Result<SendableRecordBatchStream>> + Send + 'static,+ permits: Arc<Semaphore>,+ ) -> Result<SendableRecordBatchStream> {+ // Acquire before spawning so queued runs do not become Tokio tasks.+ let permit = permits+ .acquire_owned()+ .await+ .map_err(|e| internal_datafusion_err!("Sort concurrency closed: {e}"))?;+ let stream = SpawnedTask::spawn(async move {+ // Keep the permit until this job exits, even if its caller is cancelled.+ let _permit = permit;+ sort.await+ })+ .join_unwind()+ .await+ .map_err(|e| DataFusionError::ExecutionJoin(Box::new(e)))??;+ Ok(Self::with_lookahead(stream))+ }++ /// [`Self::sort_run`] materializes all output and releases each batch's+ /// reservation when that batch is pulled. Reading one batch ahead returns+ /// extra spill workspace for the merge's current batch and cursor.+ /// Each run can hold one prefetched batch whose reservation is already released.+ fn with_lookahead(stream: SendableRecordBatchStream) -> SendableRecordBatchStream {+ let schema = stream.schema();+ // Buffer the current batch and one ahead to return borrowed spill workspace+ // before the merge processes the current batch.+ let batches = stream.map(std::future::ready).buffered(2);+ Box::pin(RecordBatchStreamAdapter::new(schema, batches))+ }+/// Concatenates `batches` into fewer, larger runs, each bounded by/// `sort_in_place_threshold_bytes`, to reduce merge fan-in. `self.reservation`/// is resized to the coalesced footprint so the caller's per-run splits stay@@ -780,22 +917,28 @@ impl ExternalSorter {.map(|_| RecordBatchMemoryCounter::new())}- /// Sorts each run into its own stream, splitting `self.reservation` between- /// them.+ /// Calculates each run's reservation size and any shared input size.///- /// Runs can share buffers, for example zero-copy slices of one larger batch,- /// and `self.reservation` covers each shared buffer once. A shared buffer- /// stays alive until the last run holding it is sorted, so if any buffers are- /// shared, the reservation for every run's input is held until all the runs- /// are sorted. Otherwise each run releases its input once it is sorted.- fn sort_run_streams(+ /// - With no shared input size (`None`), each run's size covers its input+ /// buffers and an estimated sorted copy.+ /// - With a shared input size (`Some`), that size counts all input buffers+ /// once, including buffers private to one run. Each run's size covers only+ /// its estimated sorted copy.+ ///+ /// Deduplication is local to these runs. Other operators can retain and reserve+ /// the same buffers independently, so their reservations can overlap.+ ///+ /// Runs can share buffers, for example zero-copy slices of one larger batch.+ /// Callers hold the shared input reservation until all runs are sorted.+ /// Otherwise each run holds its input charge until its own sort finishes.+ fn sort_run_sizes(&self,- runs: Vec<RecordBatch>,- ) -> Result<Vec<SendableRecordBatchStream>> {+ runs: &[RecordBatch],+ ) -> Result<(Vec<usize>, Option<usize>)> {let mut counter = self.new_buffered_batches_counter();let mut shares_buffers = false;let mut sizes = Vec::with_capacity(runs.len());- for run in &runs {+ for run in runs {let memory_size = get_record_batch_memory_size(run);let retained = counter.as_mut()@@ -804,18 +947,31 @@ impl ExternalSorter {sizes.push((retained, run.get_sliced_size()?));}- let shared_input = shares_buffers.then(|| {- let retained = sizes.iter().map(|(retained, _)| retained).sum();- Arc::new(self.reservation.split(retained))- });+ let shared_input_size =+ shares_buffers.then(|| sizes.iter().map(|(retained, _)| retained).sum());++ let sizes = sizes+ .into_iter()+ .map(|(retained, sliced)| match shared_input_size {+ Some(_) => sliced,+ None => get_reserved_bytes_for_record_batch_size(retained, sliced),+ })+ .collect();+ Ok((sizes, shared_input_size))+ }+ /// Sorts each run into its own stream, splitting `self.reservation` between+ /// them.+ fn sort_run_streams(+ &self,+ runs: Vec<RecordBatch>,+ ) -> Result<Vec<SendableRecordBatchStream>> {+ let (sizes, shared_input_size) = self.sort_run_sizes(&runs)?;+ let shared_input =+ shared_input_size.map(|size| Arc::new(self.reservation.split(size)));runs.into_iter().zip(sizes)- .map(|(run, (retained, sliced))| {- let size = match shared_input {- Some(_) => sliced,- None => get_reserved_bytes_for_record_batch_size(retained, sliced),- };+ .map(|(run, size)| {let reservation = self.reservation.split(size);self.sort_batch_stream(run, reservation, shared_input.clone())})@@ -829,17 +985,36 @@ impl ExternalSorter {/// If given, `shared_input` covers the input buffers of every run, which can/// share buffers, and `reservation` only covers sorting `batch`. This stream/// drops its handle to `shared_input` once `batch` is sorted.- /// For single-batch output cases, `reservation` will be freed immediately after sorting,- /// as the batch will be output and is expected to be reserved by the consumer of the stream.+ /// For single-batch output cases, `reservation` is freed when the batch is output,+ /// and the consumer of the stream is expected to reserve the batch./// For multi-batch output cases, `reservation` and any borrowed spill workspace/// cover the sorted output, releasing its memory as each batch is output./// (This leads to the same behaviour, as futures are only evaluated when polled by the consumer.)+ /// The whole batch is sorted on the first poll.fn sort_batch_stream(&self,batch: RecordBatch,reservation: MemoryReservation,shared_input: Option<Arc<MemoryReservation>>,) -> Result<SendableRecordBatchStream> {+ let schema = batch.schema();+ let stream =+ futures::stream::once(self.sort_run(batch, reservation, shared_input)?)+ .try_flatten();+ Ok(Box::pin(RecordBatchStreamAdapter::new(schema, stream)))+ }++ /// Sorts a batch and reserves its complete output before returning a stream.+ /// The stream releases reservations as its consumer takes each output batch.+ /// If present, `shared_input` covers the runs' shared buffers and is held+ /// until sorting finishes; `reservation` covers this run's sorting and output.+ fn sort_run(+ &self,+ batch: RecordBatch,+ reservation: MemoryReservation,+ shared_input: Option<Arc<MemoryReservation>>,+ ) -> Result<impl Future<Output = Result<SendableRecordBatchStream>> + Send + use<>>+ {let reserved_bytes = get_reserved_bytes_for_record_batch(&batch)?;match &shared_input {None => assert_eq!(reserved_bytes, reservation.size()),@@ -847,15 +1022,17 @@ impl ExternalSorter {Some(shared) => assert!(reservation.size() + shared.size() >= reserved_bytes),}- let schema = batch.schema();let expressions = self.expr.clone();let batch_size = self.batch_size;let merge_pool = Arc::clone(&self.merge_pool);+ let elapsed_compute = self.metrics.baseline.elapsed_compute().clone();- let stream = futures::stream::once(async move {+ Ok(async move {// Held until `batch` is sorted, when this block ends. The last run to// drop it releases the reservation for the runs' input.let _shared_input = shared_input;+ // Measure sorting where it runs, including on spawned tasks.+ let _timer = elapsed_compute.timer();let schema = batch.schema();// Sort the batch immediately and get all output batches@@ -899,9 +1076,6 @@ impl ExternalSorter {futures::stream::iter(batches),)) as SendableRecordBatchStream)})- .try_flatten();-- Ok(Box::pin(RecordBatchStreamAdapter::new(schema, stream)))}/// If this sort may spill, pre-allocates@@ -1117,6 +1291,15 @@ pub fn sort_batch_chunked(////// Support sorting datasets that are larger than the memory allotted/// by the memory manager, by spilling to disk.+///+/// On multithreaded Tokio runtimes, large inputs are sorted in batches on+/// separate tasks before merging. All partitions share a task limit capped by+/// both the worker count of the runtime that first performs this sorting and+/// the estimated available CPU parallelism. Clones and concurrent executions+/// of the same plan share this limit. The plan returned by+/// [`ExecutionPlan::reset_state`] starts with a fresh limit.+/// Inputs sorted in a single pass, current-thread runtimes, execution outside+/// Tokio, and TopK sorting continue sorting inline.#[derive(Debug, Clone)]pub struct SortExec {/// Input schema@@ -1138,6 +1321,9 @@ pub struct SortExec {/// If `fetch` is `Some`, this will also be set and a TopK operator may be used./// If `fetch` is `None`, this will be `None`.filter: Option<Arc<RwLock<TopKDynamicFilters>>>,+ /// Batch-sort limit shared by partitions and clones.+ /// `reset_state` returns a new plan with a fresh limit.+ sort_concurrency: Arc<OnceLock<SortConcurrency>>,}impl SortExec {@@ -1157,6 +1343,7 @@ impl SortExec {common_sort_prefix: sort_prefix,cache: Arc::new(cache),filter: None,+ sort_concurrency: Arc::new(OnceLock::new()),}}@@ -1233,6 +1420,7 @@ impl SortExec {fetch: self.fetch,cache: Arc::clone(&self.cache),filter: self.filter.clone(),+ sort_concurrency: Arc::clone(&self.sort_concurrency),}}@@ -1563,12 +1751,14 @@ impl ExecutionPlan for SortExec {.downcast_ref::<SortExec>().expect("rebuilt SortExec with new children").clone();- // Our dynamic filter and execution metrics are the state we need to reset.+ // Our dynamic filter, execution metrics and sort concurrency are the state+ // we need to reset.if let Some(filter) = &new_sort.filter {filter.read().expr().reset();}new_sort.rebuild_filter_for_current_partitioning();new_sort.metrics_set = ExecutionPlanMetricsSet::new();+ new_sort.sort_concurrency = Arc::new(OnceLock::new());Ok(Arc::new(new_sort))}@@ -1645,6 +1835,7 @@ impl ExecutionPlan for SortExec {&self.metrics_set,context.runtime_env(),)?;+ sorter.sort_concurrency = Arc::clone(&self.sort_concurrency);Ok(Box::pin(RecordBatchStreamAdapter::new(self.schema(),futures::stream::once(async move {@@ -1844,6 +2035,8 @@ impl ExecutionPlan for SortExec {// Derived plan properties, recomputed on decode.cache: _,filter,+ // Execution state, initialized on the runtime polling the decoded plan.+ sort_concurrency: _,} = self;let input = ctx.encode_child(input)?;let expr = expr@@ -2296,7 +2489,6 @@ mod tests {use super::*;use crate::coalesce_partitions::CoalescePartitionsExec;- use crate::collect;use crate::empty::EmptyExec;use crate::execution_plan::Boundedness;use crate::expressions::col;@@ -2304,8 +2496,11 @@ mod tests {use crate::statistics::StatisticsContext;use crate::test;use crate::test::TestMemoryExec;- use crate::test::exec::{BlockingExec, assert_strong_count_converges_to_zero};+ use crate::test::exec::{+ BlockingExec, TestStream, assert_strong_count_converges_to_zero,+ };use crate::test::{assert_is_pending, make_partition};+ use crate::{collect, collect_partitioned};use arrow::array::*;use arrow::compute::SortOptions;@@ -2335,6 +2530,155 @@ mod tests {use insta::assert_snapshot;use tokio::sync::Notify;+ #[tokio::test]+ async fn sort_run_records_compute_before_consumption() -> Result<()> {+ let batch = RecordBatch::try_from_iter([(+ "a",+ Arc::new(Int32Array::from_iter_values((0..4096).rev())) as ArrayRef,+ )])?;+ let sorter = ExternalSorter::new(+ 0,+ batch.schema(),+ [PhysicalSortExpr::new_default(col("a", &batch.schema())?)].into(),+ 1024,+ 0,+ 0,+ SpillCompression::Uncompressed,+ &ExecutionPlanMetricsSet::new(),+ Arc::new(RuntimeEnv::default()),+ )?;+ let elapsed_compute = sorter.metrics.baseline.elapsed_compute();+ for spawned in [false, true] {+ let reservation = sorter.reservation.new_empty();+ reservation.try_grow(get_reserved_bytes_for_record_batch(&batch)?)?;+ let before = elapsed_compute.value();+ let sort = sorter.sort_run(batch.clone(), reservation, None)?;+ assert_eq!(elapsed_compute.value(), before);++ let stream = if spawned {+ ExternalSorter::prepare_sort_run(sort, Arc::new(Semaphore::new(1)))+ .await?+ } else {+ sort.await?+ };+ assert!(elapsed_compute.value() > before);+ drop(stream);+ }+ Ok(())+ }++ #[tokio::test]+ async fn sort_preparation_holds_permit_and_releases_on_drop() {+ let permits = Arc::new(Semaphore::new(1));+ let held = permits.acquire().await.unwrap();+ let started = Arc::new(Notify::new());+ let sort = {+ let started = Arc::clone(&started);+ async move {+ started.notify_one();+ futures::future::pending::<Result<SendableRecordBatchStream>>().await+ }+ };+ let mut preparation =+ ExternalSorter::prepare_sort_run(sort, Arc::clone(&permits)).boxed();+ assert_is_pending(&mut preparation);+ assert_eq!(Handle::current().metrics().num_alive_tasks(), 0);+ drop(held);+ assert_is_pending(&mut preparation);+ started.notified().await;+ assert_eq!(permits.available_permits(), 0);+ drop(preparation);+ let _permit =+ tokio::time::timeout(std::time::Duration::from_secs(10), permits.acquire())+ .await+ .expect("cancelled preparation must release its permit")+ .unwrap();+ }++ #[tokio::test]+ async fn sort_preparation_propagates_errors_and_releases_permit() {+ let permits = Arc::new(Semaphore::new(1));+ let sort = async { Err(internal_datafusion_err!("sort failed")) };+ let error = ExternalSorter::prepare_sort_run(sort, Arc::clone(&permits))+ .await+ .err()+ .expect("preparation must propagate the sort error");+ assert!(error.to_string().contains("sort failed"), "{error}");+ assert_eq!(permits.available_permits(), 1);+ }++ #[tokio::test]+ async fn sort_preparation_restores_run_order() -> Result<()> {+ let runs = [+ record_batch!(("a", Int32, [1]), ("payload", Utf8, ["first"]))?,+ record_batch!(("a", Int32, [1]), ("payload", Utf8, ["second"]))?,+ ];+ // Completed runs arrive in reverse order; the merge needs their input order.+ let jobs = runs+ .iter()+ .cloned()+ .enumerate()+ .rev()+ .map(|(index, batch)| {+ let stream = TestStream::new(vec![batch]);+ Ok((index, Box::pin(stream) as SendableRecordBatchStream))+ });+ let streams = ExternalSorter::collect_prepared_runs(+ runs.len(),+ futures::stream::iter(jobs),+ )+ .await?;+ assert_eq!(streams.len(), runs.len());+ for (stream, expected) in streams.into_iter().zip(runs) {+ assert_eq!(stream.try_collect::<Vec<_>>().await?, vec![expected]);+ }+ Ok(())+ }++ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ async fn sort_execution_initializes_and_resets_concurrency() -> Result<()> {+ let batch = record_batch!(+ ("a", Int32, [2, 1, 2, 1]),+ ("payload", Utf8, ["b", "a", "d", "c"])+ )?;+ // Distinct payloads make equal-key ordering across runs observable.+ let runs = vec![batch.slice(0, 2), batch.slice(2, 2)];+ let input =+ TestMemoryExec::try_new_exec(&[runs.clone(), runs], batch.schema(), None)?;+ let sort = SortExec::new(+ [PhysicalSortExpr::new_default(col("a", &batch.schema())?)].into(),+ input,+ )+ .with_preserve_partitioning(true);+ let context = Arc::new(TaskContext::default().with_session_config(+ SessionConfig::new().with_sort_in_place_threshold_bytes(0),+ ));++ // Execution must initialize the cell owned by the plan, not a private sorter cell.+ let output =+ collect_partitioned(Arc::new(sort.clone()), Arc::clone(&context)).await?;+ let expected = record_batch!(+ ("a", Int32, [1, 1, 2, 2]),+ ("payload", Utf8, ["a", "c", "b", "d"])+ )?;+ assert_eq!(output.len(), 2);+ for partition in output {+ assert_eq!(concat_batches(&batch.schema(), &partition)?, expected);+ }+ assert!(sort.sort_concurrency.get().is_some());+ let reset = Arc::new(sort.clone()).reset_state()?;+ assert!(+ reset+ .downcast_ref::<SortExec>()+ .unwrap()+ .sort_concurrency+ .get()+ .is_none()+ );++ Ok(())+ }+struct PendingWriteTempFileFactory {write_started: Arc<Notify>,aborted: Arc<Notify>,diff --git a/datafusion/physical-plan/src/sorts/sort/spill_tests.rs b/datafusion/physical-plan/src/sorts/sort/spill_tests.rsindex 1314660e4e..625f96be68 100644--- a/datafusion/physical-plan/src/sorts/sort/spill_tests.rs+++ b/datafusion/physical-plan/src/sorts/sort/spill_tests.rs@@ -232,8 +232,7 @@ async fn assert_released(pool: &AdjustablePool, runtime: &RuntimeEnv) {assert_eq!(state.unchecked_over_limit, 0);}-#[tokio::test]-async fn test_spill_preserves_merge_workspace_after_limit_decreases() -> Result<()> {+async fn check_spill_preserves_merge_workspace_after_limit_decreases() -> Result<()> {let fixture = fixture()?;let pool = AdjustablePool::new(fixture.capacity);let (mut sorter, runtime) = sorter(&fixture, Arc::clone(&pool))?;@@ -266,6 +265,17 @@ async fn test_spill_preserves_merge_workspace_after_limit_decreases() -> Result<Ok(())}+#[tokio::test]+async fn test_spill_preserves_merge_workspace_after_limit_decreases() -> Result<()> {+ check_spill_preserves_merge_workspace_after_limit_decreases().await+}++#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn test_spill_preserves_merge_workspace_after_limit_decreases_multithreaded()+-> Result<()> {+ check_spill_preserves_merge_workspace_after_limit_decreases().await+}+#[tokio::test]async fn test_spill_workspace_does_not_hide_insufficient_memory() -> Result<()> {let fixture = fixture()?;@@ -520,6 +530,155 @@ async fn test_chunked_string_view_spill_can_use_workspace() -> Result<()> {check_chunked_string_view_workspace(true).await}+/// Prepared runs retain their workspace loans until merging starts. Looking+/// ahead during the merge returns enough workspace for its batch and cursors.+async fn check_sort_preparation_workspace(consume_output: bool) -> Result<()> {+ let batch_size = 1024;+ let values: ArrayRef = Arc::new(StringViewArray::from_iter_values(+ (0..4096)+ .rev()+ .map(|i| format!("row-{i:08}-{}", "x".repeat(87))),+ ));+ let batch = RecordBatch::try_from_iter([("key", values)])?;+ let ordering: LexOrdering = [PhysicalSortExpr::new_default(Arc::new(Column::new(+ "key", 0,+ )))]+ .into();+ let input_bytes = get_reserved_bytes_for_record_batch(&batch)?;+ let sorted_batches = sort_batch_chunked(&batch, &ordering, batch_size)?;+ // Each StringView output chunk retains the shared string buffers.+ // Summing the per-batch estimates counts those buffers once per chunk.+ let sorted_bytes = sorted_batches+ .iter()+ .map(get_record_batch_memory_size)+ .sum::<usize>();+ let loan_per_run = sorted_bytes - input_bytes;+ let first_bytes = get_record_batch_memory_size(&sorted_batches[0]);+ assert!(loan_per_run > first_bytes);+ // Both full loans fit before merging; only one extra batch fits beside them.+ let workspace = 2 * loan_per_run + first_bytes;+ let runtime = RuntimeEnvBuilder::new()+ .with_memory_pool(Arc::new(GreedyMemoryPool::new(2 * input_bytes + workspace)))+ .build_arc()?;+ let mut sorter = ExternalSorter::new(+ 0,+ batch.schema(),+ ordering.clone(),+ batch_size,+ workspace,+ 0,+ SpillCompression::Uncompressed,+ &ExecutionPlanMetricsSet::new(),+ Arc::clone(&runtime),+ )?;+ sorter.insert_batch(batch.clone()).await?;+ sorter.insert_batch(batch.clone()).await?;+ sorter.merge_reservation.free();+ let stream = sorter.in_mem_sort_stream(false, false).await?;+ // Verify every sort task exits before merging. Tokio can update its task+ // count shortly after the join completes.+ tokio::time::timeout(Duration::from_secs(5), async {+ while tokio::runtime::Handle::current()+ .metrics()+ .num_alive_tasks()+ != 0+ {+ tokio::task::yield_now().await;+ }+ })+ .await+ .expect("sort jobs must finish before merging");+ // Temporarily borrow the unused workspace: both runs hold their full loans,+ // leaving only the extra batch. Dropping the temporary returns it immediately.+ assert_eq!(sorter.merge_pool.borrow(usize::MAX).size(), first_bytes);+ if consume_output {+ let output: Vec<RecordBatch> = stream.try_collect().await?;+ let expected = sort_batch(+ &concat_batches(&batch.schema(), &[batch.clone(), batch.clone()])?,+ &ordering,+ None,+ )?;+ assert_eq!(concat_batches(&batch.schema(), &output)?, expected);+ } else {+ drop(stream);+ }+ drop(sorter);+ assert_eq!(runtime.memory_pool.reserved(), 0);+ Ok(())+}++#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn test_sort_preparation_retains_workspace_until_drop() -> Result<()> {+ check_sort_preparation_workspace(false).await+}++#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn test_sort_merge_lookahead_returns_cursor_workspace() -> Result<()> {+ check_sort_preparation_workspace(true).await+}++#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn test_sort_preparation_keeps_output_reserved() -> Result<()> {+ let parent = RecordBatch::try_from_iter([(+ "key",+ Arc::new(Int64Array::from_iter_values((0..8 * 4096).rev())) as ArrayRef,+ )])?;+ // All runs share one input buffer, which must be reserved only once.+ let batches = (0..8)+ .map(|run| parent.slice(run * 4096, 4096))+ .collect::<Vec<_>>();+ let ordering: LexOrdering = [PhysicalSortExpr::new_default(Arc::new(Column::new(+ "key", 0,+ )))]+ .into();+ let input_bytes = get_reserved_bytes_for_record_batch(&parent)?;+ let sorted_bytes =+ get_record_batch_memory_size(&sort_batch(&batches[0], &ordering, None)?)+ * batches.len();+ // Reserve the merge's cursor memory alongside the sorted batches.+ let merge_workspace = sorted_bytes;+ let capacity = input_bytes + merge_workspace;+ let runtime = RuntimeEnvBuilder::new()+ .with_memory_pool(Arc::new(GreedyMemoryPool::new(capacity)))+ .build_arc()?;+ let mut sorter = ExternalSorter::new(+ 0,+ batches[0].schema(),+ ordering.clone(),+ 8192,+ merge_workspace,+ 0,+ SpillCompression::Uncompressed,+ &ExecutionPlanMetricsSet::new(),+ Arc::clone(&runtime),+ )?;+ for batch in &batches {+ sorter.insert_batch(batch.clone()).await?;+ }+ sorter.merge_reservation.free();+ let stream = sorter.in_mem_sort_stream(false, false).await?;+ assert_eq!(+ runtime.memory_pool.reserved(),+ sorted_bytes + merge_workspace+ );++ // Another consumer can take the released sort scratch space, while prepared+ // batches and merge workspace remain reserved for this sort.+ let other = MemoryConsumer::new("other").register(&runtime.memory_pool);+ other.try_grow(capacity - runtime.memory_pool.reserved())?;+ let output: Vec<RecordBatch> = stream.try_collect().await?;+ let expected = sort_batch(+ &concat_batches(&batches[0].schema(), &batches)?,+ &ordering,+ None,+ )?;+ assert_eq!(concat_batches(&batches[0].schema(), &output)?, expected);+ drop(sorter);+ drop(other);+ assert_eq!(runtime.memory_pool.reserved(), 0);+ Ok(())+}+#[tokio::test]async fn test_single_batch_spill_preserves_workspace_after_limit_decreases() -> Result<()>{@@ -1014,7 +1173,7 @@ async fn test_single_batch_spill_returns_live_workspace_loan_on_drop() -> Resultsorter.merge_reservation.free();let merge_pool = Arc::clone(&sorter.merge_pool);assert_eq!(sorter.in_mem_batches.len(), 1);- let mut stream = sorter.in_mem_sort_stream(false, false)?;+ let mut stream = sorter.in_mem_sort_stream(false, false).await?;assert!(sorter.in_mem_batches.is_empty());drop(sorter);