Skip to content

Apache DataFusion SQL Query Engine

fix: bound parallel batch sorting in SortExec (#26242)

Geoffrey Claude676c003146e2
diff --git a/datafusion/core/tests/fuzz_cases/sort_query_fuzz.rs b/datafusion/core/tests/fuzz_cases/sort_query_fuzz.rs
index 149630d910..b3aec96b23 100644
--- a/datafusion/core/tests/fuzz_cases/sort_query_fuzz.rs
+++ b/datafusion/core/tests/fuzz_cases/sort_query_fuzz.rs
@@ -9,6 +9,7 @@ use std::fmt::Write as _;
use std::sync::Arc;
use arrow::array::RecordBatch;
+use arrow::compute::concat_batches;
use arrow_schema::SchemaRef;
use datafusion::datasource::MemTable;
use datafusion::prelude::{SessionConfig, SessionContext};
@@ -24,8 +25,6 @@ use rand::Rng;
use rand::prelude::IndexedRandom;
use rand::{SeedableRng, rngs::StdRng};
-use crate::fuzz_cases::aggregation_fuzzer::check_equality_of_batches;
-
use super::aggregation_fuzzer::ColumnDescr;
use super::record_batch_generator::{RecordBatchGenerator, get_supported_types_columns};
@@ -186,7 +185,7 @@ impl SortQueryFuzzer {
let init_seed = self.runner_rng.random();
for query_i in 0..self.queries_per_round {
let query_seed = self.runner_rng.random();
- let mut expected_results: Option<Vec<RecordBatch>> = None; // use first config's result as the expected result
+ let mut expected_results: Option<RecordBatch> = None; // use first config's result as the expected result
for config_i in 0..self.config_variations_per_query {
if self.should_stop_due_to_time_limit(start_time, round, query_i) {
return Ok(());
@@ -204,18 +203,16 @@ impl SortQueryFuzzer {
let results = self
.test_gen
- .fuzzer_run(init_seed, query_seed, config_seed)
+ .fuzzer_run(init_seed, query_seed, config_seed, config_i == 0)
.await?;
println!("\n"); // Separator between tested runs
if expected_results.is_none() {
expected_results = Some(results);
} else if let Some(ref expected) = expected_results {
- // `fuzzer_run` might append `LIMIT k` to either the
- // expected or actual query. The number of results is
- // checked inside `fuzzer_run()`. Here we only check
- // that the first k rows of each result are consistent.
- check_equality_of_batches(expected, &results).unwrap();
+ // Every projected column is an ORDER BY key, so tied
+ // rows have identical output and can be compared in order.
+ assert_eq!(expected, &results);
} else {
unreachable!();
}
@@ -467,6 +464,7 @@ impl SortFuzzerTestGenerator {
&self,
rng_seed: u64,
with_memory_limit: bool,
+ sort_in_place_threshold_override: Option<usize>,
) -> Result<SessionContext> {
let mut rng = StdRng::seed_from_u64(rng_seed);
let init_state = self.dataset_state.as_ref().unwrap();
@@ -495,6 +493,8 @@ impl SortFuzzerTestGenerator {
let dataset_size = self.dataset_state.as_ref().unwrap().dataset_size;
rng.random_range(0..=dataset_size * 2_usize)
};
+ let sort_in_place_threshold_bytes =
+ sort_in_place_threshold_override.unwrap_or(sort_in_place_threshold_bytes);
// Set up strings for printing
let memory_limit_str = if with_memory_limit {
@@ -526,6 +526,8 @@ impl SortFuzzerTestGenerator {
let config = SessionConfig::new()
.with_target_partitions(num_partitions)
.with_batch_size(init_state.approx_batch_num_rows / 2)
+ // Keep small input batches separate when comparing the two sort paths.
+ .with_coalesce_batches(sort_in_place_threshold_override.is_none())
.with_sort_spill_reservation_bytes(sort_spill_reservation_bytes)
.with_sort_in_place_threshold_bytes(sort_in_place_threshold_bytes);
@@ -556,7 +558,8 @@ impl SortFuzzerTestGenerator {
dataset_seed: u64,
query_seed: u64,
config_seed: u64,
- ) -> Result<Vec<RecordBatch>> {
+ compare_sort_paths: bool,
+ ) -> Result<RecordBatch> {
self.init_partitioned_staggered_batches(dataset_seed);
let (query_str, limit) = self.generate_random_query(query_seed);
println!(" Query:");
@@ -569,12 +572,13 @@ impl SortFuzzerTestGenerator {
// 2. Memory limiting is enabled in the test generator config
let with_mem_limit = !query_str.contains("LIMIT") && self.set_memory_limit;
- let ctx = self.generate_random_config(config_seed, with_mem_limit)?;
+ let ctx = self.generate_random_config(config_seed, with_mem_limit, None)?;
let df = ctx.sql(&query_str).await.unwrap();
- let results = df.collect().await.unwrap();
+ let schema = Arc::clone(df.schema().inner());
+ let results = concat_batches(&schema, &df.collect().await?)?;
// ==== Check the result size is consistent with the limit ====
- let result_num_rows = results.iter().map(|batch| batch.num_rows()).sum::<usize>();
+ let result_num_rows = results.num_rows();
let dataset_size = self.dataset_state.as_ref().unwrap().dataset_size;
if let Some(limit) = limit {
@@ -582,6 +586,21 @@ impl SortFuzzerTestGenerator {
assert_eq!(result_num_rows, expected_num_rows);
}
+ if compare_sort_paths && limit.is_none() && !with_mem_limit {
+ // Run the same query inline and with bounded parallel preparation.
+ // An unbounded memory pool allows the inline path to concatenate all runs.
+ for threshold in [usize::MAX, 0] {
+ let ctx =
+ self.generate_random_config(config_seed, false, Some(threshold))?;
+ let actual = ctx.sql(&query_str).await?.collect().await?;
+ assert_eq!(
+ results,
+ concat_batches(&schema, &actual)?,
+ "sort_in_place_threshold_bytes={threshold}"
+ );
+ }
+ }
+
Ok(results)
}
}
@@ -603,8 +622,8 @@ mod test {
gen_seed,
);
- let res1 = test_generator.fuzzer_run(1, 2, 3).await.unwrap();
- let res2 = test_generator.fuzzer_run(1, 2, 3).await.unwrap();
- check_equality_of_batches(&res1, &res2).unwrap();
+ let res1 = test_generator.fuzzer_run(1, 2, 3, false).await.unwrap();
+ let res2 = test_generator.fuzzer_run(1, 2, 3, false).await.unwrap();
+ assert_eq!(res1, res2);
}
}
diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs
index 90f9488be2..8663515c07 100644
--- a/datafusion/physical-plan/src/sorts/sort.rs
+++ b/datafusion/physical-plan/src/sorts/sort.rs
@@ -8,11 +8,11 @@
use std::fmt;
use std::fmt::{Debug, Formatter};
-use std::sync::Arc;
+use std::future::Future;
+use std::sync::{Arc, OnceLock};
use parking_lot::RwLock;
-use crate::common::spawn_buffered;
use crate::execution_plan::{
Boundedness, CardinalityEffect, EmissionType, has_same_children_properties,
replace_children_if_necessary,
@@ -25,7 +25,8 @@ use crate::filter_pushdown::{
};
use crate::limit::LimitStream;
use crate::metrics::{
- BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet, SpillMetrics,
+ BaselineMetrics, ElapsedComputeFutureExt, ExecutionPlanMetricsSet, MetricsSet,
+ SpillMetrics,
};
use crate::projection::{ProjectionExec, make_with_child, update_ordering};
use crate::sorts::IncrementalSortIterator;
@@ -49,11 +50,14 @@ use arrow::compute::{concat_batches, lexsort_to_indices, take_arrays};
use arrow::datatypes::{DataType, SchemaRef};
use datafusion_common::config::SpillCompression;
use datafusion_common::tree_node::TreeNodeRecursion;
-use datafusion_common::utils::memory::RecordBatchMemoryCounter;
+use datafusion_common::utils::{
+ get_available_parallelism, memory::RecordBatchMemoryCounter,
+};
use datafusion_common::{
DataFusionError, Result, assert_or_internal_err, internal_datafusion_err,
unwrap_or_internal_err,
};
+use datafusion_common_runtime::SpawnedTask;
use datafusion_execution::TaskContext;
use datafusion_execution::memory_pool::{
MemoryConsumer, MemoryPool, MemoryReservation, MergeMemoryPool,
@@ -63,8 +67,10 @@ use datafusion_physical_expr::LexOrdering;
use datafusion_physical_expr::PhysicalExpr;
use datafusion_physical_expr::expressions::{DynamicFilterPhysicalExpr, lit};
-use futures::{StreamExt, TryStreamExt};
+use futures::{Stream, StreamExt, TryStreamExt};
use log::{debug, trace};
+use tokio::runtime::{Handle, RuntimeFlavor};
+use tokio::sync::Semaphore;
#[cfg(test)]
mod spill_tests;
@@ -85,6 +91,43 @@ impl ExternalSorterMetrics {
}
}
+/// Limits concurrent batch sorting across the partitions of one [`SortExec`].
+#[derive(Debug, Clone)]
+struct SortConcurrency {
+ /// Each partition can use the full concurrency budget, but queues at most this
+ /// many jobs so it cannot put all its runs ahead of other partitions.
+ per_partition_limit: usize,
+ /// Shared budget: a job holds a permit until its run has been sorted.
+ permits: Arc<Semaphore>,
+}
+
+impl SortConcurrency {
+ /// Returns the shared limit on multithreaded Tokio runtimes, initialized
+ /// on first use.
+ fn for_current_runtime(cell: &OnceLock<Self>) -> Option<Self> {
+ let handle = Handle::try_current().ok()?;
+ if handle.runtime_flavor() != RuntimeFlavor::MultiThread {
+ return None;
+ }
+ Some(
+ cell.get_or_init(|| {
+ // These jobs are CPU-bound. Cap concurrency by the worker count
+ // to avoid queuing jobs that cannot run, and by estimated CPU
+ // parallelism to avoid oversubscribing CPUs on larger runtimes.
+ let limit = handle
+ .metrics()
+ .num_workers()
+ .min(get_available_parallelism());
+ Self {
+ per_partition_limit: limit,
+ permits: Arc::new(Semaphore::new(limit)),
+ }
+ })
+ .clone(),
+ )
+ }
+}
+
/// Sorts an arbitrary sized, unsorted, stream of [`RecordBatch`]es to
/// a total order. Depending on the input size and memory manager
/// configuration, writes intermediate results to disk ("spills")
@@ -266,6 +309,9 @@ struct ExternalSorter {
/// How much memory to reserve for performing in-memory sort/merges
/// prior to spilling.
sort_spill_reservation_bytes: usize,
+ /// Shared by the owning SortExec's partitions. Initialized on the runtime
+ /// that polls the sort, which may differ from the one calling `execute`.
+ sort_concurrency: Arc<OnceLock<SortConcurrency>>,
}
impl ExternalSorter {
@@ -326,6 +372,7 @@ impl ExternalSorter {
batch_size,
sort_spill_reservation_bytes,
sort_in_place_threshold_bytes,
+ sort_concurrency: Arc::new(OnceLock::new()),
})
}
@@ -390,7 +437,7 @@ impl ExternalSorter {
// workspace so another sorter can start while this stream is alive.
self.merge_pool.release_unused();
self.merge_reservation.free();
- self.in_mem_sort_stream(true, true)
+ self.in_mem_sort_stream(true, true).await
}
}
@@ -510,11 +557,13 @@ impl ExternalSorter {
// make spilling fail if another task consumes it or our share shrinks.
self.merge_reservation.free();
- let mut sorted_stream = self.in_mem_sort_stream(
- false,
- // No coalescing on the spill path: it raises per-run peak memory.
- false,
- )?;
+ let mut sorted_stream = self
+ .in_mem_sort_stream(
+ false,
+ // No coalescing on the spill path: it raises per-run peak memory.
+ false,
+ )
+ .await?;
// After `in_mem_sort_stream()` is constructed, all `in_mem_batches` is taken
// to construct a globally sorted stream.
assert_or_internal_err!(
@@ -633,7 +682,7 @@ impl ExternalSorter {
/// ```
/// `coalesce_runs` merges buffered batches into fewer, larger sorted runs to
/// reduce merge fan-in. Disabled on the spill path to keep peak memory low.
- fn in_mem_sort_stream(
+ async fn in_mem_sort_stream(
&mut self,
is_output_stream: bool,
coalesce_runs: bool,
@@ -650,13 +699,11 @@ impl ExternalSorter {
// The elapsed compute timer is updated when the value is dropped.
// There is no need for an explicit call to drop.
let elapsed_compute = self.metrics.baseline.elapsed_compute().clone();
- let _timer = elapsed_compute.timer();
+ let mut timer = elapsed_compute.timer();
- // Please pay attention that any operation inside of `in_mem_sort_stream` will
- // not perform any memory reservation. This is for avoiding the need of handling
- // reservation failure and spilling in the middle of the sort/merge. The memory
- // space for batches produced by the resulting stream will be reserved by the
- // consumer of the stream.
+ // Sorting can resize reservations and borrow spill workspace.
+ // Prepared output stays reserved until each batch is read from its run stream.
+ // The consumer of the merged stream reserves its final output.
if self.in_mem_batches.len() == 1 {
let batch = self.in_mem_batches.swap_remove(0);
@@ -694,11 +741,20 @@ impl ExternalSorter {
batches
};
- let streams = self
- .sort_run_streams(runs)?
- .into_iter()
- .map(|stream| spawn_buffered(stream, 1))
- .collect();
+ let concurrency = SortConcurrency::for_current_runtime(&self.sort_concurrency);
+ let streams = if let Some(concurrency) = concurrency {
+ // Time this task's setup and collection work, excluding waits for jobs.
+ timer.stop();
+ let streams = self
+ .prepare_sort_runs(runs, &concurrency)
+ .with_elapsed_compute(elapsed_compute.clone())
+ .await?;
+ timer.restart();
+ streams
+ } else {
+ // Outside a multithreaded runtime, sort each run when the merge polls it.
+ self.sort_run_streams(runs)?
+ };
StreamingMergeBuilder::new()
.with_streams(streams)
@@ -715,6 +771,87 @@ impl ExternalSorter {
.build()
}
+ /// Sort runs concurrently, returning their streams in input order.
+ async fn prepare_sort_runs(
+ &mut self,
+ runs: Vec<RecordBatch>,
+ concurrency: &SortConcurrency,
+ ) -> Result<Vec<SendableRecordBatchStream>> {
+ let run_count = runs.len();
+ let (sizes, shared_input_size) = self.sort_run_sizes(&runs)?;
+ let shared_input =
+ shared_input_size.map(|size| Arc::new(self.reservation.split(size)));
+ let jobs =
+ runs.into_iter()
+ .zip(sizes)
+ .enumerate()
+ .map(move |(index, (batch, size))| {
+ let reservation = self.reservation.split(size);
+ let sort = self.sort_run(batch, reservation, shared_input.clone())?;
+ let permits = Arc::clone(&concurrency.permits);
+ Ok::<_, DataFusionError>(async move {
+ let stream = Self::prepare_sort_run(sort, permits).await?;
+ Ok((index, stream))
+ })
+ });
+ // Ordered buffering would keep completed runs behind a slow earlier run,
+ // filling the window and preventing new jobs from starting.
+ let jobs = futures::stream::iter(jobs)
+ .try_buffer_unordered(concurrency.per_partition_limit);
+ Self::collect_prepared_runs(run_count, jobs).await
+ }
+
+ /// Restore input order so job completion order does not affect the merge's output.
+ async fn collect_prepared_runs(
+ run_count: usize,
+ mut jobs: impl Stream<Item = Result<(usize, SendableRecordBatchStream)>> + Unpin,
+ ) -> Result<Vec<SendableRecordBatchStream>> {
+ let mut streams: Vec<_> = (0..run_count).map(|_| None).collect();
+ while let Some((index, stream)) = jobs.try_next().await? {
+ streams[index] = Some(stream);
+ }
+ streams
+ .into_iter()
+ .collect::<Option<Vec<_>>>()
+ .ok_or_else(|| internal_datafusion_err!("Missing prepared sort run"))
+ }
+
+ /// Sort one run under a shared permit, retaining its output reservations.
+ ///
+ /// The sort must finish independently of other runs and partitions so its
+ /// permit can be returned before the merge starts consuming output.
+ async fn prepare_sort_run(
+ sort: impl Future<Output = Result<SendableRecordBatchStream>> + Send + 'static,
+ permits: Arc<Semaphore>,
+ ) -> Result<SendableRecordBatchStream> {
+ // Acquire before spawning so queued runs do not become Tokio tasks.
+ let permit = permits
+ .acquire_owned()
+ .await
+ .map_err(|e| internal_datafusion_err!("Sort concurrency closed: {e}"))?;
+ let stream = SpawnedTask::spawn(async move {
+ // Keep the permit until this job exits, even if its caller is cancelled.
+ let _permit = permit;
+ sort.await
+ })
+ .join_unwind()
+ .await
+ .map_err(|e| DataFusionError::ExecutionJoin(Box::new(e)))??;
+ Ok(Self::with_lookahead(stream))
+ }
+
+ /// [`Self::sort_run`] materializes all output and releases each batch's
+ /// reservation when that batch is pulled. Reading one batch ahead returns
+ /// extra spill workspace for the merge's current batch and cursor.
+ /// Each run can hold one prefetched batch whose reservation is already released.
+ fn with_lookahead(stream: SendableRecordBatchStream) -> SendableRecordBatchStream {
+ let schema = stream.schema();
+ // Buffer the current batch and one ahead to return borrowed spill workspace
+ // before the merge processes the current batch.
+ let batches = stream.map(std::future::ready).buffered(2);
+ Box::pin(RecordBatchStreamAdapter::new(schema, batches))
+ }
+
/// Concatenates `batches` into fewer, larger runs, each bounded by
/// `sort_in_place_threshold_bytes`, to reduce merge fan-in. `self.reservation`
/// is resized to the coalesced footprint so the caller's per-run splits stay
@@ -780,22 +917,28 @@ impl ExternalSorter {
.map(|_| RecordBatchMemoryCounter::new())
}
- /// Sorts each run into its own stream, splitting `self.reservation` between
- /// them.
+ /// Calculates each run's reservation size and any shared input size.
///
- /// Runs can share buffers, for example zero-copy slices of one larger batch,
- /// and `self.reservation` covers each shared buffer once. A shared buffer
- /// stays alive until the last run holding it is sorted, so if any buffers are
- /// shared, the reservation for every run's input is held until all the runs
- /// are sorted. Otherwise each run releases its input once it is sorted.
- fn sort_run_streams(
+ /// - With no shared input size (`None`), each run's size covers its input
+ /// buffers and an estimated sorted copy.
+ /// - With a shared input size (`Some`), that size counts all input buffers
+ /// once, including buffers private to one run. Each run's size covers only
+ /// its estimated sorted copy.
+ ///
+ /// Deduplication is local to these runs. Other operators can retain and reserve
+ /// the same buffers independently, so their reservations can overlap.
+ ///
+ /// Runs can share buffers, for example zero-copy slices of one larger batch.
+ /// Callers hold the shared input reservation until all runs are sorted.
+ /// Otherwise each run holds its input charge until its own sort finishes.
+ fn sort_run_sizes(
&self,
- runs: Vec<RecordBatch>,
- ) -> Result<Vec<SendableRecordBatchStream>> {
+ runs: &[RecordBatch],
+ ) -> Result<(Vec<usize>, Option<usize>)> {
let mut counter = self.new_buffered_batches_counter();
let mut shares_buffers = false;
let mut sizes = Vec::with_capacity(runs.len());
- for run in &runs {
+ for run in runs {
let memory_size = get_record_batch_memory_size(run);
let retained = counter
.as_mut()
@@ -804,18 +947,31 @@ impl ExternalSorter {
sizes.push((retained, run.get_sliced_size()?));
}
- let shared_input = shares_buffers.then(|| {
- let retained = sizes.iter().map(|(retained, _)| retained).sum();
- Arc::new(self.reservation.split(retained))
- });
+ let shared_input_size =
+ shares_buffers.then(|| sizes.iter().map(|(retained, _)| retained).sum());
+
+ let sizes = sizes
+ .into_iter()
+ .map(|(retained, sliced)| match shared_input_size {
+ Some(_) => sliced,
+ None => get_reserved_bytes_for_record_batch_size(retained, sliced),
+ })
+ .collect();
+ Ok((sizes, shared_input_size))
+ }
+ /// Sorts each run into its own stream, splitting `self.reservation` between
+ /// them.
+ fn sort_run_streams(
+ &self,
+ runs: Vec<RecordBatch>,
+ ) -> Result<Vec<SendableRecordBatchStream>> {
+ let (sizes, shared_input_size) = self.sort_run_sizes(&runs)?;
+ let shared_input =
+ shared_input_size.map(|size| Arc::new(self.reservation.split(size)));
runs.into_iter()
.zip(sizes)
- .map(|(run, (retained, sliced))| {
- let size = match shared_input {
- Some(_) => sliced,
- None => get_reserved_bytes_for_record_batch_size(retained, sliced),
- };
+ .map(|(run, size)| {
let reservation = self.reservation.split(size);
self.sort_batch_stream(run, reservation, shared_input.clone())
})
@@ -829,17 +985,36 @@ impl ExternalSorter {
/// If given, `shared_input` covers the input buffers of every run, which can
/// share buffers, and `reservation` only covers sorting `batch`. This stream
/// drops its handle to `shared_input` once `batch` is sorted.
- /// For single-batch output cases, `reservation` will be freed immediately after sorting,
- /// as the batch will be output and is expected to be reserved by the consumer of the stream.
+ /// For single-batch output cases, `reservation` is freed when the batch is output,
+ /// and the consumer of the stream is expected to reserve the batch.
/// For multi-batch output cases, `reservation` and any borrowed spill workspace
/// cover the sorted output, releasing its memory as each batch is output.
/// (This leads to the same behaviour, as futures are only evaluated when polled by the consumer.)
+ /// The whole batch is sorted on the first poll.
fn sort_batch_stream(
&self,
batch: RecordBatch,
reservation: MemoryReservation,
shared_input: Option<Arc<MemoryReservation>>,
) -> Result<SendableRecordBatchStream> {
+ let schema = batch.schema();
+ let stream =
+ futures::stream::once(self.sort_run(batch, reservation, shared_input)?)
+ .try_flatten();
+ Ok(Box::pin(RecordBatchStreamAdapter::new(schema, stream)))
+ }
+
+ /// Sorts a batch and reserves its complete output before returning a stream.
+ /// The stream releases reservations as its consumer takes each output batch.
+ /// If present, `shared_input` covers the runs' shared buffers and is held
+ /// until sorting finishes; `reservation` covers this run's sorting and output.
+ fn sort_run(
+ &self,
+ batch: RecordBatch,
+ reservation: MemoryReservation,
+ shared_input: Option<Arc<MemoryReservation>>,
+ ) -> Result<impl Future<Output = Result<SendableRecordBatchStream>> + Send + use<>>
+ {
let reserved_bytes = get_reserved_bytes_for_record_batch(&batch)?;
match &shared_input {
None => assert_eq!(reserved_bytes, reservation.size()),
@@ -847,15 +1022,17 @@ impl ExternalSorter {
Some(shared) => assert!(reservation.size() + shared.size() >= reserved_bytes),
}
- let schema = batch.schema();
let expressions = self.expr.clone();
let batch_size = self.batch_size;
let merge_pool = Arc::clone(&self.merge_pool);
+ let elapsed_compute = self.metrics.baseline.elapsed_compute().clone();
- let stream = futures::stream::once(async move {
+ Ok(async move {
// Held until `batch` is sorted, when this block ends. The last run to
// drop it releases the reservation for the runs' input.
let _shared_input = shared_input;
+ // Measure sorting where it runs, including on spawned tasks.
+ let _timer = elapsed_compute.timer();
let schema = batch.schema();
// Sort the batch immediately and get all output batches
@@ -899,9 +1076,6 @@ impl ExternalSorter {
futures::stream::iter(batches),
)) as SendableRecordBatchStream)
})
- .try_flatten();
-
- Ok(Box::pin(RecordBatchStreamAdapter::new(schema, stream)))
}
/// If this sort may spill, pre-allocates
@@ -1117,6 +1291,15 @@ pub fn sort_batch_chunked(
///
/// Support sorting datasets that are larger than the memory allotted
/// by the memory manager, by spilling to disk.
+///
+/// On multithreaded Tokio runtimes, large inputs are sorted in batches on
+/// separate tasks before merging. All partitions share a task limit capped by
+/// both the worker count of the runtime that first performs this sorting and
+/// the estimated available CPU parallelism. Clones and concurrent executions
+/// of the same plan share this limit. The plan returned by
+/// [`ExecutionPlan::reset_state`] starts with a fresh limit.
+/// Inputs sorted in a single pass, current-thread runtimes, execution outside
+/// Tokio, and TopK sorting continue sorting inline.
#[derive(Debug, Clone)]
pub struct SortExec {
/// Input schema
@@ -1138,6 +1321,9 @@ pub struct SortExec {
/// If `fetch` is `Some`, this will also be set and a TopK operator may be used.
/// If `fetch` is `None`, this will be `None`.
filter: Option<Arc<RwLock<TopKDynamicFilters>>>,
+ /// Batch-sort limit shared by partitions and clones.
+ /// `reset_state` returns a new plan with a fresh limit.
+ sort_concurrency: Arc<OnceLock<SortConcurrency>>,
}
impl SortExec {
@@ -1157,6 +1343,7 @@ impl SortExec {
common_sort_prefix: sort_prefix,
cache: Arc::new(cache),
filter: None,
+ sort_concurrency: Arc::new(OnceLock::new()),
}
}
@@ -1233,6 +1420,7 @@ impl SortExec {
fetch: self.fetch,
cache: Arc::clone(&self.cache),
filter: self.filter.clone(),
+ sort_concurrency: Arc::clone(&self.sort_concurrency),
}
}
@@ -1563,12 +1751,14 @@ impl ExecutionPlan for SortExec {
.downcast_ref::<SortExec>()
.expect("rebuilt SortExec with new children")
.clone();
- // Our dynamic filter and execution metrics are the state we need to reset.
+ // Our dynamic filter, execution metrics and sort concurrency are the state
+ // we need to reset.
if let Some(filter) = &new_sort.filter {
filter.read().expr().reset();
}
new_sort.rebuild_filter_for_current_partitioning();
new_sort.metrics_set = ExecutionPlanMetricsSet::new();
+ new_sort.sort_concurrency = Arc::new(OnceLock::new());
Ok(Arc::new(new_sort))
}
@@ -1645,6 +1835,7 @@ impl ExecutionPlan for SortExec {
&self.metrics_set,
context.runtime_env(),
)?;
+ sorter.sort_concurrency = Arc::clone(&self.sort_concurrency);
Ok(Box::pin(RecordBatchStreamAdapter::new(
self.schema(),
futures::stream::once(async move {
@@ -1844,6 +2035,8 @@ impl ExecutionPlan for SortExec {
// Derived plan properties, recomputed on decode.
cache: _,
filter,
+ // Execution state, initialized on the runtime polling the decoded plan.
+ sort_concurrency: _,
} = self;
let input = ctx.encode_child(input)?;
let expr = expr
@@ -2296,7 +2489,6 @@ mod tests {
use super::*;
use crate::coalesce_partitions::CoalescePartitionsExec;
- use crate::collect;
use crate::empty::EmptyExec;
use crate::execution_plan::Boundedness;
use crate::expressions::col;
@@ -2304,8 +2496,11 @@ mod tests {
use crate::statistics::StatisticsContext;
use crate::test;
use crate::test::TestMemoryExec;
- use crate::test::exec::{BlockingExec, assert_strong_count_converges_to_zero};
+ use crate::test::exec::{
+ BlockingExec, TestStream, assert_strong_count_converges_to_zero,
+ };
use crate::test::{assert_is_pending, make_partition};
+ use crate::{collect, collect_partitioned};
use arrow::array::*;
use arrow::compute::SortOptions;
@@ -2335,6 +2530,155 @@ mod tests {
use insta::assert_snapshot;
use tokio::sync::Notify;
+ #[tokio::test]
+ async fn sort_run_records_compute_before_consumption() -> Result<()> {
+ let batch = RecordBatch::try_from_iter([(
+ "a",
+ Arc::new(Int32Array::from_iter_values((0..4096).rev())) as ArrayRef,
+ )])?;
+ let sorter = ExternalSorter::new(
+ 0,
+ batch.schema(),
+ [PhysicalSortExpr::new_default(col("a", &batch.schema())?)].into(),
+ 1024,
+ 0,
+ 0,
+ SpillCompression::Uncompressed,
+ &ExecutionPlanMetricsSet::new(),
+ Arc::new(RuntimeEnv::default()),
+ )?;
+ let elapsed_compute = sorter.metrics.baseline.elapsed_compute();
+ for spawned in [false, true] {
+ let reservation = sorter.reservation.new_empty();
+ reservation.try_grow(get_reserved_bytes_for_record_batch(&batch)?)?;
+ let before = elapsed_compute.value();
+ let sort = sorter.sort_run(batch.clone(), reservation, None)?;
+ assert_eq!(elapsed_compute.value(), before);
+
+ let stream = if spawned {
+ ExternalSorter::prepare_sort_run(sort, Arc::new(Semaphore::new(1)))
+ .await?
+ } else {
+ sort.await?
+ };
+ assert!(elapsed_compute.value() > before);
+ drop(stream);
+ }
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn sort_preparation_holds_permit_and_releases_on_drop() {
+ let permits = Arc::new(Semaphore::new(1));
+ let held = permits.acquire().await.unwrap();
+ let started = Arc::new(Notify::new());
+ let sort = {
+ let started = Arc::clone(&started);
+ async move {
+ started.notify_one();
+ futures::future::pending::<Result<SendableRecordBatchStream>>().await
+ }
+ };
+ let mut preparation =
+ ExternalSorter::prepare_sort_run(sort, Arc::clone(&permits)).boxed();
+ assert_is_pending(&mut preparation);
+ assert_eq!(Handle::current().metrics().num_alive_tasks(), 0);
+ drop(held);
+ assert_is_pending(&mut preparation);
+ started.notified().await;
+ assert_eq!(permits.available_permits(), 0);
+ drop(preparation);
+ let _permit =
+ tokio::time::timeout(std::time::Duration::from_secs(10), permits.acquire())
+ .await
+ .expect("cancelled preparation must release its permit")
+ .unwrap();
+ }
+
+ #[tokio::test]
+ async fn sort_preparation_propagates_errors_and_releases_permit() {
+ let permits = Arc::new(Semaphore::new(1));
+ let sort = async { Err(internal_datafusion_err!("sort failed")) };
+ let error = ExternalSorter::prepare_sort_run(sort, Arc::clone(&permits))
+ .await
+ .err()
+ .expect("preparation must propagate the sort error");
+ assert!(error.to_string().contains("sort failed"), "{error}");
+ assert_eq!(permits.available_permits(), 1);
+ }
+
+ #[tokio::test]
+ async fn sort_preparation_restores_run_order() -> Result<()> {
+ let runs = [
+ record_batch!(("a", Int32, [1]), ("payload", Utf8, ["first"]))?,
+ record_batch!(("a", Int32, [1]), ("payload", Utf8, ["second"]))?,
+ ];
+ // Completed runs arrive in reverse order; the merge needs their input order.
+ let jobs = runs
+ .iter()
+ .cloned()
+ .enumerate()
+ .rev()
+ .map(|(index, batch)| {
+ let stream = TestStream::new(vec![batch]);
+ Ok((index, Box::pin(stream) as SendableRecordBatchStream))
+ });
+ let streams = ExternalSorter::collect_prepared_runs(
+ runs.len(),
+ futures::stream::iter(jobs),
+ )
+ .await?;
+ assert_eq!(streams.len(), runs.len());
+ for (stream, expected) in streams.into_iter().zip(runs) {
+ assert_eq!(stream.try_collect::<Vec<_>>().await?, vec![expected]);
+ }
+ Ok(())
+ }
+
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn sort_execution_initializes_and_resets_concurrency() -> Result<()> {
+ let batch = record_batch!(
+ ("a", Int32, [2, 1, 2, 1]),
+ ("payload", Utf8, ["b", "a", "d", "c"])
+ )?;
+ // Distinct payloads make equal-key ordering across runs observable.
+ let runs = vec![batch.slice(0, 2), batch.slice(2, 2)];
+ let input =
+ TestMemoryExec::try_new_exec(&[runs.clone(), runs], batch.schema(), None)?;
+ let sort = SortExec::new(
+ [PhysicalSortExpr::new_default(col("a", &batch.schema())?)].into(),
+ input,
+ )
+ .with_preserve_partitioning(true);
+ let context = Arc::new(TaskContext::default().with_session_config(
+ SessionConfig::new().with_sort_in_place_threshold_bytes(0),
+ ));
+
+ // Execution must initialize the cell owned by the plan, not a private sorter cell.
+ let output =
+ collect_partitioned(Arc::new(sort.clone()), Arc::clone(&context)).await?;
+ let expected = record_batch!(
+ ("a", Int32, [1, 1, 2, 2]),
+ ("payload", Utf8, ["a", "c", "b", "d"])
+ )?;
+ assert_eq!(output.len(), 2);
+ for partition in output {
+ assert_eq!(concat_batches(&batch.schema(), &partition)?, expected);
+ }
+ assert!(sort.sort_concurrency.get().is_some());
+ let reset = Arc::new(sort.clone()).reset_state()?;
+ assert!(
+ reset
+ .downcast_ref::<SortExec>()
+ .unwrap()
+ .sort_concurrency
+ .get()
+ .is_none()
+ );
+
+ Ok(())
+ }
+
struct PendingWriteTempFileFactory {
write_started: Arc<Notify>,
aborted: Arc<Notify>,
diff --git a/datafusion/physical-plan/src/sorts/sort/spill_tests.rs b/datafusion/physical-plan/src/sorts/sort/spill_tests.rs
index 1314660e4e..625f96be68 100644
--- a/datafusion/physical-plan/src/sorts/sort/spill_tests.rs
+++ b/datafusion/physical-plan/src/sorts/sort/spill_tests.rs
@@ -232,8 +232,7 @@ async fn assert_released(pool: &AdjustablePool, runtime: &RuntimeEnv) {
assert_eq!(state.unchecked_over_limit, 0);
}
-#[tokio::test]
-async fn test_spill_preserves_merge_workspace_after_limit_decreases() -> Result<()> {
+async fn check_spill_preserves_merge_workspace_after_limit_decreases() -> Result<()> {
let fixture = fixture()?;
let pool = AdjustablePool::new(fixture.capacity);
let (mut sorter, runtime) = sorter(&fixture, Arc::clone(&pool))?;
@@ -266,6 +265,17 @@ async fn test_spill_preserves_merge_workspace_after_limit_decreases() -> Result<
Ok(())
}
+#[tokio::test]
+async fn test_spill_preserves_merge_workspace_after_limit_decreases() -> Result<()> {
+ check_spill_preserves_merge_workspace_after_limit_decreases().await
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_spill_preserves_merge_workspace_after_limit_decreases_multithreaded()
+-> Result<()> {
+ check_spill_preserves_merge_workspace_after_limit_decreases().await
+}
+
#[tokio::test]
async fn test_spill_workspace_does_not_hide_insufficient_memory() -> Result<()> {
let fixture = fixture()?;
@@ -520,6 +530,155 @@ async fn test_chunked_string_view_spill_can_use_workspace() -> Result<()> {
check_chunked_string_view_workspace(true).await
}
+/// Prepared runs retain their workspace loans until merging starts. Looking
+/// ahead during the merge returns enough workspace for its batch and cursors.
+async fn check_sort_preparation_workspace(consume_output: bool) -> Result<()> {
+ let batch_size = 1024;
+ let values: ArrayRef = Arc::new(StringViewArray::from_iter_values(
+ (0..4096)
+ .rev()
+ .map(|i| format!("row-{i:08}-{}", "x".repeat(87))),
+ ));
+ let batch = RecordBatch::try_from_iter([("key", values)])?;
+ let ordering: LexOrdering = [PhysicalSortExpr::new_default(Arc::new(Column::new(
+ "key", 0,
+ )))]
+ .into();
+ let input_bytes = get_reserved_bytes_for_record_batch(&batch)?;
+ let sorted_batches = sort_batch_chunked(&batch, &ordering, batch_size)?;
+ // Each StringView output chunk retains the shared string buffers.
+ // Summing the per-batch estimates counts those buffers once per chunk.
+ let sorted_bytes = sorted_batches
+ .iter()
+ .map(get_record_batch_memory_size)
+ .sum::<usize>();
+ let loan_per_run = sorted_bytes - input_bytes;
+ let first_bytes = get_record_batch_memory_size(&sorted_batches[0]);
+ assert!(loan_per_run > first_bytes);
+ // Both full loans fit before merging; only one extra batch fits beside them.
+ let workspace = 2 * loan_per_run + first_bytes;
+ let runtime = RuntimeEnvBuilder::new()
+ .with_memory_pool(Arc::new(GreedyMemoryPool::new(2 * input_bytes + workspace)))
+ .build_arc()?;
+ let mut sorter = ExternalSorter::new(
+ 0,
+ batch.schema(),
+ ordering.clone(),
+ batch_size,
+ workspace,
+ 0,
+ SpillCompression::Uncompressed,
+ &ExecutionPlanMetricsSet::new(),
+ Arc::clone(&runtime),
+ )?;
+ sorter.insert_batch(batch.clone()).await?;
+ sorter.insert_batch(batch.clone()).await?;
+ sorter.merge_reservation.free();
+ let stream = sorter.in_mem_sort_stream(false, false).await?;
+ // Verify every sort task exits before merging. Tokio can update its task
+ // count shortly after the join completes.
+ tokio::time::timeout(Duration::from_secs(5), async {
+ while tokio::runtime::Handle::current()
+ .metrics()
+ .num_alive_tasks()
+ != 0
+ {
+ tokio::task::yield_now().await;
+ }
+ })
+ .await
+ .expect("sort jobs must finish before merging");
+ // Temporarily borrow the unused workspace: both runs hold their full loans,
+ // leaving only the extra batch. Dropping the temporary returns it immediately.
+ assert_eq!(sorter.merge_pool.borrow(usize::MAX).size(), first_bytes);
+ if consume_output {
+ let output: Vec<RecordBatch> = stream.try_collect().await?;
+ let expected = sort_batch(
+ &concat_batches(&batch.schema(), &[batch.clone(), batch.clone()])?,
+ &ordering,
+ None,
+ )?;
+ assert_eq!(concat_batches(&batch.schema(), &output)?, expected);
+ } else {
+ drop(stream);
+ }
+ drop(sorter);
+ assert_eq!(runtime.memory_pool.reserved(), 0);
+ Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_sort_preparation_retains_workspace_until_drop() -> Result<()> {
+ check_sort_preparation_workspace(false).await
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_sort_merge_lookahead_returns_cursor_workspace() -> Result<()> {
+ check_sort_preparation_workspace(true).await
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_sort_preparation_keeps_output_reserved() -> Result<()> {
+ let parent = RecordBatch::try_from_iter([(
+ "key",
+ Arc::new(Int64Array::from_iter_values((0..8 * 4096).rev())) as ArrayRef,
+ )])?;
+ // All runs share one input buffer, which must be reserved only once.
+ let batches = (0..8)
+ .map(|run| parent.slice(run * 4096, 4096))
+ .collect::<Vec<_>>();
+ let ordering: LexOrdering = [PhysicalSortExpr::new_default(Arc::new(Column::new(
+ "key", 0,
+ )))]
+ .into();
+ let input_bytes = get_reserved_bytes_for_record_batch(&parent)?;
+ let sorted_bytes =
+ get_record_batch_memory_size(&sort_batch(&batches[0], &ordering, None)?)
+ * batches.len();
+ // Reserve the merge's cursor memory alongside the sorted batches.
+ let merge_workspace = sorted_bytes;
+ let capacity = input_bytes + merge_workspace;
+ let runtime = RuntimeEnvBuilder::new()
+ .with_memory_pool(Arc::new(GreedyMemoryPool::new(capacity)))
+ .build_arc()?;
+ let mut sorter = ExternalSorter::new(
+ 0,
+ batches[0].schema(),
+ ordering.clone(),
+ 8192,
+ merge_workspace,
+ 0,
+ SpillCompression::Uncompressed,
+ &ExecutionPlanMetricsSet::new(),
+ Arc::clone(&runtime),
+ )?;
+ for batch in &batches {
+ sorter.insert_batch(batch.clone()).await?;
+ }
+ sorter.merge_reservation.free();
+ let stream = sorter.in_mem_sort_stream(false, false).await?;
+ assert_eq!(
+ runtime.memory_pool.reserved(),
+ sorted_bytes + merge_workspace
+ );
+
+ // Another consumer can take the released sort scratch space, while prepared
+ // batches and merge workspace remain reserved for this sort.
+ let other = MemoryConsumer::new("other").register(&runtime.memory_pool);
+ other.try_grow(capacity - runtime.memory_pool.reserved())?;
+ let output: Vec<RecordBatch> = stream.try_collect().await?;
+ let expected = sort_batch(
+ &concat_batches(&batches[0].schema(), &batches)?,
+ &ordering,
+ None,
+ )?;
+ assert_eq!(concat_batches(&batches[0].schema(), &output)?, expected);
+ drop(sorter);
+ drop(other);
+ assert_eq!(runtime.memory_pool.reserved(), 0);
+ Ok(())
+}
+
#[tokio::test]
async fn test_single_batch_spill_preserves_workspace_after_limit_decreases() -> Result<()>
{
@@ -1014,7 +1173,7 @@ async fn test_single_batch_spill_returns_live_workspace_loan_on_drop() -> Result
sorter.merge_reservation.free();
let merge_pool = Arc::clone(&sorter.merge_pool);
assert_eq!(sorter.in_mem_batches.len(), 1);
- let mut stream = sorter.in_mem_sort_stream(false, false)?;
+ let mut stream = sorter.in_mem_sort_stream(false, false).await?;
assert!(sorter.in_mem_batches.is_empty());
drop(sorter);