Apache DataFusion SQL Query Engine
Add spilling to RepartitionExec (#18014)
Adrian Garcia Badaracco1e30aed6f2c5
diff --git a/datafusion/core/tests/memory_limit/mod.rs b/datafusion/core/tests/memory_limit/mod.rsindex 89bc48b1e6..5d8a1d2418 100644--- a/datafusion/core/tests/memory_limit/mod.rs+++ b/datafusion/core/tests/memory_limit/mod.rs@@ -23,6 +23,7 @@ use std::sync::{Arc, LazyLock};#[cfg(feature = "extended_tests")]mod memory_limit_validation;+mod repartition_mem_limit;use arrow::array::{ArrayRef, DictionaryArray, Int32Array, RecordBatch, StringViewArray};use arrow::compute::SortOptions;use arrow::datatypes::{Int32Type, SchemaRef};diff --git a/datafusion/core/tests/memory_limit/repartition_mem_limit.rs b/datafusion/core/tests/memory_limit/repartition_mem_limit.rsnew file mode 100644index 0000000000..a7af2f01d1--- /dev/null+++ b/datafusion/core/tests/memory_limit/repartition_mem_limit.rs@@ -0,0 +1,116 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use std::sync::Arc;++use arrow::array::{ArrayRef, Int32Array, RecordBatch};+use datafusion::{+ assert_batches_sorted_eq,+ prelude::{SessionConfig, SessionContext},+};+use datafusion_catalog::MemTable;+use datafusion_common::tree_node::{Transformed, TreeNode};+use datafusion_execution::runtime_env::RuntimeEnvBuilder;+use datafusion_physical_plan::{repartition::RepartitionExec, ExecutionPlanProperties};+use futures::TryStreamExt;+use itertools::Itertools;++/// End to end test for spilling in RepartitionExec.+/// The idea is to make a real world query with a relatively low memory limit and+/// then drive one partition at a time, simulating dissimilar execution speed in partitions.+/// Just as some examples of real world scenarios where this can happen consider+/// lopsided groups in a group by especially if one partitions spills and others don't,+/// or in distributed systems if one upstream node is slower than others.+#[tokio::test]+async fn test_repartition_memory_limit() {+ let runtime = RuntimeEnvBuilder::new()+ .with_memory_limit(1024 * 1024, 1.0)+ .build()+ .unwrap();+ let config = SessionConfig::new()+ .with_batch_size(32)+ .with_target_partitions(2);+ let ctx = SessionContext::new_with_config_rt(config, Arc::new(runtime));+ let batches = vec![RecordBatch::try_from_iter(vec![(+ "c1",+ Arc::new(Int32Array::from_iter_values((0..10).cycle().take(100_000))) as ArrayRef,+ )])+ .unwrap()];+ let table = Arc::new(MemTable::try_new(batches[0].schema(), vec![batches]).unwrap());+ ctx.register_table("t", table).unwrap();+ let plan = ctx+ .state()+ .create_logical_plan("SELECT c1, count(*) as c FROM t GROUP BY c1;")+ .await+ .unwrap();+ let plan = ctx.state().create_physical_plan(&plan).await.unwrap();+ assert_eq!(plan.output_partitioning().partition_count(), 2);+ // Execute partition 0, this should cause items going into the rest of the partitions to queue up and because+ // of the low memory limit should spill to disk.+ let batches0 = Arc::clone(&plan)+ .execute(0, ctx.task_ctx())+ .unwrap()+ .try_collect::<Vec<_>>()+ .await+ .unwrap();++ let mut metrics = None;+ Arc::clone(&plan)+ .transform_down(|node| {+ if node.as_any().is::<RepartitionExec>() {+ metrics = node.metrics();+ }+ Ok(Transformed::no(node))+ })+ .unwrap();++ let metrics = metrics.unwrap();+ assert!(metrics.spilled_bytes().unwrap() > 0);+ assert!(metrics.spilled_rows().unwrap() > 0);+ assert!(metrics.spill_count().unwrap() > 0);++ // Execute the other partition+ let batches1 = Arc::clone(&plan)+ .execute(1, ctx.task_ctx())+ .unwrap()+ .try_collect::<Vec<_>>()+ .await+ .unwrap();++ let all_batches = batches0+ .into_iter()+ .chain(batches1.into_iter())+ .collect_vec();+ #[rustfmt::skip]+ let expected = &[+ "+----+-------+",+ "| c1 | c |",+ "+----+-------+",+ "| 0 | 10000 |",+ "| 1 | 10000 |",+ "| 2 | 10000 |",+ "| 3 | 10000 |",+ "| 4 | 10000 |",+ "| 5 | 10000 |",+ "| 6 | 10000 |",+ "| 7 | 10000 |",+ "| 8 | 10000 |",+ "| 9 | 10000 |",+ "+----+-------+",+ ];+ assert_batches_sorted_eq!(expected, &all_batches);+}diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rsindex dafde268ba..8174f71c31 100644--- a/datafusion/physical-plan/src/repartition/mod.rs+++ b/datafusion/physical-plan/src/repartition/mod.rs@@ -32,12 +32,13 @@ use super::{};use crate::execution_plan::{CardinalityEffect, EvaluationType, SchedulingType};use crate::hash_utils::create_hashes;-use crate::metrics::BaselineMetrics;+use crate::metrics::{BaselineMetrics, SpillMetrics};use crate::projection::{all_columns, make_with_child, update_expr, ProjectionExec};use crate::repartition::distributor_channels::{channels, partition_aware_channels, DistributionReceiver, DistributionSender,};use crate::sorts::streaming_merge::StreamingMergeBuilder;+use crate::spill::spill_manager::SpillManager;use crate::stream::RecordBatchStreamAdapter;use crate::{DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, Statistics};@@ -50,6 +51,7 @@ use datafusion_common::utils::transpose;use datafusion_common::{internal_err, ColumnStatistics, HashMap};use datafusion_common::{not_impl_err, DataFusionError, Result};use datafusion_common_runtime::SpawnedTask;+use datafusion_execution::disk_manager::RefCountedTempFile;use datafusion_execution::memory_pool::MemoryConsumer;use datafusion_execution::TaskContext;use datafusion_physical_expr::{EquivalenceProperties, PhysicalExpr};@@ -66,22 +68,42 @@ use parking_lot::Mutex;mod distributor_channels;-type MaybeBatch = Option<Result<RecordBatch>>;+/// A batch in the repartition queue - either in memory or spilled to disk+#[derive(Debug)]+enum RepartitionBatch {+ /// Batch held in memory (counts against memory reservation)+ Memory(RecordBatch),+ /// Batch spilled to disk (one file per batch for queue semantics)+ /// File automatically deleted when dropped via reference counting+ /// The size field stores the original batch size for validation when reading back+ Spilled {+ spill_file: RefCountedTempFile,+ size: usize,+ },+}++type MaybeBatch = Option<Result<RepartitionBatch>>;type InputPartitionsToCurrentPartitionSender = Vec<DistributionSender<MaybeBatch>>;type InputPartitionsToCurrentPartitionReceiver = Vec<DistributionReceiver<MaybeBatch>>;+/// Channels and resources for a single output partition+#[derive(Debug)]+struct PartitionChannels {+ /// Senders for each input partition to send data to this output partition+ tx: InputPartitionsToCurrentPartitionSender,+ /// Receivers for each input partition sending data to this output partition+ rx: InputPartitionsToCurrentPartitionReceiver,+ /// Memory reservation for this output partition+ reservation: SharedMemoryReservation,+ /// Spill manager for handling disk spills for this output partition+ spill_manager: Arc<SpillManager>,+}+#[derive(Debug)]struct ConsumingInputStreamsState {/// Channels for sending batches from input partitions to output partitions./// Key is the partition number.- channels: HashMap<- usize,- (- InputPartitionsToCurrentPartitionSender,- InputPartitionsToCurrentPartitionReceiver,- SharedMemoryReservation,- ),- >,+ channels: HashMap<usize, PartitionChannels>,/// Helper that ensures that that background job is killed once it is no longer needed.abort_helper: Arc<Vec<SpawnedTask<()>>>,@@ -161,8 +183,8 @@ impl RepartitionExecState {let streams_and_metrics = match self {RepartitionExecState::NotInitialized => {self.ensure_input_streams_initialized(- input,- metrics,+ Arc::clone(&input),+ metrics.clone(),partitioning.partition_count(),Arc::clone(&context),)?;@@ -205,9 +227,24 @@ impl RepartitionExecState {for (partition, (tx, rx)) in txs.into_iter().zip(rxs).enumerate() {let reservation = Arc::new(Mutex::new(MemoryConsumer::new(format!("{name}[{partition}]"))+ .with_can_spill(true).register(context.memory_pool()),));- channels.insert(partition, (tx, rx, reservation));+ let spill_metrics = SpillMetrics::new(&metrics, partition);+ let spill_manager = Arc::new(SpillManager::new(+ Arc::clone(&context.runtime_env()),+ spill_metrics,+ input.schema(),+ ));+ channels.insert(+ partition,+ PartitionChannels {+ tx,+ rx,+ reservation,+ spill_manager,+ },+ );}// launch one async task per *input* partition@@ -217,8 +254,15 @@ impl RepartitionExecState {{let txs: HashMap<_, _> = channels.iter()- .map(|(partition, (tx, _rx, reservation))| {- (*partition, (tx[i].clone(), Arc::clone(reservation)))+ .map(|(partition, channels)| {+ (+ *partition,+ (+ channels.tx[i].clone(),+ Arc::clone(&channels.reservation),+ Arc::clone(&channels.spill_manager),+ ),+ )}).collect();@@ -234,7 +278,9 @@ impl RepartitionExecState {let wait_for_task = SpawnedTask::spawn(RepartitionExec::wait_for_task(input_task,txs.into_iter()- .map(|(partition, (tx, _reservation))| (partition, tx))+ .map(|(partition, (tx, _reservation, _spill_manager))| {+ (partition, tx)+ }).collect(),));spawned_tasks.push(wait_for_task);@@ -675,7 +721,7 @@ impl ExecutionPlan for RepartitionExec {let num_input_partitions = input.output_partitioning().partition_count();// lock scope- let (mut rx, reservation, abort_helper) = {+ let (mut rx, reservation, spill_manager, abort_helper) = {// lock mutexeslet mut state = state.lock();let state = state.consume_input_streams(@@ -689,12 +735,22 @@ impl ExecutionPlan for RepartitionExec {// now return stream for the specified *output* partition which will// read from the channel- let (_tx, rx, reservation) = state+ let PartitionChannels {+ rx,+ reservation,+ spill_manager,+ ..+ } = state.channels.remove(&partition).expect("partition not used yet");- (rx, reservation, Arc::clone(&state.abort_helper))+ (+ rx,+ reservation,+ spill_manager,+ Arc::clone(&state.abort_helper),+ )};trace!(@@ -711,6 +767,8 @@ impl ExecutionPlan for RepartitionExec {receiver,_drop_helper: Arc::clone(&abort_helper),reservation: Arc::clone(&reservation),+ spill_manager: Arc::clone(&spill_manager),+ state: RepartitionStreamState::ReceivingFromChannel,}) as SendableRecordBatchStream}).collect::<Vec<_>>();@@ -739,6 +797,8 @@ impl ExecutionPlan for RepartitionExec {input: rx.swap_remove(0),_drop_helper: abort_helper,reservation,+ spill_manager,+ state: RepartitionStreamState::ReceivingFromChannel,}) as SendableRecordBatchStream)}})@@ -979,7 +1039,11 @@ impl RepartitionExec {mut stream: SendableRecordBatchStream,mut output_channels: HashMap<usize,- (DistributionSender<MaybeBatch>, SharedMemoryReservation),+ (+ DistributionSender<MaybeBatch>,+ SharedMemoryReservation,+ Arc<SpillManager>,+ ),>,partitioning: Partitioning,metrics: RepartitionMetrics,@@ -1001,18 +1065,49 @@ impl RepartitionExec {None => break,};+ // Handle empty batch+ if batch.num_rows() == 0 {+ continue;+ }+for res in partitioner.partition_iter(batch)? {let (partition, batch) = res?;let size = batch.get_array_memory_size();let timer = metrics.send_time[partition].timer();// if there is still a receiver, send to it- if let Some((tx, reservation)) = output_channels.get_mut(&partition) {- reservation.lock().try_grow(size)?;-- if tx.send(Some(Ok(batch))).await.is_err() {+ if let Some((tx, reservation, spill_manager)) =+ output_channels.get_mut(&partition)+ {+ let (batch_to_send, is_memory_batch) =+ match reservation.lock().try_grow(size) {+ Ok(_) => {+ // Memory available - send in-memory batch+ (RepartitionBatch::Memory(batch), true)+ }+ Err(_) => {+ // We're memory limited - spill this single batch to its own file+ let spill_file = spill_manager+ .spill_record_batch_and_finish(+ &[batch],+ &format!(+ "RepartitionExec spill partition {partition}"+ ),+ )?+ // Note that we handled empty batch above, so this is safe+ .expect("non-empty batch should produce spill file");++ // Store size for validation when reading back+ (RepartitionBatch::Spilled { spill_file, size }, false)+ }+ };++ if tx.send(Some(Ok(batch_to_send))).await.is_err() {// If the other end has hung up, it was an early shutdown (e.g. LIMIT)- reservation.lock().shrink(size);+ // Only shrink memory if it was a memory batch+ if is_memory_batch {+ reservation.lock().shrink(size);+ }output_channels.remove(&partition);}}@@ -1093,6 +1188,13 @@ impl RepartitionExec {}}+enum RepartitionStreamState {+ /// Waiting for next item from channel+ ReceivingFromChannel,+ /// Reading a spilled batch from disk (stream reads via tokio::fs)+ ReadingSpilledBatch(SendableRecordBatchStream),+}+struct RepartitionStream {/// Number of input partitions that will be sending batches to this output channelnum_input_partitions: usize,@@ -1111,6 +1213,12 @@ struct RepartitionStream {/// Memory reservation.reservation: SharedMemoryReservation,++ /// Spill manager for reading spilled batches+ spill_manager: Arc<SpillManager>,++ /// Current state of the stream+ state: RepartitionStreamState,}impl Stream for RepartitionStream {@@ -1121,33 +1229,67 @@ impl Stream for RepartitionStream {cx: &mut Context<'_>,) -> Poll<Option<Self::Item>> {loop {- match self.input.recv().poll_unpin(cx) {- Poll::Ready(Some(Some(v))) => {- if let Ok(batch) = &v {- self.reservation- .lock()- .shrink(batch.get_array_memory_size());+ match &mut self.state {+ RepartitionStreamState::ReceivingFromChannel => {+ let value = futures::ready!(self.input.recv().poll_unpin(cx));+ match value {+ Some(Some(v)) => match v {+ Ok(RepartitionBatch::Memory(batch)) => {+ // Release memory and return+ self.reservation+ .lock()+ .shrink(batch.get_array_memory_size());+ return Poll::Ready(Some(Ok(batch)));+ }+ Ok(RepartitionBatch::Spilled { spill_file, size }) => {+ // Read from disk - SpillReaderStream uses tokio::fs internally+ // Pass the original size for validation+ let stream = self+ .spill_manager+ .read_spill_as_stream(spill_file, Some(size))?;+ self.state =+ RepartitionStreamState::ReadingSpilledBatch(stream);+ // Continue loop to poll the stream immediately+ }+ Err(e) => {+ return Poll::Ready(Some(Err(e)));+ }+ },+ Some(None) => {+ self.num_input_partitions_processed += 1;++ if self.num_input_partitions+ == self.num_input_partitions_processed+ {+ // all input partitions have finished sending batches+ return Poll::Ready(None);+ } else {+ // other partitions still have data to send+ continue;+ }+ }+ None => {+ return Poll::Ready(None);+ }}-- return Poll::Ready(Some(v));}- Poll::Ready(Some(None)) => {- self.num_input_partitions_processed += 1;-- if self.num_input_partitions == self.num_input_partitions_processed {- // all input partitions have finished sending batches- return Poll::Ready(None);- } else {- // other partitions still have data to send- continue;+ RepartitionStreamState::ReadingSpilledBatch(stream) => {+ match futures::ready!(stream.poll_next_unpin(cx)) {+ Some(Ok(batch)) => {+ // Return batch and stay in ReadingSpilledBatch state to read more batches+ return Poll::Ready(Some(Ok(batch)));+ }+ Some(Err(e)) => {+ self.state = RepartitionStreamState::ReceivingFromChannel;+ return Poll::Ready(Some(Err(e)));+ }+ None => {+ // Spill stream ended - go back to receiving from channel+ self.state = RepartitionStreamState::ReceivingFromChannel;+ continue;+ }}}- Poll::Ready(None) => {- return Poll::Ready(None);- }- Poll::Pending => {- return Poll::Pending;- }}}}@@ -1174,6 +1316,12 @@ struct PerPartitionStream {/// Memory reservation.reservation: SharedMemoryReservation,++ /// Spill manager for reading spilled batches+ spill_manager: Arc<SpillManager>,++ /// Current state of the stream+ state: RepartitionStreamState,}impl Stream for PerPartitionStream {@@ -1183,21 +1331,59 @@ impl Stream for PerPartitionStream {mut self: Pin<&mut Self>,cx: &mut Context<'_>,) -> Poll<Option<Self::Item>> {- match self.receiver.recv().poll_unpin(cx) {- Poll::Ready(Some(Some(v))) => {- if let Ok(batch) = &v {- self.reservation- .lock()- .shrink(batch.get_array_memory_size());+ loop {+ match &mut self.state {+ RepartitionStreamState::ReceivingFromChannel => {+ let value = futures::ready!(self.receiver.recv().poll_unpin(cx));+ match value {+ Some(Some(v)) => match v {+ Ok(RepartitionBatch::Memory(batch)) => {+ // Release memory and return+ self.reservation+ .lock()+ .shrink(batch.get_array_memory_size());+ return Poll::Ready(Some(Ok(batch)));+ }+ Ok(RepartitionBatch::Spilled { spill_file, size }) => {+ // Read from disk - SpillReaderStream uses tokio::fs internally+ // Pass the original size for validation+ let stream = self+ .spill_manager+ .read_spill_as_stream(spill_file, Some(size))?;+ self.state =+ RepartitionStreamState::ReadingSpilledBatch(stream);+ // Continue loop to poll the stream immediately+ }+ Err(e) => {+ return Poll::Ready(Some(Err(e)));+ }+ },+ Some(None) => {+ // Input partition has finished sending batches+ return Poll::Ready(None);+ }+ None => return Poll::Ready(None),+ }+ }++ RepartitionStreamState::ReadingSpilledBatch(stream) => {+ match futures::ready!(stream.poll_next_unpin(cx)) {+ Some(Ok(batch)) => {+ // Return batch and stay in ReadingSpilledBatch state to read more batches+ return Poll::Ready(Some(Ok(batch)));+ }+ Some(Err(e)) => {+ self.state = RepartitionStreamState::ReceivingFromChannel;+ return Poll::Ready(Some(Err(e)));+ }+ None => {+ // Spill stream ended - go back to receiving from channel+ self.state = RepartitionStreamState::ReceivingFromChannel;+ continue;+ }+ }}- Poll::Ready(Some(v))- }- Poll::Ready(Some(None)) => {- // Input partition has finished sending batches- Poll::Ready(None)}- Poll::Ready(None) => Poll::Ready(None),- Poll::Pending => Poll::Pending,}}}@@ -1229,8 +1415,8 @@ mod tests {use arrow::array::{ArrayRef, StringArray, UInt32Array};use arrow::datatypes::{DataType, Field, Schema};use datafusion_common::cast::as_string_array;+ use datafusion_common::exec_err;use datafusion_common::test_util::batches_to_sort_string;- use datafusion_common::{arrow_datafusion_err, exec_err};use datafusion_common_runtime::JoinSet;use datafusion_execution::runtime_env::RuntimeEnvBuilder;use insta::assert_snapshot;@@ -1711,17 +1897,210 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn repartition_with_spilling() -> Result<()> {+ // Test that repartition successfully spills to disk when memory is constrained+ let schema = test_schema();+ let partition = create_vec_batches(50);+ let input_partitions = vec![partition];+ let partitioning = Partitioning::RoundRobinBatch(4);++ // Set up context with very tight memory limit to force spilling+ let runtime = RuntimeEnvBuilder::default()+ .with_memory_limit(1, 1.0)+ .build_arc()?;++ let task_ctx = TaskContext::default().with_runtime(runtime);+ let task_ctx = Arc::new(task_ctx);++ // create physical plan+ let exec =+ TestMemoryExec::try_new_exec(&input_partitions, Arc::clone(&schema), None)?;+ let exec = RepartitionExec::try_new(exec, partitioning)?;++ // Collect all partitions - should succeed by spilling to disk+ let mut total_rows = 0;+ for i in 0..exec.partitioning().partition_count() {+ let mut stream = exec.execute(i, Arc::clone(&task_ctx))?;+ while let Some(result) = stream.next().await {+ let batch = result?;+ total_rows += batch.num_rows();+ }+ }++ // Verify we got all the data (50 batches * 8 rows each)+ assert_eq!(total_rows, 50 * 8);++ // Verify spilling metrics to confirm spilling actually happened+ let metrics = exec.metrics().unwrap();+ assert!(+ metrics.spill_count().unwrap() > 0,+ "Expected spill_count > 0, but got {:?}",+ metrics.spill_count()+ );+ println!("Spilled {} times", metrics.spill_count().unwrap());+ assert!(+ metrics.spilled_bytes().unwrap() > 0,+ "Expected spilled_bytes > 0, but got {:?}",+ metrics.spilled_bytes()+ );+ println!(+ "Spilled {} bytes in {} spills",+ metrics.spilled_bytes().unwrap(),+ metrics.spill_count().unwrap()+ );+ assert!(+ metrics.spilled_rows().unwrap() > 0,+ "Expected spilled_rows > 0, but got {:?}",+ metrics.spilled_rows()+ );+ println!("Spilled {} rows", metrics.spilled_rows().unwrap());++ Ok(())+ }++ #[tokio::test]+ async fn repartition_with_partial_spilling() -> Result<()> {+ // Test that repartition can handle partial spilling (some batches in memory, some spilled)+ let schema = test_schema();+ let partition = create_vec_batches(50);+ let input_partitions = vec![partition];+ let partitioning = Partitioning::RoundRobinBatch(4);++ // Set up context with moderate memory limit to force partial spilling+ // 2KB should allow some batches in memory but force others to spill+ let runtime = RuntimeEnvBuilder::default()+ .with_memory_limit(2 * 1024, 1.0)+ .build_arc()?;++ let task_ctx = TaskContext::default().with_runtime(runtime);+ let task_ctx = Arc::new(task_ctx);++ // create physical plan+ let exec =+ TestMemoryExec::try_new_exec(&input_partitions, Arc::clone(&schema), None)?;+ let exec = RepartitionExec::try_new(exec, partitioning)?;++ // Collect all partitions - should succeed with partial spilling+ let mut total_rows = 0;+ for i in 0..exec.partitioning().partition_count() {+ let mut stream = exec.execute(i, Arc::clone(&task_ctx))?;+ while let Some(result) = stream.next().await {+ let batch = result?;+ total_rows += batch.num_rows();+ }+ }++ // Verify we got all the data (50 batches * 8 rows each)+ assert_eq!(total_rows, 50 * 8);++ // Verify partial spilling metrics+ let metrics = exec.metrics().unwrap();+ let spill_count = metrics.spill_count().unwrap();+ let spilled_rows = metrics.spilled_rows().unwrap();+ let spilled_bytes = metrics.spilled_bytes().unwrap();++ assert!(+ spill_count > 0,+ "Expected some spilling to occur, but got spill_count={spill_count}"+ );+ assert!(+ spilled_rows > 0 && spilled_rows < total_rows,+ "Expected partial spilling (0 < spilled_rows < {total_rows}), but got spilled_rows={spilled_rows}"+ );+ assert!(+ spilled_bytes > 0,+ "Expected some bytes to be spilled, but got spilled_bytes={spilled_bytes}"+ );++ println!(+ "Partial spilling: spilled {} out of {} rows ({:.1}%) in {} spills, {} bytes",+ spilled_rows,+ total_rows,+ (spilled_rows as f64 / total_rows as f64) * 100.0,+ spill_count,+ spilled_bytes+ );++ Ok(())+ }++ #[tokio::test]+ async fn repartition_without_spilling() -> Result<()> {+ // Test that repartition does not spill when there's ample memory+ let schema = test_schema();+ let partition = create_vec_batches(50);+ let input_partitions = vec![partition];+ let partitioning = Partitioning::RoundRobinBatch(4);++ // Set up context with generous memory limit - no spilling should occur+ let runtime = RuntimeEnvBuilder::default()+ .with_memory_limit(10 * 1024 * 1024, 1.0) // 10MB+ .build_arc()?;++ let task_ctx = TaskContext::default().with_runtime(runtime);+ let task_ctx = Arc::new(task_ctx);++ // create physical plan+ let exec =+ TestMemoryExec::try_new_exec(&input_partitions, Arc::clone(&schema), None)?;+ let exec = RepartitionExec::try_new(exec, partitioning)?;++ // Collect all partitions - should succeed without spilling+ let mut total_rows = 0;+ for i in 0..exec.partitioning().partition_count() {+ let mut stream = exec.execute(i, Arc::clone(&task_ctx))?;+ while let Some(result) = stream.next().await {+ let batch = result?;+ total_rows += batch.num_rows();+ }+ }++ // Verify we got all the data (50 batches * 8 rows each)+ assert_eq!(total_rows, 50 * 8);++ // Verify no spilling occurred+ let metrics = exec.metrics().unwrap();+ assert_eq!(+ metrics.spill_count(),+ Some(0),+ "Expected no spilling, but got spill_count={:?}",+ metrics.spill_count()+ );+ assert_eq!(+ metrics.spilled_bytes(),+ Some(0),+ "Expected no bytes spilled, but got spilled_bytes={:?}",+ metrics.spilled_bytes()+ );+ assert_eq!(+ metrics.spilled_rows(),+ Some(0),+ "Expected no rows spilled, but got spilled_rows={:?}",+ metrics.spilled_rows()+ );++ println!("No spilling occurred - all data processed in memory");++ Ok(())+ }+#[tokio::test]async fn oom() -> Result<()> {- // define input partitions+ use datafusion_execution::disk_manager::{DiskManagerBuilder, DiskManagerMode};++ // Test that repartition fails with OOM when disk manager is disabledlet schema = test_schema();let partition = create_vec_batches(50);let input_partitions = vec![partition];let partitioning = Partitioning::RoundRobinBatch(4);- // setup up context+ // Setup context with memory limit but NO disk manager (explicitly disabled)let runtime = RuntimeEnvBuilder::default().with_memory_limit(1, 1.0)+ .with_disk_manager_builder(+ DiskManagerBuilder::default().with_mode(DiskManagerMode::Disabled),+ ).build_arc()?;let task_ctx = TaskContext::default().with_runtime(runtime);@@ -1732,11 +2111,10 @@ mod tests {TestMemoryExec::try_new_exec(&input_partitions, Arc::clone(&schema), None)?;let exec = RepartitionExec::try_new(exec, partitioning)?;- // pull partitions+ // Attempt to execute - should fail with ResourcesExhausted errorfor i in 0..exec.partitioning().partition_count() {let mut stream = exec.execute(i, Arc::clone(&task_ctx))?;- let err =- arrow_datafusion_err!(stream.next().await.unwrap().unwrap_err().into());+ let err = stream.next().await.unwrap().unwrap_err();let err = err.find_root();assert!(matches!(err, DataFusionError::ResourcesExhausted(_)),