Apache DataFusion SQL Query Engine
fix: apply `spill_compression` to `RepartitionExec` spill files (#26142)
Adrian Garcia Badaracco048fc192a9b0
diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rsindex 51ce1345fc..629846b3de 100644--- a/datafusion/physical-plan/src/repartition/mod.rs+++ b/datafusion/physical-plan/src/repartition/mod.rs@@ -1758,7 +1758,8 @@ impl ExecutionPlan for RepartitionExec {Arc::clone(&context.runtime_env()),spill_metrics,input.schema(),- );+ )+ .with_compression_type(context.session_config().spill_compression());// Get existing ordering to use for merginglet sort_exprs = self.sort_exprs().cloned();@@ -2656,6 +2657,7 @@ mod tests {use arrow::datatypes::{DataType, Field, Schema, TimeUnit};use datafusion_common::ScalarValue;use datafusion_common::cast::{as_string_array, as_uint32_array};+ use datafusion_common::config::SpillCompression;use datafusion_common::exec_err;use datafusion_common::test_util::batches_to_sort_string;use datafusion_common_runtime::JoinSet;@@ -4589,6 +4591,80 @@ mod tests {Ok(())}+ /// Runs a spilling `RepartitionExec` with the given spill compression and+ /// returns `(spilled_rows, spilled_bytes)`.+ async fn repartition_spill_with_compression(+ spill_compression: SpillCompression,+ ) -> Result<(usize, usize)> {+ // Highly compressible input: 20 batches of 8192 identical values+ let schema = test_schema(false);+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(UInt32Array::from(vec![42; 8192]))],+ )?;+ let num_batches = 20;+ let input_partitions = vec![vec![batch; num_batches]];++ // Tight memory limit to force every batch to spill+ let runtime = RuntimeEnvBuilder::default()+ .with_memory_limit(1, 1.0)+ .build_arc()?;+ let session_config =+ SessionConfig::new().with_spill_compression(spill_compression);+ let task_ctx = Arc::new(+ TaskContext::default()+ .with_runtime(runtime)+ .with_session_config(session_config),+ );++ let exec =+ TestMemoryExec::try_new_exec(&input_partitions, Arc::clone(&schema), None)?;+ let exec = RepartitionExec::try_new(exec, Partitioning::RoundRobinBatch(4))?;++ let mut total_rows = 0;+ for i in 0..exec.partitioning().partition_count() {+ let mut stream = exec.execute(i, Arc::clone(&task_ctx))?;+ while let Some(result) = stream.next().await {+ let batch = result?;+ // Spilled data must read back intact regardless of the codec+ let values = as_uint32_array(batch.column(0))?;+ assert!(values.iter().all(|v| v == Some(42)));+ total_rows += batch.num_rows();+ }+ }+ assert_eq!(total_rows, num_batches * 8192);++ let metrics = exec.metrics().unwrap();+ Ok((+ metrics.spilled_rows().unwrap(),+ metrics.spilled_bytes().unwrap(),+ ))+ }++ #[tokio::test]+ async fn repartition_spill_honors_spill_compression() -> Result<()> {+ let (uncompressed_rows, uncompressed_bytes) =+ repartition_spill_with_compression(SpillCompression::Uncompressed).await?;+ assert!(uncompressed_rows > 0, "Expected spilling to occur");++ for spill_compression in [SpillCompression::Lz4Frame, SpillCompression::Zstd] {+ let (rows, bytes) =+ repartition_spill_with_compression(spill_compression).await?;+ assert_eq!(+ rows, uncompressed_rows,+ "Expected the same rows to spill with {spill_compression}"+ );+ // The input is constant, so any codec shrinks it by far more than 2x+ assert!(+ bytes * 2 < uncompressed_bytes,+ "Expected {spill_compression} spill files to be much smaller than \+ uncompressed ones: {bytes} vs {uncompressed_bytes} bytes"+ );+ }++ Ok(())+ }+#[tokio::test]async fn oom() -> Result<()> {use datafusion_execution::disk_manager::{DiskManagerBuilder, DiskManagerMode};