Skip to content

Apache DataFusion SQL Query Engine

feat: cap spill merge fan-in (#23066)

Yin Lib7e7b514e9f9
diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rs
index 73847b67ed..22a61cf919 100644
--- a/datafusion/core/src/execution/context/mod.rs
+++ b/datafusion/core/src/execution/context/mod.rs
@@ -82,7 +82,7 @@ use datafusion_execution::cache::cache_manager::{
};
pub use datafusion_execution::config::SessionConfig;
use datafusion_execution::disk_manager::{
- DEFAULT_MAX_TEMP_DIRECTORY_SIZE, DiskManagerBuilder,
+ DEFAULT_MAX_SPILL_MERGE_FAN_IN, DEFAULT_MAX_TEMP_DIRECTORY_SIZE, DiskManagerBuilder,
};
use datafusion_execution::registry::SerializerRegistry;
use datafusion_expr::HigherOrderUDF;
@@ -1208,6 +1208,14 @@ impl SessionContext {
let limit = Self::parse_capacity_limit(variable, value)?;
builder.with_file_statistics_cache_limit(limit)
}
+ "max_spill_merge_fan_in" => {
+ let fan_in = value.parse::<usize>().map_err(|e| {
+ DataFusionError::Plan(format!(
+ "Failed to parse non-negative integer from '{variable}', value '{value}': {e}"
+ ))
+ })?;
+ builder.with_max_spill_merge_fan_in(fan_in)
+ }
_ => return plan_err!("Unknown runtime configuration: {variable}"),
// Remember to update `reset_runtime_variable()` when adding new options
};
@@ -1252,6 +1260,10 @@ impl SessionContext {
DEFAULT_FILE_STATISTICS_MEMORY_LIMIT,
);
}
+ "max_spill_merge_fan_in" => {
+ builder =
+ builder.with_max_spill_merge_fan_in(DEFAULT_MAX_SPILL_MERGE_FAN_IN);
+ }
_ => return plan_err!("Unknown runtime configuration: {variable}"),
};
*state = SessionStateBuilder::from(state.clone())
diff --git a/datafusion/core/tests/sql/runtime_config.rs b/datafusion/core/tests/sql/runtime_config.rs
index 604d137540..5f1e0629ec 100644
--- a/datafusion/core/tests/sql/runtime_config.rs
+++ b/datafusion/core/tests/sql/runtime_config.rs
@@ -227,6 +227,34 @@ async fn test_max_temp_directory_size_enforcement() {
);
}
+#[tokio::test]
+async fn test_max_spill_merge_fan_in_runtime_config() {
+ let ctx = SessionContext::new();
+
+ ctx.sql("SET datafusion.runtime.max_spill_merge_fan_in = '8'")
+ .await
+ .unwrap()
+ .collect()
+ .await
+ .unwrap();
+ assert_eq!(ctx.runtime_env().disk_manager.max_spill_merge_fan_in(), 8);
+
+ ctx.sql("RESET datafusion.runtime.max_spill_merge_fan_in")
+ .await
+ .unwrap()
+ .collect()
+ .await
+ .unwrap();
+ assert_eq!(ctx.runtime_env().disk_manager.max_spill_merge_fan_in(), 0);
+
+ let error = ctx
+ .sql("SET datafusion.runtime.max_spill_merge_fan_in = '-1'")
+ .await
+ .unwrap_err()
+ .to_string();
+ assert!(error.contains("Failed to parse non-negative integer"));
+}
+
#[tokio::test]
async fn test_test_metadata_cache_limit() {
let ctx = SessionContext::new();
diff --git a/datafusion/execution/src/disk_manager.rs b/datafusion/execution/src/disk_manager.rs
index ff8403d916..8534c4f4ab 100644
--- a/datafusion/execution/src/disk_manager.rs
+++ b/datafusion/execution/src/disk_manager.rs
@@ -32,6 +32,7 @@ use std::sync::Arc;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use tempfile::{Builder, NamedTempFile, TempDir};
pub const DEFAULT_MAX_TEMP_DIRECTORY_SIZE: u64 = 100 * 1024 * 1024 * 1024; // 100GB
+pub const DEFAULT_MAX_SPILL_MERGE_FAN_IN: usize = 0;
/// Builder pattern for the [DiskManager] structure
#[derive(Clone)]
@@ -41,6 +42,9 @@ pub struct DiskManagerBuilder {
/// The maximum amount of data (in bytes) stored inside the temporary directories.
/// Default to 100GB
max_temp_directory_size: u64,
+ /// Maximum number of spill files opened by one external merge pass.
+ /// A value of 0 means unlimited.
+ max_spill_merge_fan_in: usize,
}
impl Debug for DiskManagerBuilder {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -55,6 +59,7 @@ impl Default for DiskManagerBuilder {
Self {
mode: DiskManagerMode::OsTmpDirectory,
max_temp_directory_size: DEFAULT_MAX_TEMP_DIRECTORY_SIZE,
+ max_spill_merge_fan_in: DEFAULT_MAX_SPILL_MERGE_FAN_IN,
}
}
}
@@ -78,12 +83,22 @@ impl DiskManagerBuilder {
self
}
+ pub fn set_max_spill_merge_fan_in(&mut self, value: usize) {
+ self.max_spill_merge_fan_in = value;
+ }
+
+ pub fn with_max_spill_merge_fan_in(mut self, value: usize) -> Self {
+ self.set_max_spill_merge_fan_in(value);
+ self
+ }
+
/// Create a DiskManager given the builder
pub fn build(self) -> Result<DiskManager> {
match self.mode {
DiskManagerMode::OsTmpDirectory => Ok(DiskManager {
local_dirs: Mutex::new(Some(vec![])),
max_temp_directory_size: AtomicU64::new(self.max_temp_directory_size),
+ max_spill_merge_fan_in: AtomicUsize::new(self.max_spill_merge_fan_in),
used_disk_space: Arc::new(AtomicU64::new(0)),
active_files_count: Arc::new(AtomicUsize::new(0)),
factory: None,
@@ -96,6 +111,7 @@ impl DiskManagerBuilder {
Ok(DiskManager {
local_dirs: Mutex::new(Some(local_dirs)),
max_temp_directory_size: AtomicU64::new(self.max_temp_directory_size),
+ max_spill_merge_fan_in: AtomicUsize::new(self.max_spill_merge_fan_in),
used_disk_space: Arc::new(AtomicU64::new(0)),
active_files_count: Arc::new(AtomicUsize::new(0)),
factory: None,
@@ -104,6 +120,7 @@ impl DiskManagerBuilder {
DiskManagerMode::Disabled => Ok(DiskManager {
local_dirs: Mutex::new(None),
max_temp_directory_size: AtomicU64::new(self.max_temp_directory_size),
+ max_spill_merge_fan_in: AtomicUsize::new(self.max_spill_merge_fan_in),
used_disk_space: Arc::new(AtomicU64::new(0)),
active_files_count: Arc::new(AtomicUsize::new(0)),
factory: None,
@@ -111,6 +128,7 @@ impl DiskManagerBuilder {
DiskManagerMode::Custom(factory) => Ok(DiskManager {
local_dirs: Mutex::new(None),
max_temp_directory_size: AtomicU64::new(self.max_temp_directory_size),
+ max_spill_merge_fan_in: AtomicUsize::new(self.max_spill_merge_fan_in),
used_disk_space: Arc::new(AtomicU64::new(0)),
active_files_count: Arc::new(AtomicUsize::new(0)),
factory: Some(factory),
@@ -161,6 +179,9 @@ pub struct DiskManager {
/// Default to 100GB. Stored as `AtomicU64` so it can be adjusted at runtime
/// without requiring exclusive (`&mut`) access to the `DiskManager`.
max_temp_directory_size: AtomicU64,
+ /// Maximum number of spill files opened by one external merge pass.
+ /// A value of 0 preserves the memory-driven, unbounded behavior.
+ max_spill_merge_fan_in: AtomicUsize,
/// Used disk space in the temporary directories. Now only spilled data for
/// external executors are counted.
used_disk_space: Arc<AtomicU64>,
@@ -250,6 +271,22 @@ impl DiskManager {
self.max_temp_directory_size.load(Ordering::Relaxed)
}
+ /// Atomically set the maximum spill merge fan-in.
+ ///
+ /// A value of 0 disables the cap. Values of 1 are accepted but external
+ /// merge code will still merge at least two spill streams to make progress.
+ pub fn set_max_spill_merge_fan_in(&self, max_spill_merge_fan_in: usize) {
+ self.max_spill_merge_fan_in
+ .store(max_spill_merge_fan_in, Ordering::Relaxed);
+ }
+
+ /// Returns the maximum number of spill files opened by one merge pass.
+ ///
+ /// A value of 0 means unlimited.
+ pub fn max_spill_merge_fan_in(&self) -> usize {
+ self.max_spill_merge_fan_in.load(Ordering::Relaxed)
+ }
+
/// Returns the current spilling progress
pub fn spilling_progress(&self) -> SpillingProgress {
SpillingProgress {
@@ -905,6 +942,25 @@ mod tests {
Ok(())
}
+ #[test]
+ fn test_max_spill_merge_fan_in_builder_and_dynamic_update() -> Result<()> {
+ let dm = Arc::new(
+ DiskManager::builder()
+ .with_max_spill_merge_fan_in(8)
+ .build()?,
+ );
+
+ assert_eq!(dm.max_spill_merge_fan_in(), 8);
+
+ dm.set_max_spill_merge_fan_in(4);
+ assert_eq!(dm.max_spill_merge_fan_in(), 4);
+
+ dm.set_max_spill_merge_fan_in(0);
+ assert_eq!(dm.max_spill_merge_fan_in(), 0);
+
+ Ok(())
+ }
+
#[test]
fn test_disabled_disk_manager_rejects_nonzero_limit() -> Result<()> {
let dm = DiskManager::builder()
diff --git a/datafusion/execution/src/runtime_env.rs b/datafusion/execution/src/runtime_env.rs
index 22b65c4189..fcfe51267e 100644
--- a/datafusion/execution/src/runtime_env.rs
+++ b/datafusion/execution/src/runtime_env.rs
@@ -90,57 +90,77 @@ impl Debug for RuntimeEnv {
}
}
-/// Creates runtime configuration entries with the provided values
-///
-/// This helper function defines the structure and metadata for all runtime configuration
-/// entries to avoid duplication between `RuntimeEnv::config_entries()` and
-/// `RuntimeEnvBuilder::entries()`.
-fn create_runtime_config_entries(
+struct RuntimeConfigValues {
memory_limit: Option<String>,
max_temp_directory_size: Option<String>,
+ max_spill_merge_fan_in: Option<String>,
temp_directory: Option<String>,
metadata_cache_limit: Option<String>,
list_files_cache_limit: Option<String>,
list_files_cache_ttl: Option<String>,
file_statistics_cache_limit: Option<String>,
-) -> Vec<ConfigEntry> {
- vec![
- ConfigEntry {
- key: "datafusion.runtime.memory_limit".to_string(),
- value: memory_limit,
- description: "Maximum memory limit for query execution. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.",
- },
- ConfigEntry {
- key: "datafusion.runtime.max_temp_directory_size".to_string(),
- value: max_temp_directory_size,
- description: "Maximum temporary file directory size. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.",
- },
- ConfigEntry {
- key: "datafusion.runtime.temp_directory".to_string(),
- value: temp_directory,
- description: "The path to the temporary file directory.",
- },
- ConfigEntry {
- key: "datafusion.runtime.metadata_cache_limit".to_string(),
- value: metadata_cache_limit,
- description: "Maximum memory to use for file metadata cache such as Parquet metadata. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.",
- },
- ConfigEntry {
- key: "datafusion.runtime.list_files_cache_limit".to_string(),
- value: list_files_cache_limit,
- description: "Maximum memory to use for list files cache. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.",
- },
- ConfigEntry {
- key: "datafusion.runtime.list_files_cache_ttl".to_string(),
- value: list_files_cache_ttl,
- description: "TTL (time-to-live) of the entries in the list file cache. Supports units m (minutes), and s (seconds). Example: '2m' for 2 minutes.",
- },
- ConfigEntry {
- key: "datafusion.runtime.file_statistics_cache_limit".to_string(),
- value: file_statistics_cache_limit,
- description: "Maximum memory to use for file statistics cache. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.",
- },
- ]
+}
+
+impl RuntimeConfigValues {
+ /// Creates runtime configuration entries with the provided values.
+ ///
+ /// This defines the structure and metadata for all runtime configuration
+ /// entries to avoid duplication between `RuntimeEnv::config_entries()` and
+ /// `RuntimeEnvBuilder::entries()`.
+ fn into_config_entries(self) -> Vec<ConfigEntry> {
+ let Self {
+ memory_limit,
+ max_temp_directory_size,
+ max_spill_merge_fan_in,
+ temp_directory,
+ metadata_cache_limit,
+ list_files_cache_limit,
+ list_files_cache_ttl,
+ file_statistics_cache_limit,
+ } = self;
+ vec![
+ ConfigEntry {
+ key: "datafusion.runtime.memory_limit".to_string(),
+ value: memory_limit,
+ description: "Maximum memory limit for query execution. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.",
+ },
+ ConfigEntry {
+ key: "datafusion.runtime.max_temp_directory_size".to_string(),
+ value: max_temp_directory_size,
+ description: "Maximum temporary file directory size. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.",
+ },
+ ConfigEntry {
+ key: "datafusion.runtime.max_spill_merge_fan_in".to_string(),
+ value: max_spill_merge_fan_in,
+ description: "Maximum number of spill files opened by one external merge pass. Use 0 for unlimited. Values below 2 still use 2 so a merge can make progress.",
+ },
+ ConfigEntry {
+ key: "datafusion.runtime.temp_directory".to_string(),
+ value: temp_directory,
+ description: "The path to the temporary file directory.",
+ },
+ ConfigEntry {
+ key: "datafusion.runtime.metadata_cache_limit".to_string(),
+ value: metadata_cache_limit,
+ description: "Maximum memory to use for file metadata cache such as Parquet metadata. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.",
+ },
+ ConfigEntry {
+ key: "datafusion.runtime.list_files_cache_limit".to_string(),
+ value: list_files_cache_limit,
+ description: "Maximum memory to use for list files cache. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.",
+ },
+ ConfigEntry {
+ key: "datafusion.runtime.list_files_cache_ttl".to_string(),
+ value: list_files_cache_ttl,
+ description: "TTL (time-to-live) of the entries in the list file cache. Supports units m (minutes), and s (seconds). Example: '2m' for 2 minutes.",
+ },
+ ConfigEntry {
+ key: "datafusion.runtime.file_statistics_cache_limit".to_string(),
+ value: file_statistics_cache_limit,
+ description: "Maximum memory to use for file statistics cache. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.",
+ },
+ ]
+ }
}
impl RuntimeEnv {
@@ -268,6 +288,8 @@ impl RuntimeEnv {
let max_temp_dir_size = self.disk_manager.max_temp_directory_size();
let max_temp_dir_value = format_byte_size(max_temp_dir_size);
+ let max_spill_merge_fan_in =
+ self.disk_manager.max_spill_merge_fan_in().to_string();
let temp_paths = self.disk_manager.temp_dir_paths();
let temp_dir_value = if temp_paths.is_empty() {
@@ -309,15 +331,17 @@ impl RuntimeEnv {
.expect("File statistics cache size conversion failed"),
);
- create_runtime_config_entries(
- memory_limit_value,
- Some(max_temp_dir_value),
- temp_dir_value,
- Some(metadata_cache_value),
- Some(list_files_cache_value),
+ RuntimeConfigValues {
+ memory_limit: memory_limit_value,
+ max_temp_directory_size: Some(max_temp_dir_value),
+ max_spill_merge_fan_in: Some(max_spill_merge_fan_in),
+ temp_directory: temp_dir_value,
+ metadata_cache_limit: Some(metadata_cache_value),
+ list_files_cache_limit: Some(list_files_cache_value),
list_files_cache_ttl,
- Some(file_statistics_cache_value),
- )
+ file_statistics_cache_limit: Some(file_statistics_cache_value),
+ }
+ .into_config_entries()
}
}
@@ -425,6 +449,14 @@ impl RuntimeEnvBuilder {
self.with_disk_manager_builder(builder.with_max_temp_directory_size(size))
}
+ /// Limit the number of spill files opened by one external merge pass.
+ ///
+ /// A value of 0 means unlimited.
+ pub fn with_max_spill_merge_fan_in(mut self, fan_in: usize) -> Self {
+ let builder = self.disk_manager_builder.take().unwrap_or_default();
+ self.with_disk_manager_builder(builder.with_max_spill_merge_fan_in(fan_in))
+ }
+
/// Specify the limit of the file-embedded metadata cache, in bytes.
pub fn with_metadata_cache_limit(mut self, limit: usize) -> Self {
self.cache_manager = self.cache_manager.with_metadata_cache_limit(limit);
@@ -516,15 +548,17 @@ impl RuntimeEnvBuilder {
/// Returns a list of all available runtime configurations with their current values and descriptions
pub fn entries(&self) -> Vec<ConfigEntry> {
- create_runtime_config_entries(
- None,
- Some("100G".to_string()),
- None,
- Some("50M".to_owned()),
- Some("1M".to_owned()),
- None,
- Some("20M".to_owned()),
- )
+ RuntimeConfigValues {
+ memory_limit: None,
+ max_temp_directory_size: Some("100G".to_string()),
+ max_spill_merge_fan_in: Some("0".to_string()),
+ temp_directory: None,
+ metadata_cache_limit: Some("50M".to_owned()),
+ list_files_cache_limit: Some("1M".to_owned()),
+ list_files_cache_ttl: None,
+ file_statistics_cache_limit: Some("20M".to_owned()),
+ }
+ .into_config_entries()
}
/// Generate documentation that can be included in the user guide
diff --git a/datafusion/physical-plan/src/sorts/multi_level_merge.rs b/datafusion/physical-plan/src/sorts/multi_level_merge.rs
index 8e292900b1..4d108ac046 100644
--- a/datafusion/physical-plan/src/sorts/multi_level_merge.rs
+++ b/datafusion/physical-plan/src/sorts/multi_level_merge.rs
@@ -415,6 +415,12 @@ impl MultiLevelMergeBuilder {
) -> Result<SpillFilesToMerge> {
assert_ne!(buffer_len, 0, "Buffer length must be greater than 0");
let mut number_of_spills_to_read_for_current_phase = 0;
+ let configured_fan_in = self
+ .spill_manager
+ .env()
+ .disk_manager
+ .max_spill_merge_fan_in();
+ let max_spill_files = effective_spill_merge_fan_in(configured_fan_in);
// Track total memory needed for spill file buffers. When the
// reservation has pre-reserved bytes (from sort_spill_reservation_bytes),
// those bytes cover the first N spill files without additional pool
@@ -422,6 +428,10 @@ impl MultiLevelMergeBuilder {
let mut total_needed: usize = 0;
for spill in &self.sorted_spill_files {
+ if number_of_spills_to_read_for_current_phase >= max_spill_files {
+ break;
+ }
+
let per_spill = get_reserved_bytes_for_record_batch_size(
spill.max_record_batch_memory,
// Size will be the same as the sliced size, bc it is a spilled batch.
@@ -617,6 +627,14 @@ fn split_batch_in_half(batch: RecordBatch) -> Vec<RecordBatch> {
vec![batch.slice(0, mid), batch.slice(mid, num_rows - mid)]
}
+fn effective_spill_merge_fan_in(configured_fan_in: usize) -> usize {
+ if configured_fan_in == 0 {
+ usize::MAX
+ } else {
+ configured_fan_in.max(2)
+ }
+}
+
struct StreamAttachedReservation {
stream: SendableRecordBatchStream,
reservation: MemoryReservation,
@@ -679,8 +697,8 @@ mod tests {
use datafusion_execution::memory_pool::{
GreedyMemoryPool, MemoryConsumer, MemoryPool,
};
- use datafusion_execution::runtime_env::RuntimeEnv;
- use datafusion_physical_expr::expressions::Column;
+ use datafusion_execution::runtime_env::{RuntimeEnv, RuntimeEnvBuilder};
+ use datafusion_physical_expr::expressions::{Column, col};
use datafusion_physical_expr_common::metrics::{
ExecutionPlanMetricsSet, SpillMetrics,
};
@@ -871,6 +889,77 @@ mod tests {
batches, got a largest batch of {max_batch_rows} rows"
);
+ Ok(())
+ }
+ #[test]
+ fn spill_merge_fan_in_is_unlimited_by_default() {
+ assert_eq!(effective_spill_merge_fan_in(0), usize::MAX);
+ }
+
+ #[test]
+ fn spill_merge_fan_in_preserves_merge_progress() {
+ assert_eq!(effective_spill_merge_fan_in(1), 2);
+ assert_eq!(effective_spill_merge_fan_in(2), 2);
+ assert_eq!(effective_spill_merge_fan_in(8), 8);
+ }
+
+ #[test]
+ fn spill_merge_phase_respects_configured_fan_in() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
+ let runtime = RuntimeEnvBuilder::new()
+ .with_max_spill_merge_fan_in(2)
+ .build_arc()?;
+ let spill_manager = SpillManager::new(
+ Arc::clone(&runtime),
+ SpillMetrics::new(&ExecutionPlanMetricsSet::new(), 0),
+ Arc::clone(&schema),
+ );
+ let sorted_spill_files = (0..4)
+ .map(|idx| {
+ Ok(SortedSpillFile {
+ file: runtime
+ .disk_manager
+ .create_tmp_file(&format!("spill fan-in test {idx}"))?,
+ max_record_batch_memory: 1,
+ })
+ })
+ .collect::<Result<Vec<_>>>()?;
+ let expr = LexOrdering::new([PhysicalSortExpr::new_default(col("a", &schema)?)])
+ .unwrap();
+ let reservation =
+ MemoryConsumer::new("spill_merge_phase_respects_configured_fan_in")
+ .register(&runtime.memory_pool);
+ let metrics = BaselineMetrics::new(&ExecutionPlanMetricsSet::new(), 0);
+ let mut builder = MultiLevelMergeBuilder::new(
+ spill_manager,
+ schema,
+ sorted_spill_files,
+ vec![],
+ expr,
+ metrics,
+ 1024,
+ reservation,
+ None,
+ false,
+ );
+ let mut merge_reservation = MemoryConsumer::new("spill_merge_fan_in_phase")
+ .register(&runtime.memory_pool);
+
+ let (spills, buffer_len) = match builder.get_sorted_spill_files_to_merge(
+ 1,
+ 2,
+ &mut merge_reservation,
+ )? {
+ SpillFilesToMerge::Ready(spills, buffer_len) => (spills, buffer_len),
+ SpillFilesToMerge::SplitThenRetry(index) => {
+ panic!("expected ready spill files, got retry for index {index}")
+ }
+ };
+
+ assert_eq!(spills.len(), 2);
+ assert_eq!(buffer_len, 1);
+ assert_eq!(builder.sorted_spill_files.len(), 2);
+
Ok(())
}
}
diff --git a/datafusion/physical-plan/src/spill/spill_manager.rs b/datafusion/physical-plan/src/spill/spill_manager.rs
index 3f305c16a6..aee9e917c7 100644
--- a/datafusion/physical-plan/src/spill/spill_manager.rs
+++ b/datafusion/physical-plan/src/spill/spill_manager.rs
@@ -76,6 +76,10 @@ impl SpillManager {
&self.schema
}
+ pub(crate) fn env(&self) -> &RuntimeEnv {
+ &self.env
+ }
+
/// Creates a temporary file for in-progress operations, returning an error
/// message if file creation fails. The file can be used to append batches
/// incrementally and then finish the file when done.
diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt
index 50f063e8d2..bf45564e26 100644
--- a/datafusion/sqllogictest/test_files/information_schema.slt
+++ b/datafusion/sqllogictest/test_files/information_schema.slt
@@ -344,6 +344,7 @@ datafusion.optimizer.use_statistics_registry false
datafusion.runtime.file_statistics_cache_limit 20M
datafusion.runtime.list_files_cache_limit 1M
datafusion.runtime.list_files_cache_ttl NULL
+datafusion.runtime.max_spill_merge_fan_in 0
datafusion.runtime.max_temp_directory_size 100G
datafusion.runtime.memory_limit unlimited
datafusion.runtime.metadata_cache_limit 50M
@@ -502,6 +503,7 @@ datafusion.optimizer.use_statistics_registry false When set to true, the physica
datafusion.runtime.file_statistics_cache_limit 20M Maximum memory to use for file statistics cache. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.
datafusion.runtime.list_files_cache_limit 1M Maximum memory to use for list files cache. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.
datafusion.runtime.list_files_cache_ttl NULL TTL (time-to-live) of the entries in the list file cache. Supports units m (minutes), and s (seconds). Example: '2m' for 2 minutes.
+datafusion.runtime.max_spill_merge_fan_in 0 Maximum number of spill files opened by one external merge pass. Use 0 for unlimited. Values below 2 still use 2 so a merge can make progress.
datafusion.runtime.max_temp_directory_size 100G Maximum temporary file directory size. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.
datafusion.runtime.memory_limit unlimited Maximum memory limit for query execution. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.
datafusion.runtime.metadata_cache_limit 50M Maximum memory to use for file metadata cache such as Parquet metadata. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.
diff --git a/datafusion/sqllogictest/test_files/push_down_filter_regression.slt b/datafusion/sqllogictest/test_files/push_down_filter_regression.slt
index f28a314764..7ab5e7c79d 100644
--- a/datafusion/sqllogictest/test_files/push_down_filter_regression.slt
+++ b/datafusion/sqllogictest/test_files/push_down_filter_regression.slt
@@ -222,7 +222,7 @@ set datafusion.execution.collect_statistics = true;
# execution (the order in which Partial aggregates publish dynamic filter
# updates races against when the scan reads each partition). The original
# Rust test only asserted matched < 4; the important invariant here is
-# that the DynamicFilter text is correct.
+# that dynamic filtering is applied and metrics are suppressed.
statement ok
set datafusion.explain.analyze_level = summary;
@@ -236,7 +236,7 @@ Plan with Metrics
01)AggregateExec: mode=Final, gby=[], aggr=[max(agg_dyn_e2e.column1)], metrics=[]
02)--CoalescePartitionsExec, metrics=[]
03)----AggregateExec: mode=Partial, gby=[], aggr=[max(agg_dyn_e2e.column1)], metrics=[]
-04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_dyn/file_0.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_dyn/file_1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_dyn/file_2.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_dyn/file_3.parquet]]}, projection=[column1], file_type=parquet, predicate=column1@0 > 1 AND DynamicFilter [ column1@0 > 4 ], dynamic_rg_pruning=eligible, pruning_predicate=column1_null_count@1 != row_count@2 AND column1_max@0 > 1 AND column1_null_count@1 != row_count@2 AND column1_max@0 > 4, required_guarantees=[], metrics=[]
+04)------DataSourceExec: file_groups=<slt:ignore> projection=[column1], file_type=parquet, predicate=column1@0 > 1 AND DynamicFilter [ column1@0 > <slt:ignore> ], dynamic_rg_pruning=eligible, pruning_predicate=column1_null_count@1 != row_count@2 AND column1_max@0 > 1 AND column1_null_count@1 != row_count@2 AND column1_max@0 > <slt:ignore>, required_guarantees=[], metrics=[]
statement ok
reset datafusion.explain.analyze_categories;
diff --git a/datafusion/sqllogictest/test_files/set_variable.slt b/datafusion/sqllogictest/test_files/set_variable.slt
index 0514deba28..c86c0007b6 100644
--- a/datafusion/sqllogictest/test_files/set_variable.slt
+++ b/datafusion/sqllogictest/test_files/set_variable.slt
@@ -611,6 +611,18 @@ SHOW datafusion.runtime.max_temp_directory_size
----
datafusion.runtime.max_temp_directory_size 10G
+# Test SET and SHOW runtime.max_spill_merge_fan_in
+statement ok
+SET datafusion.runtime.max_spill_merge_fan_in = '16'
+
+query TT
+SHOW datafusion.runtime.max_spill_merge_fan_in
+----
+datafusion.runtime.max_spill_merge_fan_in 16
+
+statement ok
+RESET datafusion.runtime.max_spill_merge_fan_in
+
# Test SET and SHOW runtime.file_statistics_cache_limit
statement ok
SET datafusion.runtime.file_statistics_cache_limit = '42M'
@@ -669,6 +681,7 @@ SELECT name FROM information_schema.df_settings WHERE name LIKE 'datafusion.runt
datafusion.runtime.file_statistics_cache_limit
datafusion.runtime.list_files_cache_limit
datafusion.runtime.list_files_cache_ttl
+datafusion.runtime.max_spill_merge_fan_in
datafusion.runtime.max_temp_directory_size
datafusion.runtime.memory_limit
datafusion.runtime.metadata_cache_limit
diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md
index 945f2622c2..03340c366d 100644
--- a/docs/source/user-guide/configs.md
+++ b/docs/source/user-guide/configs.md
@@ -244,6 +244,7 @@ The following runtime configuration settings are available:
| datafusion.runtime.file_statistics_cache_limit | 20M | Maximum memory to use for file statistics cache. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes. |
| datafusion.runtime.list_files_cache_limit | 1M | Maximum memory to use for list files cache. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes. |
| datafusion.runtime.list_files_cache_ttl | NULL | TTL (time-to-live) of the entries in the list file cache. Supports units m (minutes), and s (seconds). Example: '2m' for 2 minutes. |
+| datafusion.runtime.max_spill_merge_fan_in | 0 | Maximum number of spill files opened by one external merge pass. Use 0 for unlimited. Values below 2 still use 2 so a merge can make progress. |
| datafusion.runtime.max_temp_directory_size | 100G | Maximum temporary file directory size. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes. |
| datafusion.runtime.memory_limit | NULL | Maximum memory limit for query execution. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes. |
| datafusion.runtime.metadata_cache_limit | 50M | Maximum memory to use for file metadata cache such as Parquet metadata. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes. |