Apache DataFusion SQL Query Engine
feat: cap spill merge fan-in (#23066)
Yin Lib7e7b514e9f9
diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rsindex 73847b67ed..22a61cf919 100644--- a/datafusion/core/src/execution/context/mod.rs+++ b/datafusion/core/src/execution/context/mod.rs@@ -82,7 +82,7 @@ use datafusion_execution::cache::cache_manager::{};pub use datafusion_execution::config::SessionConfig;use datafusion_execution::disk_manager::{- DEFAULT_MAX_TEMP_DIRECTORY_SIZE, DiskManagerBuilder,+ DEFAULT_MAX_SPILL_MERGE_FAN_IN, DEFAULT_MAX_TEMP_DIRECTORY_SIZE, DiskManagerBuilder,};use datafusion_execution::registry::SerializerRegistry;use datafusion_expr::HigherOrderUDF;@@ -1208,6 +1208,14 @@ impl SessionContext {let limit = Self::parse_capacity_limit(variable, value)?;builder.with_file_statistics_cache_limit(limit)}+ "max_spill_merge_fan_in" => {+ let fan_in = value.parse::<usize>().map_err(|e| {+ DataFusionError::Plan(format!(+ "Failed to parse non-negative integer from '{variable}', value '{value}': {e}"+ ))+ })?;+ builder.with_max_spill_merge_fan_in(fan_in)+ }_ => return plan_err!("Unknown runtime configuration: {variable}"),// Remember to update `reset_runtime_variable()` when adding new options};@@ -1252,6 +1260,10 @@ impl SessionContext {DEFAULT_FILE_STATISTICS_MEMORY_LIMIT,);}+ "max_spill_merge_fan_in" => {+ builder =+ builder.with_max_spill_merge_fan_in(DEFAULT_MAX_SPILL_MERGE_FAN_IN);+ }_ => return plan_err!("Unknown runtime configuration: {variable}"),};*state = SessionStateBuilder::from(state.clone())diff --git a/datafusion/core/tests/sql/runtime_config.rs b/datafusion/core/tests/sql/runtime_config.rsindex 604d137540..5f1e0629ec 100644--- a/datafusion/core/tests/sql/runtime_config.rs+++ b/datafusion/core/tests/sql/runtime_config.rs@@ -227,6 +227,34 @@ async fn test_max_temp_directory_size_enforcement() {);}+#[tokio::test]+async fn test_max_spill_merge_fan_in_runtime_config() {+ let ctx = SessionContext::new();++ ctx.sql("SET datafusion.runtime.max_spill_merge_fan_in = '8'")+ .await+ .unwrap()+ .collect()+ .await+ .unwrap();+ assert_eq!(ctx.runtime_env().disk_manager.max_spill_merge_fan_in(), 8);++ ctx.sql("RESET datafusion.runtime.max_spill_merge_fan_in")+ .await+ .unwrap()+ .collect()+ .await+ .unwrap();+ assert_eq!(ctx.runtime_env().disk_manager.max_spill_merge_fan_in(), 0);++ let error = ctx+ .sql("SET datafusion.runtime.max_spill_merge_fan_in = '-1'")+ .await+ .unwrap_err()+ .to_string();+ assert!(error.contains("Failed to parse non-negative integer"));+}+#[tokio::test]async fn test_test_metadata_cache_limit() {let ctx = SessionContext::new();diff --git a/datafusion/execution/src/disk_manager.rs b/datafusion/execution/src/disk_manager.rsindex ff8403d916..8534c4f4ab 100644--- a/datafusion/execution/src/disk_manager.rs+++ b/datafusion/execution/src/disk_manager.rs@@ -32,6 +32,7 @@ use std::sync::Arc;use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};use tempfile::{Builder, NamedTempFile, TempDir};pub const DEFAULT_MAX_TEMP_DIRECTORY_SIZE: u64 = 100 * 1024 * 1024 * 1024; // 100GB+pub const DEFAULT_MAX_SPILL_MERGE_FAN_IN: usize = 0;/// Builder pattern for the [DiskManager] structure#[derive(Clone)]@@ -41,6 +42,9 @@ pub struct DiskManagerBuilder {/// The maximum amount of data (in bytes) stored inside the temporary directories./// Default to 100GBmax_temp_directory_size: u64,+ /// Maximum number of spill files opened by one external merge pass.+ /// A value of 0 means unlimited.+ max_spill_merge_fan_in: usize,}impl Debug for DiskManagerBuilder {fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {@@ -55,6 +59,7 @@ impl Default for DiskManagerBuilder {Self {mode: DiskManagerMode::OsTmpDirectory,max_temp_directory_size: DEFAULT_MAX_TEMP_DIRECTORY_SIZE,+ max_spill_merge_fan_in: DEFAULT_MAX_SPILL_MERGE_FAN_IN,}}}@@ -78,12 +83,22 @@ impl DiskManagerBuilder {self}+ pub fn set_max_spill_merge_fan_in(&mut self, value: usize) {+ self.max_spill_merge_fan_in = value;+ }++ pub fn with_max_spill_merge_fan_in(mut self, value: usize) -> Self {+ self.set_max_spill_merge_fan_in(value);+ self+ }+/// Create a DiskManager given the builderpub fn build(self) -> Result<DiskManager> {match self.mode {DiskManagerMode::OsTmpDirectory => Ok(DiskManager {local_dirs: Mutex::new(Some(vec![])),max_temp_directory_size: AtomicU64::new(self.max_temp_directory_size),+ max_spill_merge_fan_in: AtomicUsize::new(self.max_spill_merge_fan_in),used_disk_space: Arc::new(AtomicU64::new(0)),active_files_count: Arc::new(AtomicUsize::new(0)),factory: None,@@ -96,6 +111,7 @@ impl DiskManagerBuilder {Ok(DiskManager {local_dirs: Mutex::new(Some(local_dirs)),max_temp_directory_size: AtomicU64::new(self.max_temp_directory_size),+ max_spill_merge_fan_in: AtomicUsize::new(self.max_spill_merge_fan_in),used_disk_space: Arc::new(AtomicU64::new(0)),active_files_count: Arc::new(AtomicUsize::new(0)),factory: None,@@ -104,6 +120,7 @@ impl DiskManagerBuilder {DiskManagerMode::Disabled => Ok(DiskManager {local_dirs: Mutex::new(None),max_temp_directory_size: AtomicU64::new(self.max_temp_directory_size),+ max_spill_merge_fan_in: AtomicUsize::new(self.max_spill_merge_fan_in),used_disk_space: Arc::new(AtomicU64::new(0)),active_files_count: Arc::new(AtomicUsize::new(0)),factory: None,@@ -111,6 +128,7 @@ impl DiskManagerBuilder {DiskManagerMode::Custom(factory) => Ok(DiskManager {local_dirs: Mutex::new(None),max_temp_directory_size: AtomicU64::new(self.max_temp_directory_size),+ max_spill_merge_fan_in: AtomicUsize::new(self.max_spill_merge_fan_in),used_disk_space: Arc::new(AtomicU64::new(0)),active_files_count: Arc::new(AtomicUsize::new(0)),factory: Some(factory),@@ -161,6 +179,9 @@ pub struct DiskManager {/// Default to 100GB. Stored as `AtomicU64` so it can be adjusted at runtime/// without requiring exclusive (`&mut`) access to the `DiskManager`.max_temp_directory_size: AtomicU64,+ /// Maximum number of spill files opened by one external merge pass.+ /// A value of 0 preserves the memory-driven, unbounded behavior.+ max_spill_merge_fan_in: AtomicUsize,/// Used disk space in the temporary directories. Now only spilled data for/// external executors are counted.used_disk_space: Arc<AtomicU64>,@@ -250,6 +271,22 @@ impl DiskManager {self.max_temp_directory_size.load(Ordering::Relaxed)}+ /// Atomically set the maximum spill merge fan-in.+ ///+ /// A value of 0 disables the cap. Values of 1 are accepted but external+ /// merge code will still merge at least two spill streams to make progress.+ pub fn set_max_spill_merge_fan_in(&self, max_spill_merge_fan_in: usize) {+ self.max_spill_merge_fan_in+ .store(max_spill_merge_fan_in, Ordering::Relaxed);+ }++ /// Returns the maximum number of spill files opened by one merge pass.+ ///+ /// A value of 0 means unlimited.+ pub fn max_spill_merge_fan_in(&self) -> usize {+ self.max_spill_merge_fan_in.load(Ordering::Relaxed)+ }+/// Returns the current spilling progresspub fn spilling_progress(&self) -> SpillingProgress {SpillingProgress {@@ -905,6 +942,25 @@ mod tests {Ok(())}+ #[test]+ fn test_max_spill_merge_fan_in_builder_and_dynamic_update() -> Result<()> {+ let dm = Arc::new(+ DiskManager::builder()+ .with_max_spill_merge_fan_in(8)+ .build()?,+ );++ assert_eq!(dm.max_spill_merge_fan_in(), 8);++ dm.set_max_spill_merge_fan_in(4);+ assert_eq!(dm.max_spill_merge_fan_in(), 4);++ dm.set_max_spill_merge_fan_in(0);+ assert_eq!(dm.max_spill_merge_fan_in(), 0);++ Ok(())+ }+#[test]fn test_disabled_disk_manager_rejects_nonzero_limit() -> Result<()> {let dm = DiskManager::builder()diff --git a/datafusion/execution/src/runtime_env.rs b/datafusion/execution/src/runtime_env.rsindex 22b65c4189..fcfe51267e 100644--- a/datafusion/execution/src/runtime_env.rs+++ b/datafusion/execution/src/runtime_env.rs@@ -90,57 +90,77 @@ impl Debug for RuntimeEnv {}}-/// Creates runtime configuration entries with the provided values-///-/// This helper function defines the structure and metadata for all runtime configuration-/// entries to avoid duplication between `RuntimeEnv::config_entries()` and-/// `RuntimeEnvBuilder::entries()`.-fn create_runtime_config_entries(+struct RuntimeConfigValues {memory_limit: Option<String>,max_temp_directory_size: Option<String>,+ max_spill_merge_fan_in: Option<String>,temp_directory: Option<String>,metadata_cache_limit: Option<String>,list_files_cache_limit: Option<String>,list_files_cache_ttl: Option<String>,file_statistics_cache_limit: Option<String>,-) -> Vec<ConfigEntry> {- vec![- ConfigEntry {- key: "datafusion.runtime.memory_limit".to_string(),- value: memory_limit,- description: "Maximum memory limit for query execution. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.",- },- ConfigEntry {- key: "datafusion.runtime.max_temp_directory_size".to_string(),- value: max_temp_directory_size,- description: "Maximum temporary file directory size. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.",- },- ConfigEntry {- key: "datafusion.runtime.temp_directory".to_string(),- value: temp_directory,- description: "The path to the temporary file directory.",- },- ConfigEntry {- key: "datafusion.runtime.metadata_cache_limit".to_string(),- value: metadata_cache_limit,- description: "Maximum memory to use for file metadata cache such as Parquet metadata. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.",- },- ConfigEntry {- key: "datafusion.runtime.list_files_cache_limit".to_string(),- value: list_files_cache_limit,- description: "Maximum memory to use for list files cache. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.",- },- ConfigEntry {- key: "datafusion.runtime.list_files_cache_ttl".to_string(),- value: list_files_cache_ttl,- description: "TTL (time-to-live) of the entries in the list file cache. Supports units m (minutes), and s (seconds). Example: '2m' for 2 minutes.",- },- ConfigEntry {- key: "datafusion.runtime.file_statistics_cache_limit".to_string(),- value: file_statistics_cache_limit,- description: "Maximum memory to use for file statistics cache. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.",- },- ]+}++impl RuntimeConfigValues {+ /// Creates runtime configuration entries with the provided values.+ ///+ /// This defines the structure and metadata for all runtime configuration+ /// entries to avoid duplication between `RuntimeEnv::config_entries()` and+ /// `RuntimeEnvBuilder::entries()`.+ fn into_config_entries(self) -> Vec<ConfigEntry> {+ let Self {+ memory_limit,+ max_temp_directory_size,+ max_spill_merge_fan_in,+ temp_directory,+ metadata_cache_limit,+ list_files_cache_limit,+ list_files_cache_ttl,+ file_statistics_cache_limit,+ } = self;+ vec![+ ConfigEntry {+ key: "datafusion.runtime.memory_limit".to_string(),+ value: memory_limit,+ description: "Maximum memory limit for query execution. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.",+ },+ ConfigEntry {+ key: "datafusion.runtime.max_temp_directory_size".to_string(),+ value: max_temp_directory_size,+ description: "Maximum temporary file directory size. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.",+ },+ ConfigEntry {+ key: "datafusion.runtime.max_spill_merge_fan_in".to_string(),+ value: max_spill_merge_fan_in,+ description: "Maximum number of spill files opened by one external merge pass. Use 0 for unlimited. Values below 2 still use 2 so a merge can make progress.",+ },+ ConfigEntry {+ key: "datafusion.runtime.temp_directory".to_string(),+ value: temp_directory,+ description: "The path to the temporary file directory.",+ },+ ConfigEntry {+ key: "datafusion.runtime.metadata_cache_limit".to_string(),+ value: metadata_cache_limit,+ description: "Maximum memory to use for file metadata cache such as Parquet metadata. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.",+ },+ ConfigEntry {+ key: "datafusion.runtime.list_files_cache_limit".to_string(),+ value: list_files_cache_limit,+ description: "Maximum memory to use for list files cache. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.",+ },+ ConfigEntry {+ key: "datafusion.runtime.list_files_cache_ttl".to_string(),+ value: list_files_cache_ttl,+ description: "TTL (time-to-live) of the entries in the list file cache. Supports units m (minutes), and s (seconds). Example: '2m' for 2 minutes.",+ },+ ConfigEntry {+ key: "datafusion.runtime.file_statistics_cache_limit".to_string(),+ value: file_statistics_cache_limit,+ description: "Maximum memory to use for file statistics cache. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.",+ },+ ]+ }}impl RuntimeEnv {@@ -268,6 +288,8 @@ impl RuntimeEnv {let max_temp_dir_size = self.disk_manager.max_temp_directory_size();let max_temp_dir_value = format_byte_size(max_temp_dir_size);+ let max_spill_merge_fan_in =+ self.disk_manager.max_spill_merge_fan_in().to_string();let temp_paths = self.disk_manager.temp_dir_paths();let temp_dir_value = if temp_paths.is_empty() {@@ -309,15 +331,17 @@ impl RuntimeEnv {.expect("File statistics cache size conversion failed"),);- create_runtime_config_entries(- memory_limit_value,- Some(max_temp_dir_value),- temp_dir_value,- Some(metadata_cache_value),- Some(list_files_cache_value),+ RuntimeConfigValues {+ memory_limit: memory_limit_value,+ max_temp_directory_size: Some(max_temp_dir_value),+ max_spill_merge_fan_in: Some(max_spill_merge_fan_in),+ temp_directory: temp_dir_value,+ metadata_cache_limit: Some(metadata_cache_value),+ list_files_cache_limit: Some(list_files_cache_value),list_files_cache_ttl,- Some(file_statistics_cache_value),- )+ file_statistics_cache_limit: Some(file_statistics_cache_value),+ }+ .into_config_entries()}}@@ -425,6 +449,14 @@ impl RuntimeEnvBuilder {self.with_disk_manager_builder(builder.with_max_temp_directory_size(size))}+ /// Limit the number of spill files opened by one external merge pass.+ ///+ /// A value of 0 means unlimited.+ pub fn with_max_spill_merge_fan_in(mut self, fan_in: usize) -> Self {+ let builder = self.disk_manager_builder.take().unwrap_or_default();+ self.with_disk_manager_builder(builder.with_max_spill_merge_fan_in(fan_in))+ }+/// Specify the limit of the file-embedded metadata cache, in bytes.pub fn with_metadata_cache_limit(mut self, limit: usize) -> Self {self.cache_manager = self.cache_manager.with_metadata_cache_limit(limit);@@ -516,15 +548,17 @@ impl RuntimeEnvBuilder {/// Returns a list of all available runtime configurations with their current values and descriptionspub fn entries(&self) -> Vec<ConfigEntry> {- create_runtime_config_entries(- None,- Some("100G".to_string()),- None,- Some("50M".to_owned()),- Some("1M".to_owned()),- None,- Some("20M".to_owned()),- )+ RuntimeConfigValues {+ memory_limit: None,+ max_temp_directory_size: Some("100G".to_string()),+ max_spill_merge_fan_in: Some("0".to_string()),+ temp_directory: None,+ metadata_cache_limit: Some("50M".to_owned()),+ list_files_cache_limit: Some("1M".to_owned()),+ list_files_cache_ttl: None,+ file_statistics_cache_limit: Some("20M".to_owned()),+ }+ .into_config_entries()}/// Generate documentation that can be included in the user guidediff --git a/datafusion/physical-plan/src/sorts/multi_level_merge.rs b/datafusion/physical-plan/src/sorts/multi_level_merge.rsindex 8e292900b1..4d108ac046 100644--- a/datafusion/physical-plan/src/sorts/multi_level_merge.rs+++ b/datafusion/physical-plan/src/sorts/multi_level_merge.rs@@ -415,6 +415,12 @@ impl MultiLevelMergeBuilder {) -> Result<SpillFilesToMerge> {assert_ne!(buffer_len, 0, "Buffer length must be greater than 0");let mut number_of_spills_to_read_for_current_phase = 0;+ let configured_fan_in = self+ .spill_manager+ .env()+ .disk_manager+ .max_spill_merge_fan_in();+ let max_spill_files = effective_spill_merge_fan_in(configured_fan_in);// Track total memory needed for spill file buffers. When the// reservation has pre-reserved bytes (from sort_spill_reservation_bytes),// those bytes cover the first N spill files without additional pool@@ -422,6 +428,10 @@ impl MultiLevelMergeBuilder {let mut total_needed: usize = 0;for spill in &self.sorted_spill_files {+ if number_of_spills_to_read_for_current_phase >= max_spill_files {+ break;+ }+let per_spill = get_reserved_bytes_for_record_batch_size(spill.max_record_batch_memory,// Size will be the same as the sliced size, bc it is a spilled batch.@@ -617,6 +627,14 @@ fn split_batch_in_half(batch: RecordBatch) -> Vec<RecordBatch> {vec![batch.slice(0, mid), batch.slice(mid, num_rows - mid)]}+fn effective_spill_merge_fan_in(configured_fan_in: usize) -> usize {+ if configured_fan_in == 0 {+ usize::MAX+ } else {+ configured_fan_in.max(2)+ }+}+struct StreamAttachedReservation {stream: SendableRecordBatchStream,reservation: MemoryReservation,@@ -679,8 +697,8 @@ mod tests {use datafusion_execution::memory_pool::{GreedyMemoryPool, MemoryConsumer, MemoryPool,};- use datafusion_execution::runtime_env::RuntimeEnv;- use datafusion_physical_expr::expressions::Column;+ use datafusion_execution::runtime_env::{RuntimeEnv, RuntimeEnvBuilder};+ use datafusion_physical_expr::expressions::{Column, col};use datafusion_physical_expr_common::metrics::{ExecutionPlanMetricsSet, SpillMetrics,};@@ -871,6 +889,77 @@ mod tests {batches, got a largest batch of {max_batch_rows} rows");+ Ok(())+ }+ #[test]+ fn spill_merge_fan_in_is_unlimited_by_default() {+ assert_eq!(effective_spill_merge_fan_in(0), usize::MAX);+ }++ #[test]+ fn spill_merge_fan_in_preserves_merge_progress() {+ assert_eq!(effective_spill_merge_fan_in(1), 2);+ assert_eq!(effective_spill_merge_fan_in(2), 2);+ assert_eq!(effective_spill_merge_fan_in(8), 8);+ }++ #[test]+ fn spill_merge_phase_respects_configured_fan_in() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));+ let runtime = RuntimeEnvBuilder::new()+ .with_max_spill_merge_fan_in(2)+ .build_arc()?;+ let spill_manager = SpillManager::new(+ Arc::clone(&runtime),+ SpillMetrics::new(&ExecutionPlanMetricsSet::new(), 0),+ Arc::clone(&schema),+ );+ let sorted_spill_files = (0..4)+ .map(|idx| {+ Ok(SortedSpillFile {+ file: runtime+ .disk_manager+ .create_tmp_file(&format!("spill fan-in test {idx}"))?,+ max_record_batch_memory: 1,+ })+ })+ .collect::<Result<Vec<_>>>()?;+ let expr = LexOrdering::new([PhysicalSortExpr::new_default(col("a", &schema)?)])+ .unwrap();+ let reservation =+ MemoryConsumer::new("spill_merge_phase_respects_configured_fan_in")+ .register(&runtime.memory_pool);+ let metrics = BaselineMetrics::new(&ExecutionPlanMetricsSet::new(), 0);+ let mut builder = MultiLevelMergeBuilder::new(+ spill_manager,+ schema,+ sorted_spill_files,+ vec![],+ expr,+ metrics,+ 1024,+ reservation,+ None,+ false,+ );+ let mut merge_reservation = MemoryConsumer::new("spill_merge_fan_in_phase")+ .register(&runtime.memory_pool);++ let (spills, buffer_len) = match builder.get_sorted_spill_files_to_merge(+ 1,+ 2,+ &mut merge_reservation,+ )? {+ SpillFilesToMerge::Ready(spills, buffer_len) => (spills, buffer_len),+ SpillFilesToMerge::SplitThenRetry(index) => {+ panic!("expected ready spill files, got retry for index {index}")+ }+ };++ assert_eq!(spills.len(), 2);+ assert_eq!(buffer_len, 1);+ assert_eq!(builder.sorted_spill_files.len(), 2);+Ok(())}}diff --git a/datafusion/physical-plan/src/spill/spill_manager.rs b/datafusion/physical-plan/src/spill/spill_manager.rsindex 3f305c16a6..aee9e917c7 100644--- a/datafusion/physical-plan/src/spill/spill_manager.rs+++ b/datafusion/physical-plan/src/spill/spill_manager.rs@@ -76,6 +76,10 @@ impl SpillManager {&self.schema}+ pub(crate) fn env(&self) -> &RuntimeEnv {+ &self.env+ }+/// Creates a temporary file for in-progress operations, returning an error/// message if file creation fails. The file can be used to append batches/// incrementally and then finish the file when done.diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.sltindex 50f063e8d2..bf45564e26 100644--- a/datafusion/sqllogictest/test_files/information_schema.slt+++ b/datafusion/sqllogictest/test_files/information_schema.slt@@ -344,6 +344,7 @@ datafusion.optimizer.use_statistics_registry falsedatafusion.runtime.file_statistics_cache_limit 20Mdatafusion.runtime.list_files_cache_limit 1Mdatafusion.runtime.list_files_cache_ttl NULL+datafusion.runtime.max_spill_merge_fan_in 0datafusion.runtime.max_temp_directory_size 100Gdatafusion.runtime.memory_limit unlimiteddatafusion.runtime.metadata_cache_limit 50M@@ -502,6 +503,7 @@ datafusion.optimizer.use_statistics_registry false When set to true, the physicadatafusion.runtime.file_statistics_cache_limit 20M Maximum memory to use for file statistics cache. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.datafusion.runtime.list_files_cache_limit 1M Maximum memory to use for list files cache. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.datafusion.runtime.list_files_cache_ttl NULL TTL (time-to-live) of the entries in the list file cache. Supports units m (minutes), and s (seconds). Example: '2m' for 2 minutes.+datafusion.runtime.max_spill_merge_fan_in 0 Maximum number of spill files opened by one external merge pass. Use 0 for unlimited. Values below 2 still use 2 so a merge can make progress.datafusion.runtime.max_temp_directory_size 100G Maximum temporary file directory size. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.datafusion.runtime.memory_limit unlimited Maximum memory limit for query execution. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.datafusion.runtime.metadata_cache_limit 50M Maximum memory to use for file metadata cache such as Parquet metadata. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.diff --git a/datafusion/sqllogictest/test_files/push_down_filter_regression.slt b/datafusion/sqllogictest/test_files/push_down_filter_regression.sltindex f28a314764..7ab5e7c79d 100644--- a/datafusion/sqllogictest/test_files/push_down_filter_regression.slt+++ b/datafusion/sqllogictest/test_files/push_down_filter_regression.slt@@ -222,7 +222,7 @@ set datafusion.execution.collect_statistics = true;# execution (the order in which Partial aggregates publish dynamic filter# updates races against when the scan reads each partition). The original# Rust test only asserted matched < 4; the important invariant here is-# that the DynamicFilter text is correct.+# that dynamic filtering is applied and metrics are suppressed.statement okset datafusion.explain.analyze_level = summary;@@ -236,7 +236,7 @@ Plan with Metrics01)AggregateExec: mode=Final, gby=[], aggr=[max(agg_dyn_e2e.column1)], metrics=[]02)--CoalescePartitionsExec, metrics=[]03)----AggregateExec: mode=Partial, gby=[], aggr=[max(agg_dyn_e2e.column1)], metrics=[]-04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_dyn/file_0.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_dyn/file_1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_dyn/file_2.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_dyn/file_3.parquet]]}, projection=[column1], file_type=parquet, predicate=column1@0 > 1 AND DynamicFilter [ column1@0 > 4 ], dynamic_rg_pruning=eligible, pruning_predicate=column1_null_count@1 != row_count@2 AND column1_max@0 > 1 AND column1_null_count@1 != row_count@2 AND column1_max@0 > 4, required_guarantees=[], metrics=[]+04)------DataSourceExec: file_groups=<slt:ignore> projection=[column1], file_type=parquet, predicate=column1@0 > 1 AND DynamicFilter [ column1@0 > <slt:ignore> ], dynamic_rg_pruning=eligible, pruning_predicate=column1_null_count@1 != row_count@2 AND column1_max@0 > 1 AND column1_null_count@1 != row_count@2 AND column1_max@0 > <slt:ignore>, required_guarantees=[], metrics=[]statement okreset datafusion.explain.analyze_categories;diff --git a/datafusion/sqllogictest/test_files/set_variable.slt b/datafusion/sqllogictest/test_files/set_variable.sltindex 0514deba28..c86c0007b6 100644--- a/datafusion/sqllogictest/test_files/set_variable.slt+++ b/datafusion/sqllogictest/test_files/set_variable.slt@@ -611,6 +611,18 @@ SHOW datafusion.runtime.max_temp_directory_size----datafusion.runtime.max_temp_directory_size 10G+# Test SET and SHOW runtime.max_spill_merge_fan_in+statement ok+SET datafusion.runtime.max_spill_merge_fan_in = '16'++query TT+SHOW datafusion.runtime.max_spill_merge_fan_in+----+datafusion.runtime.max_spill_merge_fan_in 16++statement ok+RESET datafusion.runtime.max_spill_merge_fan_in+# Test SET and SHOW runtime.file_statistics_cache_limitstatement okSET datafusion.runtime.file_statistics_cache_limit = '42M'@@ -669,6 +681,7 @@ SELECT name FROM information_schema.df_settings WHERE name LIKE 'datafusion.runtdatafusion.runtime.file_statistics_cache_limitdatafusion.runtime.list_files_cache_limitdatafusion.runtime.list_files_cache_ttl+datafusion.runtime.max_spill_merge_fan_indatafusion.runtime.max_temp_directory_sizedatafusion.runtime.memory_limitdatafusion.runtime.metadata_cache_limitdiff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.mdindex 945f2622c2..03340c366d 100644--- a/docs/source/user-guide/configs.md+++ b/docs/source/user-guide/configs.md@@ -244,6 +244,7 @@ The following runtime configuration settings are available:| datafusion.runtime.file_statistics_cache_limit | 20M | Maximum memory to use for file statistics cache. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes. || datafusion.runtime.list_files_cache_limit | 1M | Maximum memory to use for list files cache. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes. || datafusion.runtime.list_files_cache_ttl | NULL | TTL (time-to-live) of the entries in the list file cache. Supports units m (minutes), and s (seconds). Example: '2m' for 2 minutes. |+| datafusion.runtime.max_spill_merge_fan_in | 0 | Maximum number of spill files opened by one external merge pass. Use 0 for unlimited. Values below 2 still use 2 so a merge can make progress. || datafusion.runtime.max_temp_directory_size | 100G | Maximum temporary file directory size. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes. || datafusion.runtime.memory_limit | NULL | Maximum memory limit for query execution. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes. || datafusion.runtime.metadata_cache_limit | 50M | Maximum memory to use for file metadata cache such as Parquet metadata. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes. |