Apache DataFusion SQL Query Engine
minor: validate config `minimum_parallel_output_files` when setting it (#23596)
Yongting You67907a6fc51e
diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rsindex 6d23d26418..bb2e36fc5f 100644--- a/datafusion/common/src/config.rs+++ b/datafusion/common/src/config.rs@@ -867,7 +867,7 @@ config_namespace! {/// RecordBatches will be distributed in round robin fashion to each/// parallel writer. Each writer is closed and a new file opened once/// soft_max_rows_per_output_file is reached.- pub minimum_parallel_output_files: usize, default = 4+ pub minimum_parallel_output_files: ConfigNonZeroUsize, default = non_zero_usize_default(4)/// Target number of rows in output files when writing multiple./// This is a soft max, so it can be exceeded slightly. There alsodiff --git a/datafusion/datasource/src/write/demux.rs b/datafusion/datasource/src/write/demux.rsindex f73c03d81d..6d7de53890 100644--- a/datafusion/datasource/src/write/demux.rs+++ b/datafusion/datasource/src/write/demux.rs@@ -155,7 +155,7 @@ async fn row_count_demuxer(let max_rows_per_file = exec_options.soft_max_rows_per_output_file.get();let max_buffered_batches = exec_options.max_buffered_batches_per_output_file;- let minimum_parallel_files = exec_options.minimum_parallel_output_files;+ let minimum_parallel_files = exec_options.minimum_parallel_output_files.get();let mut part_idx = 0;let write_id = rand::distr::Alphanumeric.sample_string(&mut rand::rng(), 16);diff --git a/datafusion/sqllogictest/test_files/set_variable.slt b/datafusion/sqllogictest/test_files/set_variable.sltindex 4101c1dc70..15267e6a2a 100644--- a/datafusion/sqllogictest/test_files/set_variable.slt+++ b/datafusion/sqllogictest/test_files/set_variable.slt@@ -724,6 +724,9 @@ SET datafusion.runtime.list_files_cache_ttl = '1m18446744073709551556s'statement error DataFusion error: Invalid or Unsupported Configuration: value must be greater than 0SET datafusion.execution.batch_size = 0+statement error DataFusion error: Invalid or Unsupported Configuration: value must be greater than 0+SET datafusion.execution.minimum_parallel_output_files = 0+statement error DataFusion error: Invalid or Unsupported Configuration: value must be greater than 0SET datafusion.execution.soft_max_rows_per_output_file = 0