Skip to content

Apache DataFusion SQL Query Engine

minor: validate config `minimum_parallel_output_files` when setting it (#23596)

Yongting You67907a6fc51e
diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs
index 6d23d26418..bb2e36fc5f 100644
--- a/datafusion/common/src/config.rs
+++ b/datafusion/common/src/config.rs
@@ -867,7 +867,7 @@ config_namespace! {
/// RecordBatches will be distributed in round robin fashion to each
/// parallel writer. Each writer is closed and a new file opened once
/// soft_max_rows_per_output_file is reached.
- pub minimum_parallel_output_files: usize, default = 4
+ pub minimum_parallel_output_files: ConfigNonZeroUsize, default = non_zero_usize_default(4)
/// Target number of rows in output files when writing multiple.
/// This is a soft max, so it can be exceeded slightly. There also
diff --git a/datafusion/datasource/src/write/demux.rs b/datafusion/datasource/src/write/demux.rs
index f73c03d81d..6d7de53890 100644
--- a/datafusion/datasource/src/write/demux.rs
+++ b/datafusion/datasource/src/write/demux.rs
@@ -155,7 +155,7 @@ async fn row_count_demuxer(
let max_rows_per_file = exec_options.soft_max_rows_per_output_file.get();
let max_buffered_batches = exec_options.max_buffered_batches_per_output_file;
- let minimum_parallel_files = exec_options.minimum_parallel_output_files;
+ let minimum_parallel_files = exec_options.minimum_parallel_output_files.get();
let mut part_idx = 0;
let write_id = rand::distr::Alphanumeric.sample_string(&mut rand::rng(), 16);
diff --git a/datafusion/sqllogictest/test_files/set_variable.slt b/datafusion/sqllogictest/test_files/set_variable.slt
index 4101c1dc70..15267e6a2a 100644
--- a/datafusion/sqllogictest/test_files/set_variable.slt
+++ b/datafusion/sqllogictest/test_files/set_variable.slt
@@ -724,6 +724,9 @@ SET datafusion.runtime.list_files_cache_ttl = '1m18446744073709551556s'
statement error DataFusion error: Invalid or Unsupported Configuration: value must be greater than 0
SET datafusion.execution.batch_size = 0
+statement error DataFusion error: Invalid or Unsupported Configuration: value must be greater than 0
+SET datafusion.execution.minimum_parallel_output_files = 0
+
statement error DataFusion error: Invalid or Unsupported Configuration: value must be greater than 0
SET datafusion.execution.soft_max_rows_per_output_file = 0