Skip to content

Apache DataFusion SQL Query Engine

minor: validate config `soft_max_rows_per_output_file` when setting it (#23597)

Yongting You18121a68433a
diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs
index fb0345eba2..6d23d26418 100644
--- a/datafusion/common/src/config.rs
+++ b/datafusion/common/src/config.rs
@@ -873,7 +873,7 @@ config_namespace! {
/// This is a soft max, so it can be exceeded slightly. There also
/// will be one file smaller than the limit if the total
/// number of rows written is not roughly divisible by the soft max
- pub soft_max_rows_per_output_file: usize, default = 50000000
+ pub soft_max_rows_per_output_file: ConfigNonZeroUsize, default = non_zero_usize_default(50000000)
/// This is the maximum number of RecordBatches buffered
/// for each output file being worked. Higher values can potentially
diff --git a/datafusion/datasource/src/write/demux.rs b/datafusion/datasource/src/write/demux.rs
index acc6435acf..f73c03d81d 100644
--- a/datafusion/datasource/src/write/demux.rs
+++ b/datafusion/datasource/src/write/demux.rs
@@ -153,7 +153,7 @@ async fn row_count_demuxer(
) -> Result<()> {
let exec_options = &context.session_config().options().execution;
- let max_rows_per_file = exec_options.soft_max_rows_per_output_file;
+ let max_rows_per_file = exec_options.soft_max_rows_per_output_file.get();
let max_buffered_batches = exec_options.max_buffered_batches_per_output_file;
let minimum_parallel_files = exec_options.minimum_parallel_output_files;
let mut part_idx = 0;
diff --git a/datafusion/sqllogictest/test_files/set_variable.slt b/datafusion/sqllogictest/test_files/set_variable.slt
index 009c5e0d93..4101c1dc70 100644
--- a/datafusion/sqllogictest/test_files/set_variable.slt
+++ b/datafusion/sqllogictest/test_files/set_variable.slt
@@ -724,6 +724,9 @@ SET datafusion.runtime.list_files_cache_ttl = '1m18446744073709551556s'
statement error DataFusion error: Invalid or Unsupported Configuration: value must be greater than 0
SET datafusion.execution.batch_size = 0
+statement error DataFusion error: Invalid or Unsupported Configuration: value must be greater than 0
+SET datafusion.execution.soft_max_rows_per_output_file = 0
+
statement error DataFusion error: Invalid or Unsupported Configuration: value must be greater than 0
SET datafusion.execution.max_spill_file_size_bytes = 0