Apache DataFusion SQL Query Engine
minor: validate config `soft_max_rows_per_output_file` when setting it (#23597)
Yongting You18121a68433a
diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rsindex fb0345eba2..6d23d26418 100644--- a/datafusion/common/src/config.rs+++ b/datafusion/common/src/config.rs@@ -873,7 +873,7 @@ config_namespace! {/// This is a soft max, so it can be exceeded slightly. There also/// will be one file smaller than the limit if the total/// number of rows written is not roughly divisible by the soft max- pub soft_max_rows_per_output_file: usize, default = 50000000+ pub soft_max_rows_per_output_file: ConfigNonZeroUsize, default = non_zero_usize_default(50000000)/// This is the maximum number of RecordBatches buffered/// for each output file being worked. Higher values can potentiallydiff --git a/datafusion/datasource/src/write/demux.rs b/datafusion/datasource/src/write/demux.rsindex acc6435acf..f73c03d81d 100644--- a/datafusion/datasource/src/write/demux.rs+++ b/datafusion/datasource/src/write/demux.rs@@ -153,7 +153,7 @@ async fn row_count_demuxer() -> Result<()> {let exec_options = &context.session_config().options().execution;- let max_rows_per_file = exec_options.soft_max_rows_per_output_file;+ let max_rows_per_file = exec_options.soft_max_rows_per_output_file.get();let max_buffered_batches = exec_options.max_buffered_batches_per_output_file;let minimum_parallel_files = exec_options.minimum_parallel_output_files;let mut part_idx = 0;diff --git a/datafusion/sqllogictest/test_files/set_variable.slt b/datafusion/sqllogictest/test_files/set_variable.sltindex 009c5e0d93..4101c1dc70 100644--- a/datafusion/sqllogictest/test_files/set_variable.slt+++ b/datafusion/sqllogictest/test_files/set_variable.slt@@ -724,6 +724,9 @@ SET datafusion.runtime.list_files_cache_ttl = '1m18446744073709551556s'statement error DataFusion error: Invalid or Unsupported Configuration: value must be greater than 0SET datafusion.execution.batch_size = 0+statement error DataFusion error: Invalid or Unsupported Configuration: value must be greater than 0+SET datafusion.execution.soft_max_rows_per_output_file = 0+statement error DataFusion error: Invalid or Unsupported Configuration: value must be greater than 0SET datafusion.execution.max_spill_file_size_bytes = 0