Apache DataFusion SQL Query Engine
minor: validate config `meta_fetch_concurrency` when setting it (#23595)
Yongting You1adbd84e4f97
diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rsindex 23c67efa74..ce739e5019 100644--- a/datafusion/catalog-listing/src/table.rs+++ b/datafusion/catalog-listing/src/table.rs@@ -815,7 +815,7 @@ impl ListingTable {})).await?;let meta_fetch_concurrency =- ctx.config_options().execution.meta_fetch_concurrency;+ ctx.config_options().execution.meta_fetch_concurrency.get();let file_list = stream::iter(file_list).flatten_unordered(meta_fetch_concurrency);// collect the statistics and ordering if required by the configlet files = file_list@@ -832,7 +832,9 @@ impl ListingTable {.with_ordering(ordering))}).boxed()- .buffer_unordered(ctx.config_options().execution.meta_fetch_concurrency);+ .buffer_unordered(+ ctx.config_options().execution.meta_fetch_concurrency.get(),+ );get_files_with_limit(files, file_limit, ctx.config().collect_statistics()).await}diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rsindex bb2e36fc5f..81f573fc2a 100644--- a/datafusion/common/src/config.rs+++ b/datafusion/common/src/config.rs@@ -861,7 +861,7 @@ config_namespace! {pub max_spill_file_size_bytes: ConfigNonZeroUsize, default = non_zero_usize_default(128 * 1024 * 1024)/// Number of files to read in parallel when inferring schema and statistics- pub meta_fetch_concurrency: usize, default = 32+ pub meta_fetch_concurrency: ConfigNonZeroUsize, default = non_zero_usize_default(32)/// Guarantees a minimum level of output files running in parallel./// RecordBatches will be distributed in round robin fashion to each@@ -1932,7 +1932,8 @@ impl ConfigOptions {}return Ok(());}- return ConfigField::set(self, inner_key, value);+ return ConfigField::set(self, inner_key, value)+ .map_err(|e| e.context(format!("Error setting config {key}")));}if !self.extensions.0.contains_key(prefix)@@ -3771,6 +3772,7 @@ impl Display for OutputFormat {#[cfg(test)]mod tests {#[cfg(feature = "parquet")]+ use crate::assert_contains;use crate::config::TableParquetOptions;use crate::config::{ConfigEntry, ConfigExtension, ConfigField, ConfigFileType, ExtensionOptions,@@ -4331,7 +4333,7 @@ mod tests {let err = config.set("datafusion.execution.parquet.writer_version", "3.0").unwrap_err();- assert_eq!(+ assert_contains!(err.to_string(),"Invalid or Unsupported Configuration: Invalid parquet writer version: 3.0. Expected one of: 1.0, 2.0");diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rsindex db2623c24d..56a5d57795 100644--- a/datafusion/core/src/datasource/listing/table.rs+++ b/datafusion/core/src/datasource/listing/table.rs@@ -542,7 +542,8 @@ mod tests {.state().config_options().execution- .meta_fetch_concurrency;+ .meta_fetch_concurrency+ .get();let expected_concurrency = files.len().min(meta_fetch_concurrency);let head_concurrency_store = ensure_head_concurrency(store, expected_concurrency);diff --git a/datafusion/core/tests/config_from_env.rs b/datafusion/core/tests/config_from_env.rsindex 6b09a6367d..15a047cbbd 100644--- a/datafusion/core/tests/config_from_env.rs+++ b/datafusion/core/tests/config_from_env.rs@@ -16,6 +16,7 @@// under the License.use datafusion::config::ConfigOptions;+use datafusion_common::assert_contains;use std::env;#[test]@@ -34,7 +35,7 @@ fn from_env() {// invalid testingenv::set_var(env_key, "ttruee");let err = ConfigOptions::from_env().unwrap_err().strip_backtrace();- assert_eq!(+ assert_contains!(err,"Error parsing 'ttruee' as bool\ncaused by\nExternal error: provided string was not `true` or `false`");@@ -50,7 +51,7 @@ fn from_env() {// for invalid testingenv::set_var(env_key, "abc");let err = ConfigOptions::from_env().unwrap_err().strip_backtrace();- assert_eq!(+ assert_contains!(err,"Error parsing 'abc' as usize\ncaused by\nExternal error: invalid digit found in string");diff --git a/datafusion/datasource-parquet/src/file_format.rs b/datafusion/datasource-parquet/src/file_format.rsindex e89cff2aaf..29083ebfb2 100644--- a/datafusion/datasource-parquet/src/file_format.rs+++ b/datafusion/datasource-parquet/src/file_format.rs@@ -367,7 +367,13 @@ impl FileFormat for ParquetFormat {}).boxed() // Workaround https://github.com/rust-lang/rust/issues/64552// fetch schemas concurrently, if requested- .buffer_unordered(state.config_options().execution.meta_fetch_concurrency)+ .buffer_unordered(+ state+ .config_options()+ .execution+ .meta_fetch_concurrency+ .get(),+ ).try_collect().await?;diff --git a/datafusion/sqllogictest/test_files/explain.slt b/datafusion/sqllogictest/test_files/explain.sltindex 24b1262e02..5405c7ce0e 100644--- a/datafusion/sqllogictest/test_files/explain.slt+++ b/datafusion/sqllogictest/test_files/explain.slt@@ -533,8 +533,13 @@ query error DataFusion error: Error during planning: EXPLAIN VERBOSE with FORMATexplain verbose format tree select * from values (1);# valid explain format-query error DataFusion error: Invalid or Unsupported Configuration: Invalid explain format. Expected 'indent', 'tree', 'pgjson' or 'graphviz'. Got 'xxx'+query errorset datafusion.explain.format = "xxx";+----+DataFusion error: Error setting config datafusion.explain.format+caused by+Invalid or Unsupported Configuration: Invalid explain format. Expected 'indent', 'tree', 'pgjson' or 'graphviz'. Got 'xxx'+# verbose uses indent mode even when a different mode (e.g tree) is setdiff --git a/datafusion/sqllogictest/test_files/set_variable.slt b/datafusion/sqllogictest/test_files/set_variable.sltindex 15267e6a2a..7da06b2fff 100644--- a/datafusion/sqllogictest/test_files/set_variable.slt+++ b/datafusion/sqllogictest/test_files/set_variable.slt@@ -93,10 +93,10 @@ datafusion.execution.coalesce_batches falsestatement okset datafusion.catalog.information_schema = true-statement error DataFusion error: Error parsing '1' as bool+statement error DataFusion error: Error setting config datafusion\.execution\.coalesce_batches\ncaused by\nError parsing '1' as boolSET datafusion.execution.coalesce_batches to 1-statement error DataFusion error: Error parsing 'abc' as bool+statement error DataFusion error: Error setting config datafusion\.execution\.coalesce_batches\ncaused by\nError parsing 'abc' as boolSET datafusion.execution.coalesce_batches to abc# set u64 variable@@ -132,10 +132,10 @@ datafusion.execution.batch_size 2statement okset datafusion.catalog.information_schema = true-statement error DataFusion error: Error parsing '-1' as usize+statement error DataFusion error: Error setting config datafusion\.execution\.batch_size\ncaused by\nError parsing '-1' as usizeSET datafusion.execution.batch_size to -1-statement error DataFusion error: Error parsing 'abc' as usize+statement error DataFusion error: Error setting config datafusion\.execution\.batch_size\ncaused by\nError parsing 'abc' as usizeSET datafusion.execution.batch_size to abcstatement error External error: invalid digit found in string@@ -580,7 +580,7 @@ SHOW datafusion.format.date_formatdatafusion.format.date_format %Y-%m-%d# Invalid format option name-statement error DataFusion error: Invalid or Unsupported Configuration: Config value "unknown_option" not found on FormatOptions+statement error DataFusion error: Error setting config datafusion\.format\.unknown_option\ncaused by\nInvalid or Unsupported Configuration: Config value "unknown_option" not found on FormatOptionsSET datafusion.format.unknown_option = true############@@ -721,20 +721,43 @@ statement error DataFusion error: Error during planning: Duration has overflowedSET datafusion.runtime.list_files_cache_ttl = '1m18446744073709551556s'# Set invalid value and ensures error-statement error DataFusion error: Invalid or Unsupported Configuration: value must be greater than 0+statement error DataFusion error: Error setting config datafusion\.execution\.batch_size\ncaused by\nInvalid or Unsupported Configuration: value must be greater than 0SET datafusion.execution.batch_size = 0-statement error DataFusion error: Invalid or Unsupported Configuration: value must be greater than 0+statement error DataFusion error: Error setting config datafusion\.execution\.meta_fetch_concurrency\ncaused by\nInvalid or Unsupported Configuration: value must be greater than 0+SET datafusion.execution.meta_fetch_concurrency = 0++statement errorSET datafusion.execution.minimum_parallel_output_files = 0+----+DataFusion error: Error setting config datafusion.execution.minimum_parallel_output_files+caused by+Invalid or Unsupported Configuration: value must be greater than 0+-statement error DataFusion error: Invalid or Unsupported Configuration: value must be greater than 0+statement errorSET datafusion.execution.soft_max_rows_per_output_file = 0+----+DataFusion error: Error setting config datafusion.execution.soft_max_rows_per_output_file+caused by+Invalid or Unsupported Configuration: value must be greater than 0+-statement error DataFusion error: Invalid or Unsupported Configuration: value must be greater than 0+statement errorSET datafusion.execution.max_spill_file_size_bytes = 0+----+DataFusion error: Error setting config datafusion.execution.max_spill_file_size_bytes+caused by+Invalid or Unsupported Configuration: value must be greater than 0+-statement error DataFusion error: Invalid or Unsupported Configuration: value must be greater than 0+statement errorSET datafusion.sql_parser.recursion_limit = 0+----+DataFusion error: Error setting config datafusion.sql_parser.recursion_limit+caused by+Invalid or Unsupported Configuration: value must be greater than 0+# Config resetstatement okdiff --git a/datafusion/sqllogictest/test_files/spark/map/str_to_map.slt b/datafusion/sqllogictest/test_files/spark/map/str_to_map.sltindex f422b50dfa..c1307468d7 100644--- a/datafusion/sqllogictest/test_files/spark/map/str_to_map.slt+++ b/datafusion/sqllogictest/test_files/spark/map/str_to_map.slt@@ -159,5 +159,9 @@ statement okset datafusion.spark.map_key_dedup_policy = 'EXCEPTION';# Invalid policy values are rejected at SET time with a clear message.-statement error DataFusion error: Invalid or Unsupported Configuration: Invalid MapKeyDedupPolicy: BOGUS\. Expected one of: EXCEPTION, LAST_WIN+statement errorset datafusion.spark.map_key_dedup_policy = 'BOGUS';+----+DataFusion error: Error setting config datafusion.spark.map_key_dedup_policy+caused by+Invalid or Unsupported Configuration: Invalid MapKeyDedupPolicy: BOGUS. Expected one of: EXCEPTION, LAST_WIN