Skip to content

Apache DataFusion SQL Query Engine

minor: validate config `meta_fetch_concurrency` when setting it (#23595)

Yongting You1adbd84e4f97
diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rs
index 23c67efa74..ce739e5019 100644
--- a/datafusion/catalog-listing/src/table.rs
+++ b/datafusion/catalog-listing/src/table.rs
@@ -815,7 +815,7 @@ impl ListingTable {
}))
.await?;
let meta_fetch_concurrency =
- ctx.config_options().execution.meta_fetch_concurrency;
+ ctx.config_options().execution.meta_fetch_concurrency.get();
let file_list = stream::iter(file_list).flatten_unordered(meta_fetch_concurrency);
// collect the statistics and ordering if required by the config
let files = file_list
@@ -832,7 +832,9 @@ impl ListingTable {
.with_ordering(ordering))
})
.boxed()
- .buffer_unordered(ctx.config_options().execution.meta_fetch_concurrency);
+ .buffer_unordered(
+ ctx.config_options().execution.meta_fetch_concurrency.get(),
+ );
get_files_with_limit(files, file_limit, ctx.config().collect_statistics()).await
}
diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs
index bb2e36fc5f..81f573fc2a 100644
--- a/datafusion/common/src/config.rs
+++ b/datafusion/common/src/config.rs
@@ -861,7 +861,7 @@ config_namespace! {
pub max_spill_file_size_bytes: ConfigNonZeroUsize, default = non_zero_usize_default(128 * 1024 * 1024)
/// Number of files to read in parallel when inferring schema and statistics
- pub meta_fetch_concurrency: usize, default = 32
+ pub meta_fetch_concurrency: ConfigNonZeroUsize, default = non_zero_usize_default(32)
/// Guarantees a minimum level of output files running in parallel.
/// RecordBatches will be distributed in round robin fashion to each
@@ -1932,7 +1932,8 @@ impl ConfigOptions {
}
return Ok(());
}
- return ConfigField::set(self, inner_key, value);
+ return ConfigField::set(self, inner_key, value)
+ .map_err(|e| e.context(format!("Error setting config {key}")));
}
if !self.extensions.0.contains_key(prefix)
@@ -3771,6 +3772,7 @@ impl Display for OutputFormat {
#[cfg(test)]
mod tests {
#[cfg(feature = "parquet")]
+ use crate::assert_contains;
use crate::config::TableParquetOptions;
use crate::config::{
ConfigEntry, ConfigExtension, ConfigField, ConfigFileType, ExtensionOptions,
@@ -4331,7 +4333,7 @@ mod tests {
let err = config
.set("datafusion.execution.parquet.writer_version", "3.0")
.unwrap_err();
- assert_eq!(
+ assert_contains!(
err.to_string(),
"Invalid or Unsupported Configuration: Invalid parquet writer version: 3.0. Expected one of: 1.0, 2.0"
);
diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rs
index db2623c24d..56a5d57795 100644
--- a/datafusion/core/src/datasource/listing/table.rs
+++ b/datafusion/core/src/datasource/listing/table.rs
@@ -542,7 +542,8 @@ mod tests {
.state()
.config_options()
.execution
- .meta_fetch_concurrency;
+ .meta_fetch_concurrency
+ .get();
let expected_concurrency = files.len().min(meta_fetch_concurrency);
let head_concurrency_store = ensure_head_concurrency(store, expected_concurrency);
diff --git a/datafusion/core/tests/config_from_env.rs b/datafusion/core/tests/config_from_env.rs
index 6b09a6367d..15a047cbbd 100644
--- a/datafusion/core/tests/config_from_env.rs
+++ b/datafusion/core/tests/config_from_env.rs
@@ -16,6 +16,7 @@
// under the License.
use datafusion::config::ConfigOptions;
+use datafusion_common::assert_contains;
use std::env;
#[test]
@@ -34,7 +35,7 @@ fn from_env() {
// invalid testing
env::set_var(env_key, "ttruee");
let err = ConfigOptions::from_env().unwrap_err().strip_backtrace();
- assert_eq!(
+ assert_contains!(
err,
"Error parsing 'ttruee' as bool\ncaused by\nExternal error: provided string was not `true` or `false`"
);
@@ -50,7 +51,7 @@ fn from_env() {
// for invalid testing
env::set_var(env_key, "abc");
let err = ConfigOptions::from_env().unwrap_err().strip_backtrace();
- assert_eq!(
+ assert_contains!(
err,
"Error parsing 'abc' as usize\ncaused by\nExternal error: invalid digit found in string"
);
diff --git a/datafusion/datasource-parquet/src/file_format.rs b/datafusion/datasource-parquet/src/file_format.rs
index e89cff2aaf..29083ebfb2 100644
--- a/datafusion/datasource-parquet/src/file_format.rs
+++ b/datafusion/datasource-parquet/src/file_format.rs
@@ -367,7 +367,13 @@ impl FileFormat for ParquetFormat {
})
.boxed() // Workaround https://github.com/rust-lang/rust/issues/64552
// fetch schemas concurrently, if requested
- .buffer_unordered(state.config_options().execution.meta_fetch_concurrency)
+ .buffer_unordered(
+ state
+ .config_options()
+ .execution
+ .meta_fetch_concurrency
+ .get(),
+ )
.try_collect()
.await?;
diff --git a/datafusion/sqllogictest/test_files/explain.slt b/datafusion/sqllogictest/test_files/explain.slt
index 24b1262e02..5405c7ce0e 100644
--- a/datafusion/sqllogictest/test_files/explain.slt
+++ b/datafusion/sqllogictest/test_files/explain.slt
@@ -533,8 +533,13 @@ query error DataFusion error: Error during planning: EXPLAIN VERBOSE with FORMAT
explain verbose format tree select * from values (1);
# valid explain format
-query error DataFusion error: Invalid or Unsupported Configuration: Invalid explain format. Expected 'indent', 'tree', 'pgjson' or 'graphviz'. Got 'xxx'
+query error
set datafusion.explain.format = "xxx";
+----
+DataFusion error: Error setting config datafusion.explain.format
+caused by
+Invalid or Unsupported Configuration: Invalid explain format. Expected 'indent', 'tree', 'pgjson' or 'graphviz'. Got 'xxx'
+
# verbose uses indent mode even when a different mode (e.g tree) is set
diff --git a/datafusion/sqllogictest/test_files/set_variable.slt b/datafusion/sqllogictest/test_files/set_variable.slt
index 15267e6a2a..7da06b2fff 100644
--- a/datafusion/sqllogictest/test_files/set_variable.slt
+++ b/datafusion/sqllogictest/test_files/set_variable.slt
@@ -93,10 +93,10 @@ datafusion.execution.coalesce_batches false
statement ok
set datafusion.catalog.information_schema = true
-statement error DataFusion error: Error parsing '1' as bool
+statement error DataFusion error: Error setting config datafusion\.execution\.coalesce_batches\ncaused by\nError parsing '1' as bool
SET datafusion.execution.coalesce_batches to 1
-statement error DataFusion error: Error parsing 'abc' as bool
+statement error DataFusion error: Error setting config datafusion\.execution\.coalesce_batches\ncaused by\nError parsing 'abc' as bool
SET datafusion.execution.coalesce_batches to abc
# set u64 variable
@@ -132,10 +132,10 @@ datafusion.execution.batch_size 2
statement ok
set datafusion.catalog.information_schema = true
-statement error DataFusion error: Error parsing '-1' as usize
+statement error DataFusion error: Error setting config datafusion\.execution\.batch_size\ncaused by\nError parsing '-1' as usize
SET datafusion.execution.batch_size to -1
-statement error DataFusion error: Error parsing 'abc' as usize
+statement error DataFusion error: Error setting config datafusion\.execution\.batch_size\ncaused by\nError parsing 'abc' as usize
SET datafusion.execution.batch_size to abc
statement error External error: invalid digit found in string
@@ -580,7 +580,7 @@ SHOW datafusion.format.date_format
datafusion.format.date_format %Y-%m-%d
# Invalid format option name
-statement error DataFusion error: Invalid or Unsupported Configuration: Config value "unknown_option" not found on FormatOptions
+statement error DataFusion error: Error setting config datafusion\.format\.unknown_option\ncaused by\nInvalid or Unsupported Configuration: Config value "unknown_option" not found on FormatOptions
SET datafusion.format.unknown_option = true
############
@@ -721,20 +721,43 @@ statement error DataFusion error: Error during planning: Duration has overflowed
SET datafusion.runtime.list_files_cache_ttl = '1m18446744073709551556s'
# Set invalid value and ensures error
-statement error DataFusion error: Invalid or Unsupported Configuration: value must be greater than 0
+statement error DataFusion error: Error setting config datafusion\.execution\.batch_size\ncaused by\nInvalid or Unsupported Configuration: value must be greater than 0
SET datafusion.execution.batch_size = 0
-statement error DataFusion error: Invalid or Unsupported Configuration: value must be greater than 0
+statement error DataFusion error: Error setting config datafusion\.execution\.meta_fetch_concurrency\ncaused by\nInvalid or Unsupported Configuration: value must be greater than 0
+SET datafusion.execution.meta_fetch_concurrency = 0
+
+statement error
SET datafusion.execution.minimum_parallel_output_files = 0
+----
+DataFusion error: Error setting config datafusion.execution.minimum_parallel_output_files
+caused by
+Invalid or Unsupported Configuration: value must be greater than 0
+
-statement error DataFusion error: Invalid or Unsupported Configuration: value must be greater than 0
+statement error
SET datafusion.execution.soft_max_rows_per_output_file = 0
+----
+DataFusion error: Error setting config datafusion.execution.soft_max_rows_per_output_file
+caused by
+Invalid or Unsupported Configuration: value must be greater than 0
+
-statement error DataFusion error: Invalid or Unsupported Configuration: value must be greater than 0
+statement error
SET datafusion.execution.max_spill_file_size_bytes = 0
+----
+DataFusion error: Error setting config datafusion.execution.max_spill_file_size_bytes
+caused by
+Invalid or Unsupported Configuration: value must be greater than 0
+
-statement error DataFusion error: Invalid or Unsupported Configuration: value must be greater than 0
+statement error
SET datafusion.sql_parser.recursion_limit = 0
+----
+DataFusion error: Error setting config datafusion.sql_parser.recursion_limit
+caused by
+Invalid or Unsupported Configuration: value must be greater than 0
+
# Config reset
statement ok
diff --git a/datafusion/sqllogictest/test_files/spark/map/str_to_map.slt b/datafusion/sqllogictest/test_files/spark/map/str_to_map.slt
index f422b50dfa..c1307468d7 100644
--- a/datafusion/sqllogictest/test_files/spark/map/str_to_map.slt
+++ b/datafusion/sqllogictest/test_files/spark/map/str_to_map.slt
@@ -159,5 +159,9 @@ statement ok
set datafusion.spark.map_key_dedup_policy = 'EXCEPTION';
# Invalid policy values are rejected at SET time with a clear message.
-statement error DataFusion error: Invalid or Unsupported Configuration: Invalid MapKeyDedupPolicy: BOGUS\. Expected one of: EXCEPTION, LAST_WIN
+statement error
set datafusion.spark.map_key_dedup_policy = 'BOGUS';
+----
+DataFusion error: Error setting config datafusion.spark.map_key_dedup_policy
+caused by
+Invalid or Unsupported Configuration: Invalid MapKeyDedupPolicy: BOGUS. Expected one of: EXCEPTION, LAST_WIN