Skip to content

Apache DataFusion SQL Query Engine

Support config matrix for SLT tests (#24493)

Oleks V791b32eedd33
diff --git a/datafusion/sqllogictest/README.md b/datafusion/sqllogictest/README.md
index f0a54cf978..76c28b5ecd 100644
--- a/datafusion/sqllogictest/README.md
+++ b/datafusion/sqllogictest/README.md
@@ -184,6 +184,56 @@ EXPLAIN ANALYZE SELECT * FROM generate_series(100);
Plan with Metrics LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=0, end=100, batch_size=8192], metrics=[output_rows=101, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>]
```
+## Cookbook: Sweeping config with `configMatrix`
+
+Runs the same `.slt` once per combination of config values. Each directive is a comment:
+
+```text
+# configMatrix: <key>=<v1>,<v2>[,...]
+```
+
+- Repeat the directive to nest keys. Values are the cartesian product.
+- Whitespace-trimmed and deduped; repeated keys merge value lists.
+- Unknown key or invalid value fails fast, naming the file, key, and value.
+- Each override is applied like `SET <key> = <value>`, so `datafusion.runtime.*`
+ keys (e.g. `datafusion.runtime.memory_limit`) and config-dependent UDFs behave
+ exactly as they would for an in-file `SET`.
+- Supported by the default runner and by `--substrait-round-trip`. `--complete`
+ rejects a file that declares directives, since it would overwrite the file
+ with the output of a single combination. The Postgres runner ignores them.
+- Every combination runs even if an earlier one fails. Each failure is prefixed
+ with the combination that produced it, and a file with more than one failing
+ combination reports all of them.
+
+Nested example (2 × 2 = 4 runs):
+
+```text
+# configMatrix: datafusion.execution.parquet.coerce_int96=ms,us
+# configMatrix: datafusion.execution.parquet.coerce_int96_tz=UTC,America/New_York
+
+statement ok
+CREATE EXTERNAL TABLE int96_from_spark
+STORED AS PARQUET
+LOCATION '../../parquet-testing/data/int96_from_spark.parquet';
+
+query I
+select count(*) from int96_from_spark
+----
+6
+```
+
+Failure output:
+
+```text
+[configMatrix: datafusion.execution.parquet.coerce_int96=ms, datafusion.execution.parquet.coerce_int96_tz=UTC]
+caused by
+External error: 1 errors in file .../parquet_int96_matrix.slt
+```
+
+When more than one combination fails, all of them are reported together under a
+`N configMatrix combinations failed:` header, each still prefixed with its own
+combination.
+
# Reference
## Running tests: Validation Mode
diff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rs
index 81b0c41d6a..2ae1ae149f 100644
--- a/datafusion/sqllogictest/bin/sqllogictests.rs
+++ b/datafusion/sqllogictest/bin/sqllogictests.rs
@@ -23,9 +23,9 @@ use datafusion::common::{DataFusionError, Result, exec_datafusion_err, exec_err}
use datafusion_sqllogictest::DataFusionSubstraitRoundTrip;
use datafusion_sqllogictest::TestFile;
use datafusion_sqllogictest::{
- CurrentlyExecutingSqlTracker, DataFusion, Filter, TestContext, df_value_validator,
- read_dir_recursive, setup_scratch_dir, should_skip_file, should_skip_record,
- value_normalizer,
+ CurrentlyExecutingSqlTracker, DFColumnType, DataFusion, Filter, TestContext,
+ df_value_validator, read_dir_recursive, run_each_configuration, setup_scratch_dir,
+ should_skip_file, should_skip_record, test_configurations, value_normalizer,
};
use futures::stream::StreamExt;
use indicatif::{
@@ -429,6 +429,27 @@ fn is_env_truthy(name: &str) -> bool {
})
}
+/// Selects which engine a matrix run drives each combination with. Also names
+/// the label used for record-condition matching and for counting records.
+#[derive(Clone, Copy)]
+enum Engine {
+ DataFusion,
+ #[cfg(feature = "substrait")]
+ SubstraitRoundTrip,
+}
+
+impl Engine {
+ /// Label used both to match record conditions (`add_label`) and to count the
+ /// records the progress bar expects (`count_records`).
+ fn label(self) -> &'static str {
+ match self {
+ Engine::DataFusion => "Datafusion",
+ #[cfg(feature = "substrait")]
+ Engine::SubstraitRoundTrip => "DatafusionSubstraitRoundTrip",
+ }
+ }
+}
+
#[cfg(feature = "substrait")]
async fn run_test_file_substrait_round_trip(
test_file: TestFile,
@@ -439,37 +460,17 @@ async fn run_test_file_substrait_round_trip(
currently_executing_sql_tracker: CurrentlyExecutingSqlTracker,
colored_output: bool,
) -> Result<()> {
- let TestFile {
- path,
- relative_path,
- } = test_file;
- let Some(test_ctx) = TestContext::try_new_for_test_file(&relative_path).await else {
- info!("Skipping: {}", path.display());
- return Ok(());
- };
- setup_scratch_dir(&relative_path)?;
-
- let count: u64 = get_record_count(&path, "DatafusionSubstraitRoundTrip".to_string());
- let pb = mp.add(ProgressBar::new(count));
-
- pb.set_style(mp_style);
- pb.set_message(relative_path.display().to_string());
-
- let mut runner = sqllogictest::Runner::new(|| async {
- Ok(DataFusionSubstraitRoundTrip::new(
- test_ctx.session_ctx().clone(),
- relative_path.clone(),
- pb.clone(),
- )
- .with_currently_executing_sql_tracker(currently_executing_sql_tracker.clone()))
- });
- runner.add_label("DatafusionSubstraitRoundTrip");
- runner.with_column_validator(strict_column_validator);
- runner.with_normalizer(value_normalizer);
- runner.with_validator(validator);
- let res = run_file_in_runner(path, &mut runner, filters, colored_output).await;
- pb.finish_and_clear();
- res
+ run_matrix(
+ Engine::SubstraitRoundTrip,
+ test_file,
+ validator,
+ mp,
+ mp_style,
+ filters,
+ currently_executing_sql_tracker,
+ colored_output,
+ )
+ .await
}
#[cfg(not(feature = "substrait"))]
@@ -497,69 +498,233 @@ async fn run_test_file(
filters: &[Filter],
currently_executing_sql_tracker: CurrentlyExecutingSqlTracker,
colored_output: bool,
+) -> Result<()> {
+ run_matrix(
+ Engine::DataFusion,
+ test_file,
+ validator,
+ mp,
+ mp_style,
+ filters,
+ currently_executing_sql_tracker,
+ colored_output,
+ )
+ .await
+}
+
+/// Runs `test_file` once per `# configMatrix:` combination with `engine`. The
+/// default and Substrait runners both delegate here, so neither bypasses the
+/// matrix loop. The Postgres runner does not participate: it runs a file once,
+/// ignoring any `# configMatrix:` directives.
+#[expect(
+ clippy::too_many_arguments,
+ reason = "mirrors the sibling run_* runners plus the engine selector"
+)]
+async fn run_matrix(
+ engine: Engine,
+ test_file: TestFile,
+ validator: Validator,
+ mp: MultiProgress,
+ mp_style: ProgressStyle,
+ filters: &[Filter],
+ currently_executing_sql_tracker: CurrentlyExecutingSqlTracker,
+ colored_output: bool,
) -> Result<()> {
let TestFile {
path,
relative_path,
} = test_file;
- let Some(test_ctx) = TestContext::try_new_for_test_file(&relative_path).await else {
- info!("Skipping: {}", path.display());
- return Ok(());
+
+ // Parsed once and replayed for every configuration.
+ let records = parse_records(&path)?;
+ let count = count_records(&records, engine.label());
+ let configurations = test_configurations(&path)?;
+
+ // Borrow the parse-once state so every combination reuses it. Each field is
+ // `Copy` or a shared reference, so the closure below stays callable per
+ // combination and its future stays `Send` for the spawned task.
+ let matrix = MatrixRunner {
+ engine,
+ path: &path,
+ relative_path: &relative_path,
+ records: &records,
+ filters,
+ currently_executing_sql_tracker: &currently_executing_sql_tracker,
+ validator,
+ colored_output,
+ mp: &mp,
+ mp_style: &mp_style,
+ count,
};
- setup_scratch_dir(&relative_path)?;
+ let matrix = &matrix;
- let count: u64 = get_record_count(&path, "Datafusion".to_string());
- let pb = mp.add(ProgressBar::new(count));
+ // The single dispatch through the configMatrix loop.
+ run_each_configuration(configurations, |configuration| async move {
+ matrix.run_combination(configuration.settings()).await
+ })
+ .await
+}
- pb.set_style(mp_style);
- pb.set_message(relative_path.display().to_string());
+/// Everything a matrix run needs to replay one file's records for a single
+/// combination. Holds the parse-once state by reference so it is shared across
+/// combinations rather than re-parsed.
+struct MatrixRunner<'a> {
+ engine: Engine,
+ path: &'a Path,
+ relative_path: &'a Path,
+ records: &'a [Record<DFColumnType>],
+ filters: &'a [Filter],
+ currently_executing_sql_tracker: &'a CurrentlyExecutingSqlTracker,
+ validator: Validator,
+ colored_output: bool,
+ mp: &'a MultiProgress,
+ mp_style: &'a ProgressStyle,
+ count: u64,
+}
- // If DataFusion configuration has changed during test file runs, errors will be
- // pushed to this vec.
- // HACK: managed externally because `sqllogictest` is an external dependency, and
- // it doesn't have an API to directly access the inner runner.
- let config_change_errors = Arc::new(Mutex::new(Vec::new()));
- let mut runner = sqllogictest::Runner::new(|| async {
- Ok(DataFusion::new(
- test_ctx.session_ctx().clone(),
- relative_path.clone(),
- pb.clone(),
- )
- .with_currently_executing_sql_tracker(currently_executing_sql_tracker.clone())
- .with_config_change_errors(Arc::clone(&config_change_errors)))
- });
- runner.add_label("Datafusion");
- runner.with_column_validator(strict_column_validator);
- runner.with_normalizer(value_normalizer);
- runner.with_validator(validator);
- let result = run_file_in_runner(path, &mut runner, filters, colored_output).await;
- pb.finish_and_clear();
+impl MatrixRunner<'_> {
+ /// Set up the context for one combination, apply its overrides, dispatch to
+ /// the engine, and finalize. `settings` are this combination's overrides.
+ async fn run_combination(&self, settings: &[(String, String)]) -> Result<()> {
+ let Some((test_ctx, pb)) = self.setup_combination(settings).await? else {
+ info!("Skipping: {}", self.path.display());
+ return Ok(());
+ };
- result?;
+ match self.engine {
+ Engine::DataFusion => self.run_datafusion(&test_ctx, pb).await,
+ #[cfg(feature = "substrait")]
+ Engine::SubstraitRoundTrip => {
+ self.run_substrait_round_trip(&test_ctx, pb).await
+ }
+ }
+ }
- // If there was no correctness error, check that the config is unchanged.
- runner.shutdown_async().await;
- config_change_result(&config_change_errors)
+ /// Per-combination setup: build the test context, apply this combination's
+ /// configMatrix overrides, and create the progress bar. Returns `Ok(None)`
+ /// when the file should be skipped (unsupported feature); the caller logs it.
+ async fn setup_combination(
+ &self,
+ settings: &[(String, String)],
+ ) -> Result<Option<(TestContext, ProgressBar)>> {
+ let Some(test_ctx) = TestContext::try_new_for_test_file(self.relative_path).await
+ else {
+ return Ok(None);
+ };
+ setup_scratch_dir(self.relative_path)?;
+ // Before the engine is built: it snapshots config to detect drift.
+ test_ctx
+ .apply_config_overrides(settings, self.relative_path)
+ .await?;
+
+ let pb = self.mp.add(ProgressBar::new(self.count));
+ pb.set_style(self.mp_style.clone());
+ pb.set_message(self.relative_path.display().to_string());
+ Ok(Some((test_ctx, pb)))
+ }
+
+ /// Default engine. Also collects any config the file left modified and shuts
+ /// the runner's connections down afterward.
+ async fn run_datafusion(
+ &self,
+ test_ctx: &TestContext,
+ pb: ProgressBar,
+ ) -> Result<()> {
+ // If DataFusion configuration has changed during test file runs, errors
+ // will be pushed to this vec.
+ // HACK: managed externally because `sqllogictest` is an external
+ // dependency, and it doesn't have an API to directly access the inner
+ // runner.
+ let config_change_errors = Arc::new(Mutex::new(Vec::new()));
+ let mut runner = sqllogictest::Runner::new(|| async {
+ Ok(DataFusion::new(
+ test_ctx.session_ctx().clone(),
+ self.relative_path.to_path_buf(),
+ pb.clone(),
+ )
+ .with_currently_executing_sql_tracker(
+ self.currently_executing_sql_tracker.clone(),
+ )
+ .with_config_change_errors(Arc::clone(&config_change_errors)))
+ });
+ let result = self.drive_runner(&mut runner, &pb).await;
+
+ runner.shutdown_async().await;
+
+ // A correctness failure takes precedence; otherwise surface any config
+ // the file left modified.
+ result.and_then(|()| config_change_result(&config_change_errors))
+ }
+
+ /// Substrait round-trip engine: each query's logical plan is converted to
+ /// Substrait and back before execution.
+ #[cfg(feature = "substrait")]
+ async fn run_substrait_round_trip(
+ &self,
+ test_ctx: &TestContext,
+ pb: ProgressBar,
+ ) -> Result<()> {
+ let mut runner = sqllogictest::Runner::new(|| async {
+ Ok(DataFusionSubstraitRoundTrip::new(
+ test_ctx.session_ctx().clone(),
+ self.relative_path.to_path_buf(),
+ pb.clone(),
+ )
+ .with_currently_executing_sql_tracker(
+ self.currently_executing_sql_tracker.clone(),
+ ))
+ });
+ self.drive_runner(&mut runner, &pb).await
+ }
+
+ /// Label the runner, install the shared validators, replay the records, and
+ /// clear the progress bar. Engine-independent, so every engine shares it.
+ async fn drive_runner<D, M>(
+ &self,
+ runner: &mut sqllogictest::Runner<D, M>,
+ pb: &ProgressBar,
+ ) -> Result<()>
+ where
+ D: AsyncDB<ColumnType = DFColumnType>,
+ M: MakeConnection<Conn = D>,
+ {
+ runner.add_label(self.engine.label());
+ runner.with_column_validator(strict_column_validator);
+ runner.with_normalizer(value_normalizer);
+ runner.with_validator(self.validator);
+ let result = run_file_in_runner(
+ self.path,
+ self.records,
+ runner,
+ self.filters,
+ self.colored_output,
+ )
+ .await;
+ pb.finish_and_clear();
+ result
+ }
}
-async fn run_file_in_runner<D: AsyncDB, M: MakeConnection<Conn = D>>(
- path: PathBuf,
+async fn run_file_in_runner<D, M>(
+ path: &Path,
+ records: &[Record<DFColumnType>],
runner: &mut sqllogictest::Runner<D, M>,
filters: &[Filter],
colored_output: bool,
-) -> Result<()> {
- let path = path.canonicalize()?;
- let records =
- parse_file(&path).map_err(|e| DataFusionError::External(Box::new(e)))?;
+) -> Result<()>
+where
+ D: AsyncDB<ColumnType = DFColumnType>,
+ M: MakeConnection<Conn = D>,
+{
let mut errs = vec![];
- for record in records.into_iter() {
+ for record in records {
if let Record::Halt { .. } = record {
break;
}
- if should_skip_record::<D>(&record, filters) {
+ if should_skip_record::<D>(record, filters) {
continue;
}
- if let Err(err) = runner.run_async(record).await {
+ if let Err(err) = runner.run_async(record.clone()).await {
if colored_output {
errs.push(format!("{}", err.display(true)));
} else {
@@ -569,6 +734,7 @@ async fn run_file_in_runner<D: AsyncDB, M: MakeConnection<Conn = D>>(
}
if !errs.is_empty() {
+ let path = path.canonicalize()?;
let mut msg = format!("{} errors in file {}\n\n", errs.len(), path.display());
for (i, err) in errs.iter().enumerate() {
if i >= ERRS_PER_FILE_LIMIT {
@@ -587,30 +753,28 @@ async fn run_file_in_runner<D: AsyncDB, M: MakeConnection<Conn = D>>(
Ok(())
}
-#[expect(clippy::needless_pass_by_value)]
-fn get_record_count(path: &PathBuf, label: String) -> u64 {
- let records: Vec<Record<<DataFusion as AsyncDB>::ColumnType>> =
- parse_file(path).unwrap();
- let mut count: u64 = 0;
-
- for rec in &records {
- match rec {
- Record::Query { conditions, .. } | Record::Statement { conditions, .. }
- if conditions.is_empty()
- || !conditions.contains(&Condition::SkipIf {
- label: label.clone(),
- })
- || conditions.contains(&Condition::OnlyIf {
- label: label.clone(),
- }) =>
- {
- count += 1;
- }
- _ => {}
- }
- }
+fn parse_records(path: &Path) -> Result<Vec<Record<DFColumnType>>> {
+ parse_file(path).map_err(|e| DataFusionError::External(Box::new(e)))
+}
- count
+fn count_records(records: &[Record<DFColumnType>], label: &str) -> u64 {
+ let skip_if = Condition::SkipIf {
+ label: label.to_string(),
+ };
+ let only_if = Condition::OnlyIf {
+ label: label.to_string(),
+ };
+ records
+ .iter()
+ .filter(|rec| match rec {
+ Record::Query { conditions, .. } | Record::Statement { conditions, .. } => {
+ conditions.is_empty()
+ || !conditions.contains(&skip_if)
+ || conditions.contains(&only_if)
+ }
+ _ => false,
+ })
+ .count() as u64
}
#[cfg(feature = "postgres")]
@@ -629,7 +793,8 @@ async fn run_test_file_with_postgres(
} = test_file;
setup_scratch_dir(&relative_path)?;
- let count: u64 = get_record_count(&path, "postgresql".to_string());
+ let records = parse_records(&path)?;
+ let count = count_records(&records, "postgresql");
let pb = mp.add(ProgressBar::new(count));
pb.set_style(mp_style);
@@ -646,7 +811,7 @@ async fn run_test_file_with_postgres(
runner.with_column_validator(strict_column_validator);
runner.with_normalizer(value_normalizer);
runner.with_validator(validator);
- let result = run_file_in_runner(path, &mut runner, filters, false).await;
+ let result = run_file_in_runner(&path, &records, &mut runner, filters, false).await;
pb.finish_and_clear();
result
}
@@ -682,13 +847,27 @@ async fn run_complete_file(
info!("Using complete mode to complete: {}", path.display());
+ // `update_test_file` rewrites the file from a single run, so it cannot hold
+ // one expected-output set per configuration.
+ let declares_config_matrix =
+ test_configurations(&path)?.iter().any(|c| !c.is_empty());
+ if declares_config_matrix {
+ return exec_err!(
+ "Cannot use --complete on {}: it declares `# configMatrix:` \
+ directives, and completion would overwrite the file with the \
+ output of a single configuration",
+ relative_path.display()
+ );
+ }
+
let Some(test_ctx) = TestContext::try_new_for_test_file(&relative_path).await else {
info!("Skipping: {}", path.display());
return Ok(());
};
setup_scratch_dir(&relative_path)?;
- let count: u64 = get_record_count(&path, "Datafusion".to_string());
+ // `update_test_file` re-parses the file itself, so only the count is needed.
+ let count = count_records(&parse_records(&path)?, "Datafusion");
let pb = mp.add(ProgressBar::new(count));
pb.set_style(mp_style);
@@ -744,7 +923,7 @@ async fn run_complete_file_with_postgres(
);
setup_scratch_dir(&relative_path)?;
- let count: u64 = get_record_count(&path, "postgresql".to_string());
+ let count = count_records(&parse_records(&path)?, "postgresql");
let pb = mp.add(ProgressBar::new(count));
pb.set_style(mp_style);
diff --git a/datafusion/sqllogictest/src/config_matrix.rs b/datafusion/sqllogictest/src/config_matrix.rs
new file mode 100644
index 0000000000..4e5881c7ae
--- /dev/null
+++ b/datafusion/sqllogictest/src/config_matrix.rs
@@ -0,0 +1,549 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! `# configMatrix:` directives sweep DataFusion config across repeated runs of
+//! one SLT file:
+//!
+//! ```text
+//! # configMatrix: <key>=<v1>,<v2>[,...]
+//! ```
+//!
+//! This module validates directives, dedups their values, and expands them into
+//! the [`TestConfiguration`]s the runner applies to a session. Repeated
+//! directives combine as a cartesian product.
+
+use std::fmt;
+use std::fs;
+use std::future::Future;
+use std::path::Path;
+
+use datafusion::common::{DataFusionError, Result, exec_datafusion_err};
+use itertools::Itertools;
+
+/// Compared case-insensitively; this spelling is the documented one.
+const DIRECTIVE_MARKER: &str = "configMatrix:";
+
+/// Config values to apply before a single run of an SLT file.
+///
+/// Empty when the file declared no directives, which means "run once,
+/// unmodified".
+#[derive(Debug, Clone)]
+pub struct TestConfiguration(Vec<(String, String)>);
+
+impl TestConfiguration {
+ /// The `key = value` pairs to set on the session.
+ pub fn settings(&self) -> &[(String, String)] {
+ &self.0
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.0.is_empty()
+ }
+
+ /// Prefix a failure with the configuration that produced it, so a sweep
+ /// names the values that broke. No-op when no directives were declared.
+ fn attribute_failure<T>(&self, result: Result<T>) -> Result<T> {
+ if self.is_empty() {
+ return result;
+ }
+ result.map_err(|e| e.context(self.to_string()))
+ }
+}
+
+impl fmt::Display for TestConfiguration {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ write!(
+ f,
+ "[configMatrix: {}]",
+ self.0.iter().map(|(k, v)| format!("{k}={v}")).join(", ")
+ )
+ }
+}
+
+/// Validate, dedup, and expand the directives declared by `path`.
+///
+/// Never empty: a file without directives yields one empty
+/// [`TestConfiguration`], so callers keep a single loop shape.
+pub fn test_configurations(path: &Path) -> Result<Vec<TestConfiguration>> {
+ let content = fs::read_to_string(path).map_err(|e| {
+ exec_datafusion_err!(
+ "Failed to read {} for configMatrix parsing: {e}",
+ path.display()
+ )
+ })?;
+ parse_configurations(&content, path)
+}
+
+/// Run `run_one` once per configuration, continuing past failures so one broken
+/// combination never hides the others. Each failure is prefixed with the
+/// configuration that produced it (see `TestConfiguration::attribute_failure`).
+///
+/// Returns `Ok(())` if all passed, the lone failure verbatim if exactly one
+/// failed (so a matrix-free file keeps its original error), or an aggregate
+/// naming every failing combination.
+///
+/// `run_one` takes the configuration by value so its future holds no
+/// higher-ranked borrow and stays `Send` for the runner's spawned task.
+pub async fn run_each_configuration<F, Fut>(
+ configurations: Vec<TestConfiguration>,
+ mut run_one: F,
+) -> Result<()>
+where
+ F: FnMut(TestConfiguration) -> Fut,
+ Fut: Future<Output = Result<()>>,
+{
+ let mut failures = Vec::new();
+ for configuration in configurations {
+ // Keep a copy to attribute the failure after `run_one` consumes it.
+ let label = configuration.clone();
+ if let Err(err) = label.attribute_failure(run_one(configuration).await) {
+ failures.push(err);
+ }
+ }
+ combine_configuration_failures(failures)
+}
+
+/// Collapse per-configuration failures into a single [`Result`], returned by
+/// [`run_each_configuration`].
+fn combine_configuration_failures(mut failures: Vec<DataFusionError>) -> Result<()> {
+ match failures.len() {
+ 0 => Ok(()),
+ 1 => Err(failures.pop().unwrap()),
+ n => {
+ let combined = failures.iter().join("\n\n");
+ Err(DataFusionError::External(
+ format!("{n} configMatrix combinations failed:\n\n{combined}").into(),
+ ))
+ }
+ }
+}
+
+/// `path` is used only for error messages.
+fn parse_configurations(content: &str, path: &Path) -> Result<Vec<TestConfiguration>> {
+ // Dimensions borrow `content`; values are owned only in the product below.
+ let mut dimensions: Vec<(&str, Vec<&str>)> = Vec::new();
+
+ for (idx, line) in content.lines().enumerate() {
+ let Some(directive) = strip_directive_prefix(line) else {
+ continue;
+ };
+ let invalid = |detail: String| {
+ DataFusionError::Configuration(format!(
+ "Invalid configMatrix directive in {}:{}: {detail}",
+ path.display(),
+ idx + 1
+ ))
+ };
+
+ let (key, values) = directive.split_once('=').ok_or_else(|| {
+ invalid(format!(
+ "expected `# configMatrix: <key>=<v1>,<v2>[,...]`, got `{directive}`"
+ ))
+ })?;
+
+ let key = key.trim();
+ if key.is_empty() {
+ return Err(invalid("missing config key".to_string()));
+ }
+
+ let values: Vec<&str> = values
+ .split(',')
+ .map(str::trim)
+ .filter(|value| !value.is_empty())
+ .collect();
+ if values.is_empty() {
+ return Err(invalid(format!("no values provided for `{key}`")));
+ }
+
+ // A repeated key unions its values instead of adding a dimension.
+ match dimensions.iter_mut().find(|(seen, _)| *seen == key) {
+ Some((_, merged)) => merged.extend(values),
+ None => dimensions.push((key, values)),
+ }
+ }
+
+ Ok(dimensions
+ .into_iter()
+ .map(|(key, values)| {
+ // Dedup keeps first-seen order.
+ values
+ .into_iter()
+ .unique()
+ .map(|value| (key.to_string(), value.to_string()))
+ .collect_vec()
+ })
+ .multi_cartesian_product()
+ .map(TestConfiguration)
+ .collect())
+}
+
+/// Text after `configMatrix:` when `line` is a directive comment. Tolerates
+/// `##` banners and any marker casing.
+fn strip_directive_prefix(line: &str) -> Option<&str> {
+ let after_hash = line.trim_start().strip_prefix('#')?;
+ let after_hash = after_hash.trim_start_matches('#').trim_start();
+
+ let (marker, rest) = after_hash.split_at_checked(DIRECTIVE_MARKER.len())?;
+ if !marker.eq_ignore_ascii_case(DIRECTIVE_MARKER) {
+ return None;
+ }
+ Some(rest.trim())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use datafusion::common::exec_err;
+ use std::io::Write;
+
+ /// Settings of every configuration parsed from `content`.
+ fn parse(content: &str) -> Vec<Vec<(String, String)>> {
+ parse_configurations(content, Path::new("test.slt"))
+ .unwrap()
+ .into_iter()
+ .map(|test_configuration| test_configuration.0)
+ .collect()
+ }
+
+ fn test_configuration(settings: &[(&str, &str)]) -> TestConfiguration {
+ TestConfiguration(
+ settings
+ .iter()
+ .map(|(key, value)| (key.to_string(), value.to_string()))
+ .collect(),
+ )
+ }
+
+ #[test]
+ fn parses_no_directives() {
+ // Zero dimensions expand to exactly one empty configuration.
+ let content = "# ordinary comment\nquery I\nSELECT 1\n----\n1\n";
+ assert_eq!(parse(content), vec![Vec::new()]);
+ }
+
+ #[test]
+ fn parses_single_directive() {
+ let configurations = parse(
+ "# configMatrix: datafusion.optimizer.enable_piecewise_merge_join=true,false\n",
+ );
+ assert_eq!(configurations.len(), 2);
+ assert_eq!(
+ configurations[0],
+ vec![(
+ "datafusion.optimizer.enable_piecewise_merge_join".to_string(),
+ "true".to_string(),
+ )]
+ );
+ assert_eq!(
+ configurations[1],
+ vec![(
+ "datafusion.optimizer.enable_piecewise_merge_join".to_string(),
+ "false".to_string(),
+ )]
+ );
+ }
+
+ #[test]
+ fn parses_case_insensitive_marker() {
+ let configurations =
+ parse("## CONFIGMATRIX: datafusion.execution.batch_size=1024,4096\n");
+ assert_eq!(configurations.len(), 2);
+ assert_eq!(configurations[0][0].0, "datafusion.execution.batch_size");
+ }
+
+ #[test]
+ fn parses_multiple_directives_and_computes_cartesian_product() {
+ let configurations = parse("# configMatrix: a=1,2\n# configMatrix: b=x,y,z\n");
+ assert_eq!(configurations.len(), 6);
+ assert_eq!(
+ configurations[0],
+ vec![
+ ("a".to_string(), "1".to_string()),
+ ("b".to_string(), "x".to_string()),
+ ]
+ );
+ assert_eq!(
+ configurations[5],
+ vec![
+ ("a".to_string(), "2".to_string()),
+ ("b".to_string(), "z".to_string()),
+ ]
+ );
+ }
+
+ #[test]
+ fn rejects_missing_equals() {
+ let err = parse_configurations(
+ "# configMatrix: bogus_no_equals\n",
+ Path::new("test.slt"),
+ )
+ .unwrap_err();
+ assert!(err.to_string().contains("<key>=<v1>"));
+ }
+
+ #[test]
+ fn rejects_missing_key() {
+ let err =
+ parse_configurations("# configMatrix: =true,false\n", Path::new("test.slt"))
+ .unwrap_err();
+ assert!(err.to_string().contains("missing config key"));
+ }
+
+ #[test]
+ fn rejects_empty_values() {
+ let err =
+ parse_configurations("# configMatrix: some.key=\n", Path::new("test.slt"))
+ .unwrap_err();
+ assert!(err.to_string().contains("no values provided"));
+ }
+
+ #[test]
+ fn rejects_empty_values_on_a_repeated_key() {
+ let err = parse_configurations(
+ "# configMatrix: k=1,2\n# configMatrix: k=\n",
+ Path::new("test.slt"),
+ )
+ .unwrap_err();
+ assert!(err.to_string().contains("no values provided"));
+ }
+
+ #[test]
+ fn deduplicates_repeated_values_preserving_first_order() {
+ let configurations =
+ parse("# configMatrix: some.key=false,true,false,true,false\n");
+ assert_eq!(configurations.len(), 2);
+ assert_eq!(configurations[0][0].1, "false");
+ assert_eq!(configurations[1][0].1, "true");
+ }
+
+ #[test]
+ fn strips_whitespace_around_values() {
+ let configurations = parse("# configMatrix: k= 1024 ,\t2048 , \t4096\n");
+ let values: Vec<&str> = configurations.iter().map(|c| c[0].1.as_str()).collect();
+ assert_eq!(values, vec!["1024", "2048", "4096"]);
+ }
+
+ #[test]
+ fn strips_whitespace_around_key_and_around_equals() {
+ let configurations = parse(
+ "# configMatrix: datafusion.execution.batch_size = 1024 , 2048 \n",
+ );
+ assert_eq!(configurations.len(), 2);
+ assert_eq!(configurations[0][0].0, "datafusion.execution.batch_size");
+ }
+
+ #[test]
+ fn dedup_runs_after_whitespace_is_stripped() {
+ // `1024`, ` 1024`, and `\t1024 ` all normalize to the same value.
+ let configurations = parse("# configMatrix: k= 1024 , 1024,\t1024 \n");
+ assert_eq!(configurations.len(), 1);
+ assert_eq!(configurations[0][0].1, "1024");
+ }
+
+ #[test]
+ fn trailing_and_repeated_commas_are_ignored() {
+ let configurations = parse("# configMatrix: k=1024,,2048,\n");
+ let values: Vec<&str> = configurations.iter().map(|c| c[0].1.as_str()).collect();
+ assert_eq!(values, vec!["1024", "2048"]);
+ }
+
+ #[test]
+ fn merges_repeated_directives_for_same_key() {
+ let configurations = parse(
+ "# configMatrix: datafusion.execution.batch_size=1024,2048\n\
+ # configMatrix: datafusion.execution.batch_size=1024,2048\n",
+ );
+ assert_eq!(configurations.len(), 2);
+ }
+
+ #[test]
+ fn merges_repeated_directives_unioning_values() {
+ let configurations =
+ parse("# configMatrix: k=1024,2048\n# configMatrix: k=2048,4096\n");
+ let values: Vec<&str> = configurations.iter().map(|c| c[0].1.as_str()).collect();
+ assert_eq!(values, vec!["1024", "2048", "4096"]);
+ }
+
+ #[test]
+ fn nested_matrices_expand_as_cartesian_product() {
+ let configurations = parse(
+ "# configMatrix: datafusion.execution.batch_size=1024,2048\n\
+ # configMatrix: datafusion.execution.param1=true,false\n",
+ );
+ assert_eq!(configurations.len(), 4);
+ for configuration in &configurations {
+ assert_eq!(configuration.len(), 2);
+ assert_eq!(configuration[0].0, "datafusion.execution.batch_size");
+ assert_eq!(configuration[1].0, "datafusion.execution.param1");
+ }
+ assert_eq!(configurations[0][0].1, "1024");
+ assert_eq!(configurations[0][1].1, "true");
+ assert_eq!(configurations[3][0].1, "2048");
+ assert_eq!(configurations[3][1].1, "false");
+ }
+
+ #[test]
+ fn ignores_non_comment_lines_with_marker_text() {
+ assert_eq!(parse("SELECT 'configMatrix: foo=bar';\n"), vec![Vec::new()]);
+ }
+
+ #[test]
+ fn displays_key_value_pairs() {
+ assert_eq!(
+ test_configuration(&[("a", "1"), ("b", "x")]).to_string(),
+ "[configMatrix: a=1, b=x]"
+ );
+ }
+
+ #[test]
+ fn test_configurations_yields_one_empty_entry_when_no_matrix() {
+ let mut file = tempfile::NamedTempFile::new().unwrap();
+ file.write_all(b"# just a comment\nquery I\nSELECT 1\n----\n1\n")
+ .unwrap();
+ let configurations = test_configurations(file.path()).unwrap();
+ assert_eq!(configurations.len(), 1);
+ assert!(configurations[0].is_empty());
+ }
+
+ #[test]
+ fn test_configurations_yields_every_combination() {
+ let mut file = tempfile::NamedTempFile::new().unwrap();
+ file.write_all(
+ b"# configMatrix: a=1,2\n\
+ # configMatrix: b=x,y\n\
+ query I\nSELECT 1\n----\n1\n",
+ )
+ .unwrap();
+ let configurations = test_configurations(file.path()).unwrap();
+ assert_eq!(configurations.len(), 4);
+ assert!(configurations.iter().all(|c| !c.is_empty()));
+ }
+
+ #[test]
+ fn attribute_failure_is_noop_without_directives() {
+ let err: Result<()> = Err(exec_datafusion_err!("boom"));
+ assert_eq!(
+ test_configuration(&[])
+ .attribute_failure(err)
+ .unwrap_err()
+ .to_string(),
+ exec_datafusion_err!("boom").to_string()
+ );
+ }
+
+ #[test]
+ fn attribute_failure_names_the_configuration() {
+ let err: Result<()> = Err(exec_datafusion_err!("boom"));
+ let msg = test_configuration(&[("k", "1")])
+ .attribute_failure(err)
+ .unwrap_err()
+ .to_string();
+ assert!(msg.starts_with("[configMatrix: k=1]"), "got {msg}");
+ assert!(msg.contains("boom"), "got {msg}");
+ }
+
+ #[tokio::test]
+ async fn run_each_configuration_runs_every_combination_past_failures() {
+ use std::sync::{Arc, Mutex};
+
+ // Drives the exact helper both runner paths use: every configuration
+ // must run and be attributed even though an earlier one fails.
+ let configurations = vec![
+ test_configuration(&[("k", "1")]),
+ test_configuration(&[("k", "2")]),
+ test_configuration(&[("k", "3")]),
+ ];
+ let seen = Arc::new(Mutex::new(Vec::new()));
+ let recorder = Arc::clone(&seen);
+
+ let result = run_each_configuration(configurations, |configuration| {
+ let recorder = Arc::clone(&recorder);
+ async move {
+ let value = configuration.settings()[0].1.clone();
+ recorder.lock().unwrap().push(value.clone());
+ // The middle combination fails; the last must still run.
+ if value == "2" {
+ exec_err!("boom")
+ } else {
+ Ok(())
+ }
+ }
+ })
+ .await;
+
+ // All three ran even though the second failed.
+ assert_eq!(*seen.lock().unwrap(), vec!["1", "2", "3"]);
+ let msg = result.unwrap_err().to_string();
+ assert!(msg.contains("[configMatrix: k=2]"), "got {msg}");
+ assert!(msg.contains("boom"), "got {msg}");
+ }
+
+ #[tokio::test]
+ async fn run_each_configuration_is_ok_when_all_pass() {
+ let configurations = vec![
+ test_configuration(&[("k", "1")]),
+ test_configuration(&[("k", "2")]),
+ ];
+ let result = run_each_configuration(configurations, |_| async { Ok(()) }).await;
+ assert!(result.is_ok());
+ }
+
+ #[test]
+ fn combine_configuration_failures_is_ok_when_empty() {
+ assert!(combine_configuration_failures(vec![]).is_ok());
+ }
+
+ #[test]
+ fn combine_configuration_failures_returns_a_lone_failure_verbatim() {
+ // A file without a matrix expands to one empty configuration, so its
+ // failure must be reported exactly as before (no combination prefix,
+ // no aggregate header).
+ let boom: Result<()> = exec_err!("boom");
+ let failure = test_configuration(&[]).attribute_failure(boom).unwrap_err();
+ let msg = combine_configuration_failures(vec![failure])
+ .unwrap_err()
+ .to_string();
+ assert_eq!(msg, exec_datafusion_err!("boom").to_string());
+ }
+
+ #[test]
+ fn combine_configuration_failures_aggregates_and_attributes_all_failures() {
+ let boom_1: Result<()> = exec_err!("boom-1");
+ let boom_2: Result<()> = exec_err!("boom-2");
+ let failures = vec![
+ test_configuration(&[("k", "1")])
+ .attribute_failure(boom_1)
+ .unwrap_err(),
+ test_configuration(&[("k", "2")])
+ .attribute_failure(boom_2)
+ .unwrap_err(),
+ ];
+
+ let msg = combine_configuration_failures(failures)
+ .unwrap_err()
+ .to_string();
+
+ assert!(
+ msg.contains("2 configMatrix combinations failed"),
+ "got {msg}"
+ );
+ assert!(msg.contains("[configMatrix: k=1]"), "got {msg}");
+ assert!(msg.contains("boom-1"), "got {msg}");
+ assert!(msg.contains("[configMatrix: k=2]"), "got {msg}");
+ assert!(msg.contains("boom-2"), "got {msg}");
+ }
+}
diff --git a/datafusion/sqllogictest/src/lib.rs b/datafusion/sqllogictest/src/lib.rs
index 6b6c40365f..926b1fb1e8 100644
--- a/datafusion/sqllogictest/src/lib.rs
+++ b/datafusion/sqllogictest/src/lib.rs
@@ -26,6 +26,7 @@
//! DataFusion sqllogictest driver
+mod config_matrix;
mod engines;
mod test_file;
@@ -46,6 +47,7 @@ mod filters;
mod test_context;
mod util;
+pub use config_matrix::{TestConfiguration, run_each_configuration, test_configurations};
pub use filters::*;
pub use test_context::TestContext;
pub use test_file::TestFile;
diff --git a/datafusion/sqllogictest/src/test_context.rs b/datafusion/sqllogictest/src/test_context.rs
index b7615301dc..be5830f4e7 100644
--- a/datafusion/sqllogictest/src/test_context.rs
+++ b/datafusion/sqllogictest/src/test_context.rs
@@ -242,6 +242,35 @@ impl TestContext {
pub fn session_ctx(&self) -> &SessionContext {
&self.ctx
}
+
+ /// Apply `key = value` config overrides to the `SessionContext` in place,
+ /// used by the runner to sweep `# configMatrix:` directives.
+ ///
+ /// Each override runs as `SET <key> = '<value>'`, the same path an in-file
+ /// `SET` takes, so it accepts any key `SET` does: `datafusion.runtime.*` keys
+ /// reach the runtime environment and config-dependent UDFs are refreshed.
+ /// Values are single-quoted (embedded quotes doubled) so strings like
+ /// timezones and `100M` parse as literals.
+ ///
+ /// `origin` labels error messages, typically the test file path.
+ pub async fn apply_config_overrides(
+ &self,
+ overrides: &[(String, String)],
+ origin: &Path,
+ ) -> Result<()> {
+ for (key, value) in overrides {
+ // Single-quote as a string literal, doubling embedded quotes.
+ let escaped = value.replace('\'', "''");
+ let sql = format!("SET {key} = '{escaped}'");
+ self.ctx.sql(&sql).await.map_err(|e| {
+ e.context(format!(
+ "configMatrix in {}: failed to set `{key}` = `{value}`",
+ origin.display()
+ ))
+ })?;
+ }
+ Ok(())
+ }
}
// ==============================================================================
@@ -808,3 +837,77 @@ fn register_conflicting_metadata_tables(ctx: &SessionContext) {
RecordBatch::try_new(Arc::new(schema_right), vec![Arc::new(data_right)]).unwrap();
ctx.register_batch("smaller_table", batch_right).unwrap();
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use datafusion::execution::memory_pool::MemoryConsumer;
+
+ /// Non-runtime keys land on `ConfigOptions`, same as a plain `SET`.
+ #[tokio::test]
+ async fn apply_config_overrides_sets_config_options() {
+ let test_ctx = TestContext::new(SessionContext::new());
+ test_ctx
+ .apply_config_overrides(
+ &[(
+ "datafusion.execution.batch_size".to_string(),
+ "1234".to_string(),
+ )],
+ Path::new("test.slt"),
+ )
+ .await
+ .unwrap();
+
+ assert_eq!(
+ test_ctx
+ .session_ctx()
+ .state()
+ .config()
+ .options()
+ .execution
+ .batch_size
+ .to_string(),
+ "1234"
+ );
+ }
+
+ /// `datafusion.runtime.*` keys reach the runtime environment, not just
+ /// `ConfigOptions` (a direct write would reject `runtime` keys).
+ #[tokio::test]
+ async fn apply_config_overrides_routes_runtime_keys() {
+ let test_ctx = TestContext::new(SessionContext::new());
+ test_ctx
+ .apply_config_overrides(
+ &[(
+ "datafusion.runtime.memory_limit".to_string(),
+ "100M".to_string(),
+ )],
+ Path::new("test.slt"),
+ )
+ .await
+ .unwrap();
+
+ // The override took effect: the pool now caps reservations at 100M.
+ let pool = Arc::clone(&test_ctx.session_ctx().runtime_env().memory_pool);
+ let reservation = MemoryConsumer::new("test").register(&pool);
+ assert!(reservation.try_grow(50 * 1024 * 1024).is_ok());
+ assert!(reservation.try_grow(100 * 1024 * 1024).is_err());
+ }
+
+ /// An unknown key still fails fast, naming the originating file.
+ #[tokio::test]
+ async fn apply_config_overrides_reports_invalid_key() {
+ let test_ctx = TestContext::new(SessionContext::new());
+ let err = test_ctx
+ .apply_config_overrides(
+ &[("datafusion.does.not.exist".to_string(), "1".to_string())],
+ Path::new("bad.slt"),
+ )
+ .await
+ .unwrap_err()
+ .to_string();
+
+ assert!(err.contains("bad.slt"), "got {err}");
+ assert!(err.contains("datafusion.does.not.exist"), "got {err}");
+ }
+}