Apache DataFusion SQL Query Engine
feat: feature-gate `sqllogictests` datafusion-substrait behind optional 'substrait' feature (#21268)
Qi Zhu2d5f016ddae3
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.ymlindex 0d12ddc375..ba2a206b5f 100644--- a/.github/workflows/rust.yml+++ b/.github/workflows/rust.yml@@ -305,7 +305,7 @@ jobs:--lib \--tests \--bins \- --features serde,avro,json,backtrace,integration-tests,parquet_encryption+ --features serde,avro,json,backtrace,integration-tests,parquet_encryption,substrait- name: Verify Working Directory Cleanrun: git diff --exit-code# Check no temporary directories created during test.@@ -473,7 +473,7 @@ jobs:export RUST_MIN_STACK=20971520export TPCH_DATA=`realpath datafusion/sqllogictest/test_files/tpch/data`cargo test plan_q --package datafusion-benchmarks --profile ci --features=ci -- --test-threads=1- INCLUDE_TPCH=true cargo test --features backtrace,parquet_encryption --profile ci --package datafusion-sqllogictest --test sqllogictests+ INCLUDE_TPCH=true cargo test --features backtrace,parquet_encryption,substrait --profile ci --package datafusion-sqllogictest --test sqllogictests- name: Verify Working Directory Cleanrun: git diff --exit-code@@ -537,7 +537,7 @@ jobs:# command cannot be run for all the .slt files. Run it for just one that works (limit.slt)# until most of the tickets in https://github.com/apache/datafusion/issues/16248 are addressed# and this command can be run without filters.- run: cargo test --test sqllogictests -- --substrait-round-trip limit.slt+ run: cargo test -p datafusion-sqllogictest --test sqllogictests --features substrait -- --substrait-round-trip limit.slt# Temporarily commenting out the Windows flow, the reason is enormously slow running build# Waiting for new Windows 2025 github runner@@ -570,7 +570,7 @@ jobs:uses: ./.github/actions/setup-macos-aarch64-builder- name: Run tests (excluding doctests)shell: bash- run: cargo test --profile ci --exclude datafusion-cli --workspace --lib --tests --bins --features avro,json,backtrace,integration-tests+ run: cargo test --profile ci --exclude datafusion-cli --workspace --lib --tests --bins --features avro,json,backtrace,integration-tests,substraitvendor:name: Verify Vendored Codediff --git a/datafusion/sqllogictest/Cargo.toml b/datafusion/sqllogictest/Cargo.tomlindex 7a341988d5..d7bb2583c9 100644--- a/datafusion/sqllogictest/Cargo.toml+++ b/datafusion/sqllogictest/Cargo.toml@@ -48,7 +48,7 @@ chrono = { workspace = true, optional = true }clap = { version = "4.5.60", features = ["derive", "env"] }datafusion = { workspace = true, default-features = true, features = ["avro"] }datafusion-spark = { workspace = true, features = ["core"] }-datafusion-substrait = { workspace = true, default-features = true }+datafusion-substrait = { workspace = true, default-features = true, optional = true }futures = { workspace = true }half = { workspace = true, default-features = true }indicatif = "0.18"@@ -79,6 +79,7 @@ postgres = [parquet_encryption = ["datafusion/parquet_encryption",]+substrait = ["datafusion-substrait"][dev-dependencies]env_logger = { workspace = true }diff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rsindex cb76cc30fd..8ca19025bd 100644--- a/datafusion/sqllogictest/bin/sqllogictests.rs+++ b/datafusion/sqllogictest/bin/sqllogictests.rs@@ -19,11 +19,13 @@ use clap::{ColorChoice, Parser};use datafusion::common::instant::Instant;use datafusion::common::utils::get_available_parallelism;use datafusion::common::{DataFusionError, Result, exec_datafusion_err, exec_err};+#[cfg(feature = "substrait")]+use datafusion_sqllogictest::DataFusionSubstraitRoundTrip;use datafusion_sqllogictest::TestFile;use datafusion_sqllogictest::{- CurrentlyExecutingSqlTracker, DataFusion, DataFusionSubstraitRoundTrip, Filter,- TestContext, df_value_validator, read_dir_recursive, setup_scratch_dir,- should_skip_file, should_skip_record, value_normalizer,+ CurrentlyExecutingSqlTracker, DataFusion, Filter, TestContext, df_value_validator,+ read_dir_recursive, setup_scratch_dir, should_skip_file, should_skip_record,+ value_normalizer,};use futures::stream::StreamExt;use indicatif::{@@ -426,6 +428,7 @@ fn is_env_truthy(name: &str) -> bool {})}+#[cfg(feature = "substrait")]async fn run_test_file_substrait_round_trip(test_file: TestFile,validator: Validator,@@ -468,6 +471,19 @@ async fn run_test_file_substrait_round_trip(res}+#[cfg(not(feature = "substrait"))]+async fn run_test_file_substrait_round_trip(+ _test_file: TestFile,+ _validator: Validator,+ _mp: MultiProgress,+ _mp_style: ProgressStyle,+ _filters: &[Filter],+ _currently_executing_sql_tracker: CurrentlyExecutingSqlTracker,+ _colored_output: bool,+) -> Result<()> {+ exec_err!("Cannot run substrait round-trip: the 'substrait' feature is not enabled")+}+async fn run_test_file(test_file: TestFile,validator: Validator,diff --git a/datafusion/sqllogictest/src/engines/mod.rs b/datafusion/sqllogictest/src/engines/mod.rsindex ee2987db07..0c9643b1c7 100644--- a/datafusion/sqllogictest/src/engines/mod.rs+++ b/datafusion/sqllogictest/src/engines/mod.rs@@ -19,6 +19,7 @@mod conversion;mod currently_executed_sql;mod datafusion_engine;+#[cfg(feature = "substrait")]mod datafusion_substrait_roundtrip_engine;mod output;@@ -26,6 +27,7 @@ pub use datafusion_engine::DFSqlLogicTestError;pub use datafusion_engine::DataFusion;pub use datafusion_engine::convert_batches;pub use datafusion_engine::convert_schema_to_types;+#[cfg(feature = "substrait")]pub use datafusion_substrait_roundtrip_engine::DataFusionSubstraitRoundTrip;pub use output::DFColumnType;pub use output::DFOutput;diff --git a/datafusion/sqllogictest/src/lib.rs b/datafusion/sqllogictest/src/lib.rsindex bb12c58bdc..6b6c40365f 100644--- a/datafusion/sqllogictest/src/lib.rs+++ b/datafusion/sqllogictest/src/lib.rs@@ -34,6 +34,7 @@ pub use engines::DFColumnType;pub use engines::DFOutput;pub use engines::DFSqlLogicTestError;pub use engines::DataFusion;+#[cfg(feature = "substrait")]pub use engines::DataFusionSubstraitRoundTrip;pub use engines::convert_batches;pub use engines::convert_schema_to_types;