Skip to content

Apache DataFusion SQL Query Engine

fix(substrait): roundtrip conditionless joins (#23469)

Bruno Volpatof9b7f3482a41
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index f17429656d..254ca80c53 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -700,10 +700,12 @@ jobs:
rust-version: stable
- name: Run sqllogictest
# TODO: Right now several tests are failing in Substrait round-trip mode, so this
- # command cannot be run for all the .slt files. Run it for just one that works (limit.slt)
+ # command cannot be run for all the .slt files. Run a supported subset
# until most of the tickets in https://github.com/apache/datafusion/issues/16248 are addressed
# and this command can be run without filters.
- run: cargo xtask ci step test substrait
+ run: |
+ cargo xtask ci step test substrait
+ cargo xtask ci step test substrait-optimized
# Temporarily commenting out the Windows flow, the reason is enormously slow running build
# Waiting for new Windows 2025 github runner
diff --git a/datafusion/sqllogictest/README.md b/datafusion/sqllogictest/README.md
index ad6c552f75..e6da0ae1dd 100644
--- a/datafusion/sqllogictest/README.md
+++ b/datafusion/sqllogictest/README.md
@@ -437,7 +437,8 @@ Not all statements will be round-tripped, some statements like CREATE, INSERT, S
issued as is, but any other statement will be round-tripped to/from Substrait.
_WARNING_: this mode lives behind the `substrait` feature, and the full suite still reports failures. CI therefore
-runs it over a single file, through `cargo xtask ci step test substrait`, which filters to `limit.slt`. Some of the
+runs `limit.slt` through `cargo xtask ci step test substrait` and optimized conditionless-join tests through
+`cargo xtask ci step test substrait-optimized`. Some of the
failures are collected in https://github.com/apache/datafusion/issues/16248. To run the default suite in this mode:
```shell
@@ -450,6 +451,14 @@ For focusing on one specific failing test, a file:line filter can be used:
cargo test --test sqllogictests --features substrait -- --substrait-round-trip binary.slt:23
```
+Add `--substrait-optimize` to optimize the logical plan before serialization. This exercises
+producer inputs created by optimizer rewrites, such as conditionless joins from `LEFT JOIN ... ON true`
+and uncorrelated `WHERE EXISTS`:
+
+```shell
+cargo test --test sqllogictests --features substrait -- --substrait-round-trip --substrait-optimize joins_conditionless.slt
+```
+
## `.slt` file format
[`sqllogictest`] was originally written for SQLite to verify the
diff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rs
index c64864cb77..1fd5d9d469 100644
--- a/datafusion/sqllogictest/bin/sqllogictests.rs
+++ b/datafusion/sqllogictest/bin/sqllogictests.rs
@@ -239,6 +239,7 @@ async fn run_tests() -> Result<()> {
filters.as_ref(),
currently_running_sql_tracker_clone,
colored_output,
+ options.substrait_optimize,
)
.await
}
@@ -461,7 +462,9 @@ fn is_env_truthy(name: &str) -> bool {
enum Engine {
DataFusion,
#[cfg(feature = "substrait")]
- SubstraitRoundTrip,
+ SubstraitRoundTrip {
+ optimize: bool,
+ },
}
impl Engine {
@@ -471,12 +474,13 @@ impl Engine {
match self {
Engine::DataFusion => "Datafusion",
#[cfg(feature = "substrait")]
- Engine::SubstraitRoundTrip => "DatafusionSubstraitRoundTrip",
+ Engine::SubstraitRoundTrip { .. } => "DatafusionSubstraitRoundTrip",
}
}
}
#[cfg(feature = "substrait")]
+#[expect(clippy::too_many_arguments, reason = "mirrors the other file runners")]
async fn run_test_file_substrait_round_trip(
test_file: TestFile,
validator: Validator,
@@ -485,9 +489,10 @@ async fn run_test_file_substrait_round_trip(
filters: &[Filter],
currently_executing_sql_tracker: CurrentlyExecutingSqlTracker,
colored_output: bool,
+ optimize: bool,
) -> Result<()> {
run_matrix(
- Engine::SubstraitRoundTrip,
+ Engine::SubstraitRoundTrip { optimize },
test_file,
validator,
mp,
@@ -504,6 +509,10 @@ async fn run_test_file_substrait_round_trip(
clippy::unused_async,
reason = "matches the substrait-enabled implementation"
)]
+#[expect(
+ clippy::too_many_arguments,
+ reason = "matches the enabled implementation"
+)]
async fn run_test_file_substrait_round_trip(
_test_file: TestFile,
_validator: Validator,
@@ -512,6 +521,7 @@ async fn run_test_file_substrait_round_trip(
_filters: &[Filter],
_currently_executing_sql_tracker: CurrentlyExecutingSqlTracker,
_colored_output: bool,
+ _optimize: bool,
) -> Result<()> {
exec_err!("Cannot run substrait round-trip: the 'substrait' feature is not enabled")
}
@@ -620,8 +630,8 @@ impl MatrixRunner<'_> {
match self.engine {
Engine::DataFusion => self.run_datafusion(&test_ctx, pb).await,
#[cfg(feature = "substrait")]
- Engine::SubstraitRoundTrip => {
- self.run_substrait_round_trip(&test_ctx, pb).await
+ Engine::SubstraitRoundTrip { optimize } => {
+ self.run_substrait_round_trip(&test_ctx, pb, optimize).await
}
}
}
@@ -689,6 +699,7 @@ impl MatrixRunner<'_> {
&self,
test_ctx: &TestContext,
pb: ProgressBar,
+ optimize: bool,
) -> Result<()> {
let mut runner = sqllogictest::Runner::new(|| async {
Ok(DataFusionSubstraitRoundTrip::new(
@@ -696,6 +707,7 @@ impl MatrixRunner<'_> {
self.relative_path.to_path_buf(),
pb.clone(),
)
+ .with_optimization(optimize)
.with_currently_executing_sql_tracker(
self.currently_executing_sql_tracker.clone(),
))
@@ -1065,6 +1077,13 @@ struct Options {
)]
substrait_round_trip: bool,
+ #[clap(
+ long,
+ requires = "substrait_round_trip",
+ help = "Optimize logical plans before serializing them in Substrait round-trip mode"
+ )]
+ substrait_optimize: bool,
+
#[clap(long, env = "INCLUDE_SQLITE", help = "Include sqlite files")]
include_sqlite: bool,
diff --git a/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs b/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs
index 02580eec91..a8a18ed3fd 100644
--- a/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs
+++ b/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs
@@ -42,6 +42,7 @@ pub struct DataFusionSubstraitRoundTrip {
relative_path: PathBuf,
pb: ProgressBar,
currently_executing_sql_tracker: CurrentlyExecutingSqlTracker,
+ optimize_before_round_trip: bool,
}
impl DataFusionSubstraitRoundTrip {
@@ -51,9 +52,16 @@ impl DataFusionSubstraitRoundTrip {
relative_path,
pb,
currently_executing_sql_tracker: CurrentlyExecutingSqlTracker::default(),
+ optimize_before_round_trip: false,
}
}
+ /// Optimize the logical plan before serializing it to Substrait.
+ pub fn with_optimization(mut self, optimize: bool) -> Self {
+ self.optimize_before_round_trip = optimize;
+ self
+ }
+
/// Add a tracker that will track the currently executed SQL statement.
///
/// This is useful for logging and debugging purposes.
@@ -101,7 +109,12 @@ impl sqllogictest::AsyncDB for DataFusionSubstraitRoundTrip {
let tracked_sql = self.currently_executing_sql_tracker.set_sql(sql);
let start = Instant::now();
- let result = run_query_substrait_round_trip(&self.ctx, sql).await;
+ let result = run_query_substrait_round_trip(
+ &self.ctx,
+ sql,
+ self.optimize_before_round_trip,
+ )
+ .await;
let duration = start.elapsed();
rewrap_replaced_pool(&self.ctx, &self.relative_path.display().to_string());
@@ -143,19 +156,25 @@ impl sqllogictest::AsyncDB for DataFusionSubstraitRoundTrip {
async fn run_query_substrait_round_trip(
ctx: &SessionContext,
sql: impl Into<String>,
+ optimize: bool,
) -> Result<DFOutput> {
let df = ctx.sql(sql.into().as_str()).await?;
let task_ctx = Arc::new(df.task_ctx());
let state = ctx.state();
- let round_tripped_plan = match df.logical_plan() {
+ let logical_plan = if optimize {
+ df.into_optimized_plan()?
+ } else {
+ df.into_unoptimized_plan()
+ };
+ let round_tripped_plan = match &logical_plan {
// Substrait does not handle these plans
LogicalPlan::Ddl(_)
| LogicalPlan::Explain(_)
| LogicalPlan::Dml(_)
| LogicalPlan::Copy(_)
| LogicalPlan::DescribeTable(_)
- | LogicalPlan::Statement(_) => df.logical_plan().clone(),
+ | LogicalPlan::Statement(_) => logical_plan,
// For any other plan, convert to Substrait
logical_plan => {
let plan = to_substrait_plan(logical_plan, &state)?;
diff --git a/datafusion/sqllogictest/test_files/joins_conditionless.slt b/datafusion/sqllogictest/test_files/joins_conditionless.slt
new file mode 100644
index 0000000000..5c7c05f0ee
--- /dev/null
+++ b/datafusion/sqllogictest/test_files/joins_conditionless.slt
@@ -0,0 +1,77 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+statement ok
+CREATE TABLE conditionless_left(a BIGINT) AS VALUES (1), (2);
+
+statement ok
+CREATE TABLE conditionless_right(b BIGINT) AS VALUES (10), (20);
+
+statement ok
+CREATE TABLE conditionless_empty(b BIGINT);
+
+# Optimization removes the true filter. Substrait still requires a join condition.
+query II rowsort
+SELECT a, b FROM conditionless_left LEFT JOIN conditionless_right ON true;
+----
+1 10
+1 20
+2 10
+2 20
+
+query II rowsort
+SELECT a, b FROM conditionless_left RIGHT JOIN conditionless_right ON true;
+----
+1 10
+1 20
+2 10
+2 20
+
+query II rowsort
+SELECT a, b FROM conditionless_left FULL JOIN conditionless_right ON true;
+----
+1 10
+1 20
+2 10
+2 20
+
+query II rowsort
+SELECT a, b FROM conditionless_left LEFT JOIN conditionless_empty ON true;
+----
+1 NULL
+2 NULL
+
+# Uncorrelated EXISTS and NOT EXISTS become conditionless semi and anti joins.
+query I rowsort
+SELECT a FROM conditionless_left WHERE EXISTS (SELECT 1 FROM conditionless_right);
+----
+1
+2
+
+query I rowsort
+SELECT a FROM conditionless_left WHERE EXISTS (SELECT 1 FROM conditionless_empty);
+----
+
+query I rowsort
+SELECT a FROM conditionless_left WHERE NOT EXISTS (SELECT 1 FROM conditionless_right);
+----
+
+query I rowsort
+SELECT a FROM conditionless_left WHERE NOT EXISTS (SELECT 1 FROM conditionless_empty);
+----
+1
+2
diff --git a/datafusion/substrait/src/logical_plan/producer/rel/join.rs b/datafusion/substrait/src/logical_plan/producer/rel/join.rs
index 9094774780..e3445fd33b 100644
--- a/datafusion/substrait/src/logical_plan/producer/rel/join.rs
+++ b/datafusion/substrait/src/logical_plan/producer/rel/join.rs
@@ -18,11 +18,11 @@
use crate::logical_plan::producer::SubstraitProducer;
use datafusion::common::{JoinConstraint, JoinType, NullEquality, not_impl_err};
use datafusion::logical_expr::utils::conjunction;
-use datafusion::logical_expr::{Expr, Join, Operator};
+use datafusion::logical_expr::{Expr, Join, Operator, lit};
use datafusion::prelude::binary_expr;
use std::sync::Arc;
use substrait::proto::rel::RelType;
-use substrait::proto::{JoinRel, Rel, join_rel};
+use substrait::proto::{CrossRel, JoinRel, Rel, join_rel};
pub fn from_join(
producer: &mut impl SubstraitProducer,
@@ -38,15 +38,28 @@ pub fn from_join(
let right = producer.handle_plan(join.right.as_ref())?;
let join_type = to_substrait_jointype(join.join_type);
- let join_expr =
- to_substrait_join_expr(join.on.clone(), join.null_equality, join.filter.clone());
- let join_expression = match join_expr {
+ let join_expression = match to_substrait_join_expr(
+ join.on.clone(),
+ join.null_equality,
+ join.filter.clone(),
+ ) {
Some(expr) => {
let in_join_schema = Arc::new(join.left.schema().join(join.right.schema())?);
- let expression = producer.handle_expr(&expr, &in_join_schema)?;
- Some(Box::new(expression))
+ producer.handle_expr(&expr, &in_join_schema)?
}
- None => None,
+ None if join.join_type == JoinType::Inner => {
+ return Ok(Box::new(Rel {
+ rel_type: Some(RelType::Cross(Box::new(CrossRel {
+ common: None,
+ left: Some(left),
+ right: Some(right),
+ advanced_extension: None,
+ }))),
+ }));
+ }
+ // The literal needs no input column resolution. Semi, anti, and mark joins
+ // can have overlapping input names that cannot form a combined schema.
+ None => producer.handle_expr(&lit(true), &join.schema)?,
};
Ok(Box::new(Rel {
@@ -55,7 +68,7 @@ pub fn from_join(
left: Some(left),
right: Some(right),
r#type: join_type as i32,
- expression: join_expression,
+ expression: Some(Box::new(join_expression)),
post_join_filter: None,
advanced_extension: None,
}))),
@@ -105,9 +118,10 @@ mod tests {
use datafusion::logical_expr::utils::conjunction;
use datafusion::logical_expr::{Join, col, table_scan};
use std::sync::Arc;
- use substrait::proto::expression::{RexType, ScalarFunction};
+ use substrait::proto::expression::literal::LiteralType;
+ use substrait::proto::expression::{Literal, RexType, ScalarFunction};
use substrait::proto::rel::RelType;
- use substrait::proto::{Expression, JoinRel, Rel, join_rel};
+ use substrait::proto::{CrossRel, Expression, JoinRel, Rel, join_rel};
#[test]
fn test_from_join() -> datafusion::common::Result<()> {
@@ -174,4 +188,97 @@ mod tests {
Ok(())
}
+
+ #[test]
+ fn conditionless_inner_join_is_cross_rel() {
+ let state = SessionStateBuilder::default().build();
+ let mut producer = DefaultSubstraitProducer::new(&state);
+ let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
+ let left = Arc::new(
+ table_scan(Some("t1"), &schema, None)
+ .unwrap()
+ .build()
+ .unwrap(),
+ );
+ let right = Arc::new(
+ table_scan(Some("t2"), &schema, None)
+ .unwrap()
+ .build()
+ .unwrap(),
+ );
+
+ let inner = Join::try_new(
+ Arc::clone(&left),
+ Arc::clone(&right),
+ vec![],
+ None,
+ JoinType::Inner,
+ JoinConstraint::On,
+ NullEquality::NullEqualsNothing,
+ false,
+ )
+ .unwrap();
+ let expected = Rel {
+ rel_type: Some(RelType::Cross(Box::new(CrossRel {
+ common: None,
+ left: Some(producer.handle_plan(&left).unwrap()),
+ right: Some(producer.handle_plan(&right).unwrap()),
+ advanced_extension: None,
+ }))),
+ };
+
+ assert_eq!(producer.handle_join(&inner).unwrap(), Box::new(expected));
+ }
+
+ #[test]
+ fn conditionless_joins_with_overlapping_inputs() -> datafusion::common::Result<()> {
+ let state = SessionStateBuilder::default().build();
+ let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
+ let qualified = table_scan(Some("t1"), &schema, None)?.build()?;
+ let unqualified = table_scan(Some("t1"), &schema, None)?
+ .project(vec![col("t1.a").alias("a")])?
+ .build()?;
+
+ for input in [qualified, unqualified] {
+ let input = Arc::new(input);
+ for join_type in [
+ JoinType::LeftSemi,
+ JoinType::LeftAnti,
+ JoinType::RightSemi,
+ JoinType::RightAnti,
+ JoinType::LeftMark,
+ JoinType::RightMark,
+ ] {
+ let mut producer = DefaultSubstraitProducer::new(&state);
+ let join = Join::try_new(
+ Arc::clone(&input),
+ Arc::clone(&input),
+ vec![],
+ None,
+ join_type,
+ JoinConstraint::On,
+ NullEquality::NullEqualsNothing,
+ false,
+ )?;
+ let rel = producer.handle_join(&join)?;
+ let Some(RelType::Join(join_rel)) = rel.rel_type else {
+ panic!("expected JoinRel for {join_type:?}");
+ };
+ assert_eq!(
+ join_rel.r#type,
+ super::to_substrait_jointype(join_type) as i32
+ );
+ assert!(matches!(
+ join_rel.expression.as_deref(),
+ Some(Expression {
+ rex_type: Some(RexType::Literal(Literal {
+ literal_type: Some(LiteralType::Boolean(true)),
+ ..
+ })),
+ })
+ ));
+ }
+ }
+ Ok(())
+ }
}
diff --git a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs
index 6d783af16e..db6119a209 100644
--- a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs
+++ b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs
@@ -845,6 +845,50 @@ async fn roundtrip_scalar_subquery_substrait() -> Result<()> {
Ok(())
}
+#[tokio::test]
+async fn roundtrip_scalar_subquery_projection_with_join_rewrite() -> Result<()> {
+ let ctx = create_context().await?;
+ ctx.sql(
+ "SET datafusion.optimizer.enable_physical_uncorrelated_scalar_subquery = false",
+ )
+ .await?
+ .collect()
+ .await?;
+
+ let plan = generate_plan_from_sql_with_ctx(
+ "SELECT a, (SELECT MAX(b) FROM data2) AS max_b FROM data",
+ false,
+ true,
+ &ctx,
+ )
+ .await?;
+
+ assert_snapshot!(
+ plan,
+ @r"
+ Projection: data.a, max(data2.b) AS max_b
+ Left Join:
+ TableScan: data projection=[a]
+ Aggregate: groupBy=[[]], aggr=[[max(data2.b)]]
+ TableScan: data2 projection=[b], partial_filters=[Boolean(true)]
+ "
+ );
+
+ let results = DataFrame::new(ctx.state(), plan).collect().await?;
+ datafusion::assert_batches_sorted_eq!(
+ [
+ "+---+-------+",
+ "| a | max_b |",
+ "+---+-------+",
+ "| 1 | 4.5 |",
+ "| 3 | 4.5 |",
+ "+---+-------+",
+ ],
+ &results
+ );
+ Ok(())
+}
+
#[tokio::test]
async fn roundtrip_exists_substrait() -> Result<()> {
let ctx = create_context().await?;
diff --git a/xtask/src/ci_steps.rs b/xtask/src/ci_steps.rs
index 521d527ab8..1f0a2406d6 100644
--- a/xtask/src/ci_steps.rs
+++ b/xtask/src/ci_steps.rs
@@ -55,7 +55,7 @@ static CI_STEPS: &[StepInfo] = &[
},
StepInfo {
command: "test",
- help_usage: "test <workspace|cli|doctest|ffi|benchmark-plan|benchmark-sqllogic|postgres|substrait|extended|hash-collisions|sqlite> [--explain]",
+ help_usage: "test <workspace|cli|doctest|ffi|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|hash-collisions|sqlite> [--explain]",
help_examples: &["test cli", "test workspace"],
help_description: "Run a CI test suite",
error_message: "Cargo test step failed",
@@ -344,7 +344,7 @@ impl StepContext {
.env("PG_COMPAT", "true")
.env("PG_URI", uri);
}
- "substrait" => {
+ "substrait" | "substrait-optimized" => {
command.args([
"test",
"-p",
@@ -355,8 +355,12 @@ impl StepContext {
"substrait",
"--",
"--substrait-round-trip",
- "limit.slt",
]);
+ if variant == "substrait-optimized" {
+ command.args(["--substrait-optimize", "joins_conditionless.slt"]);
+ } else {
+ command.args(["limit.slt"]);
+ }
}
"extended" => {
command
@@ -610,7 +614,7 @@ fn required_env(name: &str) -> Result<String> {
mod tests {
use super::*;
- const TEST_USAGE: &str = "usage: cargo xtask ci step test <workspace|cli|doctest|ffi|benchmark-plan|benchmark-sqllogic|postgres|substrait|extended|hash-collisions|sqlite> [--explain]";
+ const TEST_USAGE: &str = "usage: cargo xtask ci step test <workspace|cli|doctest|ffi|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|hash-collisions|sqlite> [--explain]";
fn args(args: &[&str]) -> Vec<String> {
args.iter().map(|arg| (*arg).to_string()).collect()
@@ -848,6 +852,19 @@ cargo test \
--test sqllogictests \
--features substrait \
-- --substrait-round-trip limit.slt
+")
+ },
+ },
+ Cmd {
+ command: &["test", "substrait-optimized"],
+ expected: |actual| {
+ insta::assert_snapshot!(actual, @r"
+cd /workspace && \
+cargo test \
+-p datafusion-sqllogictest \
+--test sqllogictests \
+--features substrait \
+-- --substrait-round-trip --substrait-optimize joins_conditionless.slt
")
},
},