Apache DataFusion SQL Query Engine
fix(substrait): roundtrip conditionless joins (#23469)
Bruno Volpatof9b7f3482a41
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.ymlindex f17429656d..254ca80c53 100644--- a/.github/workflows/rust.yml+++ b/.github/workflows/rust.yml@@ -700,10 +700,12 @@ jobs:rust-version: stable- name: Run sqllogictest# TODO: Right now several tests are failing in Substrait round-trip mode, so this- # command cannot be run for all the .slt files. Run it for just one that works (limit.slt)+ # command cannot be run for all the .slt files. Run a supported subset# until most of the tickets in https://github.com/apache/datafusion/issues/16248 are addressed# and this command can be run without filters.- run: cargo xtask ci step test substrait+ run: |+ cargo xtask ci step test substrait+ cargo xtask ci step test substrait-optimized# Temporarily commenting out the Windows flow, the reason is enormously slow running build# Waiting for new Windows 2025 github runnerdiff --git a/datafusion/sqllogictest/README.md b/datafusion/sqllogictest/README.mdindex ad6c552f75..e6da0ae1dd 100644--- a/datafusion/sqllogictest/README.md+++ b/datafusion/sqllogictest/README.md@@ -437,7 +437,8 @@ Not all statements will be round-tripped, some statements like CREATE, INSERT, Sissued as is, but any other statement will be round-tripped to/from Substrait._WARNING_: this mode lives behind the `substrait` feature, and the full suite still reports failures. CI therefore-runs it over a single file, through `cargo xtask ci step test substrait`, which filters to `limit.slt`. Some of the+runs `limit.slt` through `cargo xtask ci step test substrait` and optimized conditionless-join tests through+`cargo xtask ci step test substrait-optimized`. Some of thefailures are collected in https://github.com/apache/datafusion/issues/16248. To run the default suite in this mode:```shell@@ -450,6 +451,14 @@ For focusing on one specific failing test, a file:line filter can be used:cargo test --test sqllogictests --features substrait -- --substrait-round-trip binary.slt:23```+Add `--substrait-optimize` to optimize the logical plan before serialization. This exercises+producer inputs created by optimizer rewrites, such as conditionless joins from `LEFT JOIN ... ON true`+and uncorrelated `WHERE EXISTS`:++```shell+cargo test --test sqllogictests --features substrait -- --substrait-round-trip --substrait-optimize joins_conditionless.slt+```+## `.slt` file format[`sqllogictest`] was originally written for SQLite to verify thediff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rsindex c64864cb77..1fd5d9d469 100644--- a/datafusion/sqllogictest/bin/sqllogictests.rs+++ b/datafusion/sqllogictest/bin/sqllogictests.rs@@ -239,6 +239,7 @@ async fn run_tests() -> Result<()> {filters.as_ref(),currently_running_sql_tracker_clone,colored_output,+ options.substrait_optimize,).await}@@ -461,7 +462,9 @@ fn is_env_truthy(name: &str) -> bool {enum Engine {DataFusion,#[cfg(feature = "substrait")]- SubstraitRoundTrip,+ SubstraitRoundTrip {+ optimize: bool,+ },}impl Engine {@@ -471,12 +474,13 @@ impl Engine {match self {Engine::DataFusion => "Datafusion",#[cfg(feature = "substrait")]- Engine::SubstraitRoundTrip => "DatafusionSubstraitRoundTrip",+ Engine::SubstraitRoundTrip { .. } => "DatafusionSubstraitRoundTrip",}}}#[cfg(feature = "substrait")]+#[expect(clippy::too_many_arguments, reason = "mirrors the other file runners")]async fn run_test_file_substrait_round_trip(test_file: TestFile,validator: Validator,@@ -485,9 +489,10 @@ async fn run_test_file_substrait_round_trip(filters: &[Filter],currently_executing_sql_tracker: CurrentlyExecutingSqlTracker,colored_output: bool,+ optimize: bool,) -> Result<()> {run_matrix(- Engine::SubstraitRoundTrip,+ Engine::SubstraitRoundTrip { optimize },test_file,validator,mp,@@ -504,6 +509,10 @@ async fn run_test_file_substrait_round_trip(clippy::unused_async,reason = "matches the substrait-enabled implementation")]+#[expect(+ clippy::too_many_arguments,+ reason = "matches the enabled implementation"+)]async fn run_test_file_substrait_round_trip(_test_file: TestFile,_validator: Validator,@@ -512,6 +521,7 @@ async fn run_test_file_substrait_round_trip(_filters: &[Filter],_currently_executing_sql_tracker: CurrentlyExecutingSqlTracker,_colored_output: bool,+ _optimize: bool,) -> Result<()> {exec_err!("Cannot run substrait round-trip: the 'substrait' feature is not enabled")}@@ -620,8 +630,8 @@ impl MatrixRunner<'_> {match self.engine {Engine::DataFusion => self.run_datafusion(&test_ctx, pb).await,#[cfg(feature = "substrait")]- Engine::SubstraitRoundTrip => {- self.run_substrait_round_trip(&test_ctx, pb).await+ Engine::SubstraitRoundTrip { optimize } => {+ self.run_substrait_round_trip(&test_ctx, pb, optimize).await}}}@@ -689,6 +699,7 @@ impl MatrixRunner<'_> {&self,test_ctx: &TestContext,pb: ProgressBar,+ optimize: bool,) -> Result<()> {let mut runner = sqllogictest::Runner::new(|| async {Ok(DataFusionSubstraitRoundTrip::new(@@ -696,6 +707,7 @@ impl MatrixRunner<'_> {self.relative_path.to_path_buf(),pb.clone(),)+ .with_optimization(optimize).with_currently_executing_sql_tracker(self.currently_executing_sql_tracker.clone(),))@@ -1065,6 +1077,13 @@ struct Options {)]substrait_round_trip: bool,+ #[clap(+ long,+ requires = "substrait_round_trip",+ help = "Optimize logical plans before serializing them in Substrait round-trip mode"+ )]+ substrait_optimize: bool,+#[clap(long, env = "INCLUDE_SQLITE", help = "Include sqlite files")]include_sqlite: bool,diff --git a/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs b/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rsindex 02580eec91..a8a18ed3fd 100644--- a/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs+++ b/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs@@ -42,6 +42,7 @@ pub struct DataFusionSubstraitRoundTrip {relative_path: PathBuf,pb: ProgressBar,currently_executing_sql_tracker: CurrentlyExecutingSqlTracker,+ optimize_before_round_trip: bool,}impl DataFusionSubstraitRoundTrip {@@ -51,9 +52,16 @@ impl DataFusionSubstraitRoundTrip {relative_path,pb,currently_executing_sql_tracker: CurrentlyExecutingSqlTracker::default(),+ optimize_before_round_trip: false,}}+ /// Optimize the logical plan before serializing it to Substrait.+ pub fn with_optimization(mut self, optimize: bool) -> Self {+ self.optimize_before_round_trip = optimize;+ self+ }+/// Add a tracker that will track the currently executed SQL statement.////// This is useful for logging and debugging purposes.@@ -101,7 +109,12 @@ impl sqllogictest::AsyncDB for DataFusionSubstraitRoundTrip {let tracked_sql = self.currently_executing_sql_tracker.set_sql(sql);let start = Instant::now();- let result = run_query_substrait_round_trip(&self.ctx, sql).await;+ let result = run_query_substrait_round_trip(+ &self.ctx,+ sql,+ self.optimize_before_round_trip,+ )+ .await;let duration = start.elapsed();rewrap_replaced_pool(&self.ctx, &self.relative_path.display().to_string());@@ -143,19 +156,25 @@ impl sqllogictest::AsyncDB for DataFusionSubstraitRoundTrip {async fn run_query_substrait_round_trip(ctx: &SessionContext,sql: impl Into<String>,+ optimize: bool,) -> Result<DFOutput> {let df = ctx.sql(sql.into().as_str()).await?;let task_ctx = Arc::new(df.task_ctx());let state = ctx.state();- let round_tripped_plan = match df.logical_plan() {+ let logical_plan = if optimize {+ df.into_optimized_plan()?+ } else {+ df.into_unoptimized_plan()+ };+ let round_tripped_plan = match &logical_plan {// Substrait does not handle these plansLogicalPlan::Ddl(_)| LogicalPlan::Explain(_)| LogicalPlan::Dml(_)| LogicalPlan::Copy(_)| LogicalPlan::DescribeTable(_)- | LogicalPlan::Statement(_) => df.logical_plan().clone(),+ | LogicalPlan::Statement(_) => logical_plan,// For any other plan, convert to Substraitlogical_plan => {let plan = to_substrait_plan(logical_plan, &state)?;diff --git a/datafusion/sqllogictest/test_files/joins_conditionless.slt b/datafusion/sqllogictest/test_files/joins_conditionless.sltnew file mode 100644index 0000000000..5c7c05f0ee--- /dev/null+++ b/datafusion/sqllogictest/test_files/joins_conditionless.slt@@ -0,0 +1,77 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++statement ok+CREATE TABLE conditionless_left(a BIGINT) AS VALUES (1), (2);++statement ok+CREATE TABLE conditionless_right(b BIGINT) AS VALUES (10), (20);++statement ok+CREATE TABLE conditionless_empty(b BIGINT);++# Optimization removes the true filter. Substrait still requires a join condition.+query II rowsort+SELECT a, b FROM conditionless_left LEFT JOIN conditionless_right ON true;+----+1 10+1 20+2 10+2 20++query II rowsort+SELECT a, b FROM conditionless_left RIGHT JOIN conditionless_right ON true;+----+1 10+1 20+2 10+2 20++query II rowsort+SELECT a, b FROM conditionless_left FULL JOIN conditionless_right ON true;+----+1 10+1 20+2 10+2 20++query II rowsort+SELECT a, b FROM conditionless_left LEFT JOIN conditionless_empty ON true;+----+1 NULL+2 NULL++# Uncorrelated EXISTS and NOT EXISTS become conditionless semi and anti joins.+query I rowsort+SELECT a FROM conditionless_left WHERE EXISTS (SELECT 1 FROM conditionless_right);+----+1+2++query I rowsort+SELECT a FROM conditionless_left WHERE EXISTS (SELECT 1 FROM conditionless_empty);+----++query I rowsort+SELECT a FROM conditionless_left WHERE NOT EXISTS (SELECT 1 FROM conditionless_right);+----++query I rowsort+SELECT a FROM conditionless_left WHERE NOT EXISTS (SELECT 1 FROM conditionless_empty);+----+1+2diff --git a/datafusion/substrait/src/logical_plan/producer/rel/join.rs b/datafusion/substrait/src/logical_plan/producer/rel/join.rsindex 9094774780..e3445fd33b 100644--- a/datafusion/substrait/src/logical_plan/producer/rel/join.rs+++ b/datafusion/substrait/src/logical_plan/producer/rel/join.rs@@ -18,11 +18,11 @@use crate::logical_plan::producer::SubstraitProducer;use datafusion::common::{JoinConstraint, JoinType, NullEquality, not_impl_err};use datafusion::logical_expr::utils::conjunction;-use datafusion::logical_expr::{Expr, Join, Operator};+use datafusion::logical_expr::{Expr, Join, Operator, lit};use datafusion::prelude::binary_expr;use std::sync::Arc;use substrait::proto::rel::RelType;-use substrait::proto::{JoinRel, Rel, join_rel};+use substrait::proto::{CrossRel, JoinRel, Rel, join_rel};pub fn from_join(producer: &mut impl SubstraitProducer,@@ -38,15 +38,28 @@ pub fn from_join(let right = producer.handle_plan(join.right.as_ref())?;let join_type = to_substrait_jointype(join.join_type);- let join_expr =- to_substrait_join_expr(join.on.clone(), join.null_equality, join.filter.clone());- let join_expression = match join_expr {+ let join_expression = match to_substrait_join_expr(+ join.on.clone(),+ join.null_equality,+ join.filter.clone(),+ ) {Some(expr) => {let in_join_schema = Arc::new(join.left.schema().join(join.right.schema())?);- let expression = producer.handle_expr(&expr, &in_join_schema)?;- Some(Box::new(expression))+ producer.handle_expr(&expr, &in_join_schema)?}- None => None,+ None if join.join_type == JoinType::Inner => {+ return Ok(Box::new(Rel {+ rel_type: Some(RelType::Cross(Box::new(CrossRel {+ common: None,+ left: Some(left),+ right: Some(right),+ advanced_extension: None,+ }))),+ }));+ }+ // The literal needs no input column resolution. Semi, anti, and mark joins+ // can have overlapping input names that cannot form a combined schema.+ None => producer.handle_expr(&lit(true), &join.schema)?,};Ok(Box::new(Rel {@@ -55,7 +68,7 @@ pub fn from_join(left: Some(left),right: Some(right),r#type: join_type as i32,- expression: join_expression,+ expression: Some(Box::new(join_expression)),post_join_filter: None,advanced_extension: None,}))),@@ -105,9 +118,10 @@ mod tests {use datafusion::logical_expr::utils::conjunction;use datafusion::logical_expr::{Join, col, table_scan};use std::sync::Arc;- use substrait::proto::expression::{RexType, ScalarFunction};+ use substrait::proto::expression::literal::LiteralType;+ use substrait::proto::expression::{Literal, RexType, ScalarFunction};use substrait::proto::rel::RelType;- use substrait::proto::{Expression, JoinRel, Rel, join_rel};+ use substrait::proto::{CrossRel, Expression, JoinRel, Rel, join_rel};#[test]fn test_from_join() -> datafusion::common::Result<()> {@@ -174,4 +188,97 @@ mod tests {Ok(())}++ #[test]+ fn conditionless_inner_join_is_cross_rel() {+ let state = SessionStateBuilder::default().build();+ let mut producer = DefaultSubstraitProducer::new(&state);+ let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);+ let left = Arc::new(+ table_scan(Some("t1"), &schema, None)+ .unwrap()+ .build()+ .unwrap(),+ );+ let right = Arc::new(+ table_scan(Some("t2"), &schema, None)+ .unwrap()+ .build()+ .unwrap(),+ );++ let inner = Join::try_new(+ Arc::clone(&left),+ Arc::clone(&right),+ vec![],+ None,+ JoinType::Inner,+ JoinConstraint::On,+ NullEquality::NullEqualsNothing,+ false,+ )+ .unwrap();+ let expected = Rel {+ rel_type: Some(RelType::Cross(Box::new(CrossRel {+ common: None,+ left: Some(producer.handle_plan(&left).unwrap()),+ right: Some(producer.handle_plan(&right).unwrap()),+ advanced_extension: None,+ }))),+ };++ assert_eq!(producer.handle_join(&inner).unwrap(), Box::new(expected));+ }++ #[test]+ fn conditionless_joins_with_overlapping_inputs() -> datafusion::common::Result<()> {+ let state = SessionStateBuilder::default().build();+ let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);+ let qualified = table_scan(Some("t1"), &schema, None)?.build()?;+ let unqualified = table_scan(Some("t1"), &schema, None)?+ .project(vec![col("t1.a").alias("a")])?+ .build()?;++ for input in [qualified, unqualified] {+ let input = Arc::new(input);+ for join_type in [+ JoinType::LeftSemi,+ JoinType::LeftAnti,+ JoinType::RightSemi,+ JoinType::RightAnti,+ JoinType::LeftMark,+ JoinType::RightMark,+ ] {+ let mut producer = DefaultSubstraitProducer::new(&state);+ let join = Join::try_new(+ Arc::clone(&input),+ Arc::clone(&input),+ vec![],+ None,+ join_type,+ JoinConstraint::On,+ NullEquality::NullEqualsNothing,+ false,+ )?;+ let rel = producer.handle_join(&join)?;+ let Some(RelType::Join(join_rel)) = rel.rel_type else {+ panic!("expected JoinRel for {join_type:?}");+ };+ assert_eq!(+ join_rel.r#type,+ super::to_substrait_jointype(join_type) as i32+ );+ assert!(matches!(+ join_rel.expression.as_deref(),+ Some(Expression {+ rex_type: Some(RexType::Literal(Literal {+ literal_type: Some(LiteralType::Boolean(true)),+ ..+ })),+ })+ ));+ }+ }+ Ok(())+ }}diff --git a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rsindex 6d783af16e..db6119a209 100644--- a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs+++ b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs@@ -845,6 +845,50 @@ async fn roundtrip_scalar_subquery_substrait() -> Result<()> {Ok(())}+#[tokio::test]+async fn roundtrip_scalar_subquery_projection_with_join_rewrite() -> Result<()> {+ let ctx = create_context().await?;+ ctx.sql(+ "SET datafusion.optimizer.enable_physical_uncorrelated_scalar_subquery = false",+ )+ .await?+ .collect()+ .await?;++ let plan = generate_plan_from_sql_with_ctx(+ "SELECT a, (SELECT MAX(b) FROM data2) AS max_b FROM data",+ false,+ true,+ &ctx,+ )+ .await?;++ assert_snapshot!(+ plan,+ @r"+ Projection: data.a, max(data2.b) AS max_b+ Left Join:+ TableScan: data projection=[a]+ Aggregate: groupBy=[[]], aggr=[[max(data2.b)]]+ TableScan: data2 projection=[b], partial_filters=[Boolean(true)]+ "+ );++ let results = DataFrame::new(ctx.state(), plan).collect().await?;+ datafusion::assert_batches_sorted_eq!(+ [+ "+---+-------+",+ "| a | max_b |",+ "+---+-------+",+ "| 1 | 4.5 |",+ "| 3 | 4.5 |",+ "+---+-------+",+ ],+ &results+ );+ Ok(())+}+#[tokio::test]async fn roundtrip_exists_substrait() -> Result<()> {let ctx = create_context().await?;diff --git a/xtask/src/ci_steps.rs b/xtask/src/ci_steps.rsindex 521d527ab8..1f0a2406d6 100644--- a/xtask/src/ci_steps.rs+++ b/xtask/src/ci_steps.rs@@ -55,7 +55,7 @@ static CI_STEPS: &[StepInfo] = &[},StepInfo {command: "test",- help_usage: "test <workspace|cli|doctest|ffi|benchmark-plan|benchmark-sqllogic|postgres|substrait|extended|hash-collisions|sqlite> [--explain]",+ help_usage: "test <workspace|cli|doctest|ffi|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|hash-collisions|sqlite> [--explain]",help_examples: &["test cli", "test workspace"],help_description: "Run a CI test suite",error_message: "Cargo test step failed",@@ -344,7 +344,7 @@ impl StepContext {.env("PG_COMPAT", "true").env("PG_URI", uri);}- "substrait" => {+ "substrait" | "substrait-optimized" => {command.args(["test","-p",@@ -355,8 +355,12 @@ impl StepContext {"substrait","--","--substrait-round-trip",- "limit.slt",]);+ if variant == "substrait-optimized" {+ command.args(["--substrait-optimize", "joins_conditionless.slt"]);+ } else {+ command.args(["limit.slt"]);+ }}"extended" => {command@@ -610,7 +614,7 @@ fn required_env(name: &str) -> Result<String> {mod tests {use super::*;- const TEST_USAGE: &str = "usage: cargo xtask ci step test <workspace|cli|doctest|ffi|benchmark-plan|benchmark-sqllogic|postgres|substrait|extended|hash-collisions|sqlite> [--explain]";+ const TEST_USAGE: &str = "usage: cargo xtask ci step test <workspace|cli|doctest|ffi|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|hash-collisions|sqlite> [--explain]";fn args(args: &[&str]) -> Vec<String> {args.iter().map(|arg| (*arg).to_string()).collect()@@ -848,6 +852,19 @@ cargo test \--test sqllogictests \--features substrait \-- --substrait-round-trip limit.slt+")+ },+ },+ Cmd {+ command: &["test", "substrait-optimized"],+ expected: |actual| {+ insta::assert_snapshot!(actual, @r"+cd /workspace && \+cargo test \+-p datafusion-sqllogictest \+--test sqllogictests \+--features substrait \+-- --substrait-round-trip --substrait-optimize joins_conditionless.slt")},},