Skip to content

Apache DataFusion SQL Query Engine

Fix Schema Duplication Errors in Self‑Referential INTERSECT/EXCEPT by Requalifying Input Sides (#18814)

kosiewa3a020f6b0be
diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rs
index 835f99c274..6f654428e4 100644
--- a/datafusion/expr/src/logical_plan/builder.rs
+++ b/datafusion/expr/src/logical_plan/builder.rs
@@ -1352,6 +1352,15 @@ impl LogicalPlanBuilder {
);
}
+ // Requalify sides if needed to avoid duplicate qualified field names
+ // (e.g., when both sides reference the same table)
+ let left_builder = LogicalPlanBuilder::from(left_plan);
+ let right_builder = LogicalPlanBuilder::from(right_plan);
+ let (left_builder, right_builder, _requalified) =
+ requalify_sides_if_needed(left_builder, right_builder)?;
+ let left_plan = left_builder.build()?;
+ let right_plan = right_builder.build()?;
+
let join_keys = left_plan
.schema()
.fields()
@@ -1731,23 +1740,61 @@ pub fn requalify_sides_if_needed(
) -> Result<(LogicalPlanBuilder, LogicalPlanBuilder, bool)> {
let left_cols = left.schema().columns();
let right_cols = right.schema().columns();
- if left_cols.iter().any(|l| {
- right_cols.iter().any(|r| {
- l == r || (l.name == r.name && (l.relation.is_none() || r.relation.is_none()))
- })
- }) {
- // These names have no connection to the original plan, but they'll make the columns
- // (mostly) unique.
- Ok((
- left.alias(TableReference::bare("left"))?,
- right.alias(TableReference::bare("right"))?,
- true,
- ))
- } else {
- Ok((left, right, false))
+
+ // Requalify if merging the schemas would cause an error during join.
+ // This can happen in several cases:
+ // 1. Duplicate qualified fields: both sides have same relation.name
+ // 2. Duplicate unqualified fields: both sides have same unqualified name
+ // 3. Ambiguous reference: one side qualified, other unqualified, same name
+ //
+ // Implementation note: This uses a simple O(n*m) nested loop rather than
+ // a HashMap-based O(n+m) approach. The nested loop is preferred because:
+ // - Schemas are typically small (in TPCH benchmark, max is 16 columns),
+ // so n*m is negligible
+ // - Early return on first conflict makes common case very fast
+ // - Code is simpler and easier to reason about
+ // - Called only during plan construction, not in execution hot path
+ for l in &left_cols {
+ for r in &right_cols {
+ if l.name != r.name {
+ continue;
+ }
+
+ // Same name - check if this would cause a conflict
+ match (&l.relation, &r.relation) {
+ // Both qualified with same relation - duplicate qualified field
+ (Some(l_rel), Some(r_rel)) if l_rel == r_rel => {
+ return Ok((
+ left.alias(TableReference::bare("left"))?,
+ right.alias(TableReference::bare("right"))?,
+ true,
+ ));
+ }
+ // Both unqualified - duplicate unqualified field
+ (None, None) => {
+ return Ok((
+ left.alias(TableReference::bare("left"))?,
+ right.alias(TableReference::bare("right"))?,
+ true,
+ ));
+ }
+ // One qualified, one not - ambiguous reference
+ (Some(_), None) | (None, Some(_)) => {
+ return Ok((
+ left.alias(TableReference::bare("left"))?,
+ right.alias(TableReference::bare("right"))?,
+ true,
+ ));
+ }
+ // Different qualifiers - OK, no conflict
+ _ => {}
+ }
+ }
}
-}
+ // No conflicts found
+ Ok((left, right, false))
+}
/// Add additional "synthetic" group by expressions based on functional
/// dependencies.
///
diff --git a/datafusion/optimizer/tests/optimizer_integration.rs b/datafusion/optimizer/tests/optimizer_integration.rs
index c0f48b8ebf..31ec026c1c 100644
--- a/datafusion/optimizer/tests/optimizer_integration.rs
+++ b/datafusion/optimizer/tests/optimizer_integration.rs
@@ -270,12 +270,14 @@ fn intersect() -> Result<()> {
assert_snapshot!(
format!("{plan}"),
@r#"
-LeftSemi Join: test.col_int32 = test.col_int32, test.col_utf8 = test.col_utf8
- Aggregate: groupBy=[[test.col_int32, test.col_utf8]], aggr=[[]]
- LeftSemi Join: test.col_int32 = test.col_int32, test.col_utf8 = test.col_utf8
- Aggregate: groupBy=[[test.col_int32, test.col_utf8]], aggr=[[]]
+LeftSemi Join: left.col_int32 = test.col_int32, left.col_utf8 = test.col_utf8
+ Aggregate: groupBy=[[left.col_int32, left.col_utf8]], aggr=[[]]
+ LeftSemi Join: left.col_int32 = right.col_int32, left.col_utf8 = right.col_utf8
+ Aggregate: groupBy=[[left.col_int32, left.col_utf8]], aggr=[[]]
+ SubqueryAlias: left
+ TableScan: test projection=[col_int32, col_utf8]
+ SubqueryAlias: right
TableScan: test projection=[col_int32, col_utf8]
- TableScan: test projection=[col_int32, col_utf8]
TableScan: test projection=[col_int32, col_utf8]
"#
);
diff --git a/datafusion/substrait/src/logical_plan/consumer/rel/set_rel.rs b/datafusion/substrait/src/logical_plan/consumer/rel/set_rel.rs
index 6688a80f52..36bf8dbae4 100644
--- a/datafusion/substrait/src/logical_plan/consumer/rel/set_rel.rs
+++ b/datafusion/substrait/src/logical_plan/consumer/rel/set_rel.rs
@@ -81,7 +81,7 @@ async fn intersect_rels(
rel,
consumer.consume_rel(input).await?,
is_all,
- )?
+ )?;
}
Ok(rel)
@@ -95,7 +95,8 @@ async fn except_rels(
let mut rel = consumer.consume_rel(&rels[0]).await?;
for input in &rels[1..] {
- rel = LogicalPlanBuilder::except(rel, consumer.consume_rel(input).await?, is_all)?
+ rel =
+ LogicalPlanBuilder::except(rel, consumer.consume_rel(input).await?, is_all)?;
}
Ok(rel)
diff --git a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs
index c79a501ec8..34cb05fbf7 100644
--- a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs
+++ b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs
@@ -1162,6 +1162,96 @@ async fn simple_intersect_table_reuse() -> Result<()> {
Ok(())
}
+#[tokio::test]
+async fn self_referential_intersect() -> Result<()> {
+ // Test INTERSECT with the same table on both sides
+ // This previously failed with "Schema contains duplicate qualified field name"
+ // The fix ensures requalify_sides_if_needed is called in intersect_or_except
+ // After roundtrip through Substrait, SubqueryAlias is lost and requalification
+ // produces "left" and "right" aliases
+ // Note: INTERSECT (without ALL) includes DISTINCT, but the outer Aggregate
+ // is optimized away, resulting in just the **LeftSemi** join
+ // (LeftSemi returns rows from left that exist in right)
+ assert_expected_plan(
+ "SELECT a FROM data WHERE a > 0 INTERSECT SELECT a FROM data WHERE a < 5",
+ "LeftSemi Join: left.a = right.a\
+ \n SubqueryAlias: left\
+ \n Aggregate: groupBy=[[data.a]], aggr=[[]]\
+ \n Filter: data.a > Int64(0)\
+ \n TableScan: data projection=[a], partial_filters=[data.a > Int64(0)]\
+ \n SubqueryAlias: right\
+ \n Filter: data.a < Int64(5)\
+ \n TableScan: data projection=[a], partial_filters=[data.a < Int64(5)]",
+ true,
+ )
+ .await
+}
+
+#[tokio::test]
+async fn self_referential_except() -> Result<()> {
+ // Test EXCEPT with the same table on both sides
+ // This previously failed with "Schema contains duplicate qualified field name"
+ // The fix ensures requalify_sides_if_needed is called in intersect_or_except
+ // After roundtrip through Substrait, SubqueryAlias is lost and requalification
+ // produces "left" and "right" aliases
+ // Note: EXCEPT (without ALL) includes DISTINCT, but the outer Aggregate
+ // is optimized away, resulting in just the **LeftAnti** join
+ // (LeftAnti returns rows from left that don't exist in right)
+ assert_expected_plan(
+ "SELECT a FROM data WHERE a > 0 EXCEPT SELECT a FROM data WHERE a < 5",
+ "LeftAnti Join: left.a = right.a\
+ \n SubqueryAlias: left\
+ \n Aggregate: groupBy=[[data.a]], aggr=[[]]\
+ \n Filter: data.a > Int64(0)\
+ \n TableScan: data projection=[a], partial_filters=[data.a > Int64(0)]\
+ \n SubqueryAlias: right\
+ \n Filter: data.a < Int64(5)\
+ \n TableScan: data projection=[a], partial_filters=[data.a < Int64(5)]",
+ true,
+ )
+ .await
+}
+
+#[tokio::test]
+async fn self_referential_intersect_all() -> Result<()> {
+ // Test INTERSECT ALL with the same table on both sides
+ // INTERSECT ALL preserves duplicates and does not include DISTINCT
+ // Uses **LeftSemi** join (returns rows from left that exist in right)
+ // The requalification ensures no duplicate field name errors
+ assert_expected_plan(
+ "SELECT a FROM data WHERE a > 0 INTERSECT ALL SELECT a FROM data WHERE a < 5",
+ "LeftSemi Join: left.a = right.a\
+ \n SubqueryAlias: left\
+ \n Filter: data.a > Int64(0)\
+ \n TableScan: data projection=[a], partial_filters=[data.a > Int64(0)]\
+ \n SubqueryAlias: right\
+ \n Filter: data.a < Int64(5)\
+ \n TableScan: data projection=[a], partial_filters=[data.a < Int64(5)]",
+ true,
+ )
+ .await
+}
+
+#[tokio::test]
+async fn self_referential_except_all() -> Result<()> {
+ // Test EXCEPT ALL with the same table on both sides
+ // EXCEPT ALL preserves duplicates and does not include DISTINCT
+ // Uses **LeftAnti** join (returns rows from left that don't exist in right)
+ // The requalification ensures no duplicate field name errors
+ assert_expected_plan(
+ "SELECT a FROM data WHERE a > 0 EXCEPT ALL SELECT a FROM data WHERE a < 5",
+ "LeftAnti Join: left.a = right.a\
+ \n SubqueryAlias: left\
+ \n Filter: data.a > Int64(0)\
+ \n TableScan: data projection=[a], partial_filters=[data.a > Int64(0)]\
+ \n SubqueryAlias: right\
+ \n Filter: data.a < Int64(5)\
+ \n TableScan: data projection=[a], partial_filters=[data.a < Int64(5)]",
+ true,
+ )
+ .await
+}
+
#[tokio::test]
async fn simple_window_function() -> Result<()> {
roundtrip("SELECT RANK() OVER (PARTITION BY a ORDER BY b), d, sum(b) OVER (PARTITION BY a) FROM data;").await