Apache DataFusion SQL Query Engine
fix: preserve column qualifiers during type coercion (#27188)
alexanderbianchi3b87602337eb
diff --git a/datafusion/expr/src/expr_rewriter/mod.rs b/datafusion/expr/src/expr_rewriter/mod.rsindex f02b15d04f..e19a03b5af 100644--- a/datafusion/expr/src/expr_rewriter/mod.rs+++ b/datafusion/expr/src/expr_rewriter/mod.rs@@ -282,8 +282,9 @@ fn coerce_exprs_for_schema(let new_type = dst_schema.field(idx).data_type();if new_type != &expr.get_type(src_schema)? {match expr {- Expr::Alias(Alias { expr, name, .. }) => {- Ok(expr.cast_to(new_type, src_schema)?.alias(name))+ Expr::Alias(mut alias) => {+ alias.expr = Box::new(alias.expr.cast_to(new_type, src_schema)?);+ Ok(Expr::Alias(alias))}#[expect(deprecated)]Expr::Wildcard { .. } => Ok(expr),@@ -293,8 +294,12 @@ fn coerce_exprs_for_schema(// tablename being added to it when not explicitly set by the query// (see: https://github.com/apache/datafusion/issues/18818)Expr::Column(ref column) => {+ // Keep the qualifier too: joined columns may share a name.+ let relation = column.relation.clone();let name = column.name().to_owned();- Ok(expr.cast_to(new_type, src_schema)?.alias(name))+ Ok(expr+ .cast_to(new_type, src_schema)?+ .alias_qualified(relation, name))}_ => Ok(expr.cast_to(new_type, src_schema)?),}@@ -515,6 +520,51 @@ mod test {assert_eq!(error, expected);}+ #[test]+ fn coerce_plan_preserves_qualified_column_names() -> Result<()> {+ let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);+ let right =+ crate::logical_plan::table_scan(Some("right"), &schema, None)?.build()?;+ let join = crate::logical_plan::table_scan(Some("left"), &schema, None)?+ .cross_join(right)?+ .build()?;+ let target = DFSchema::try_from(Schema::new(vec![+ Field::new("x", DataType::Int64, false),+ Field::new("y", DataType::Int64, false),+ ]))?;++ // Exercise both adding a projection and coercing an existing projection,+ // including qualified aliases produced by earlier rewrites.+ let columns = vec![col("left.a"), col("right.a")];+ let projected = LogicalPlanBuilder::from(join.clone())+ .project(columns.clone())?+ .build()?;+ let aliased = LogicalPlanBuilder::from(join.clone())+ .project([+ col("left.a").alias_qualified(Some("left"), "a"),+ col("right.a").alias_qualified(Some("right"), "a"),+ ])?+ .build()?;+ for plan in [join, projected, aliased] {+ let coerced = coerce_plan_expr_for_schema(plan, &target)?;+ assert_eq!(+ coerced.schema().columns(),+ vec![+ Column::from_qualified_name("left.a"),+ Column::from_qualified_name("right.a"),+ ]+ );+ assert!(coerced.schema().fields().iter().all(|field| {+ field.name() == "a" && field.data_type() == &DataType::Int64+ }));+ // Both columns remain independently addressable after casting.+ LogicalPlanBuilder::from(coerced)+ .project(columns.clone())?+ .build()?;+ }+ Ok(())+ }+#[test]fn normalize_batch_schema_precedence() -> Result<()> {let schema = Schema::new(vec![diff --git a/datafusion/sqllogictest/test_files/union.slt b/datafusion/sqllogictest/test_files/union.sltindex c44b657ab2..6d1962dbd1 100644--- a/datafusion/sqllogictest/test_files/union.slt+++ b/datafusion/sqllogictest/test_files/union.slt@@ -1145,6 +1145,16 @@ UNION ALL----2+# Coercing same-named columns from different join inputs must retain their qualifiers.+query II rowsort+SELECT 1::BIGINT AS a, 2::BIGINT AS b+UNION ALL+SELECT x.a, y.a+FROM (VALUES (3::INT)) AS x(a) CROSS JOIN (VALUES (4::INT)) AS y(a);+----+1 2+3 4+# Config resetstatement okRESET datafusion.execution.batch_size;diff --git a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rsindex 5ebf2364ab..fe2a8d2a37 100644--- a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs+++ b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs@@ -1463,6 +1463,63 @@ async fn roundtrip_union2() -> Result<()> {.await}+#[tokio::test]+async fn roundtrip_union_join_duplicate_column_names() -> Result<()> {+ use datafusion::arrow::array::{Int64Array, ListBuilder, MapBuilder, StringBuilder};+ use datafusion::arrow::record_batch::RecordBatch;++ // Non-nullable list items expose type coercion after consumption widens the+ // typed NULL's list items to nullable. Both joined columns need that cast.+ let mut tags = MapBuilder::new(+ None,+ StringBuilder::new(),+ ListBuilder::new(StringBuilder::new()).with_field(Field::new(+ "item",+ DataType::Utf8,+ false,+ )),+ );+ tags.keys().append_value("env");+ tags.values().values().append_value("test");+ tags.values().append(true);+ tags.append(true)?;+ let batch = RecordBatch::try_from_iter(vec![+ ("id", Arc::new(Int64Array::from(vec![1])) as ArrayRef),+ ("_tags", Arc::new(tags.finish()) as ArrayRef),+ ])?;+ let ctx = SessionContext::new();+ for table in ["t1", "t2", "t3"] {+ ctx.register_batch(table, batch.clone())?;+ }++ // List-item nullability changes during the round trip, so the schemas differ.+ let plan = generate_plan_from_sql_with_ctx(+ "SELECT NULL AS a, NULL AS b FROM t1 \+ UNION ALL \+ SELECT x._tags AS a, y._tags AS b \+ FROM t2 AS x INNER JOIN t3 AS y ON x.id = y.id",+ false,+ true,+ &ctx,+ )+ .await?;++ assert_snapshot!(+ plan,+ @r#"+ Projection: Map([NULL]) AS a, Map([NULL])__temp__0 AS b+ Union+ Projection: Map([NULL]), Map([NULL]) AS Map([NULL])__temp__0+ TableScan: t1 projection=[]+ Projection: CAST(t2._tags AS Map("entries": non-null Struct("key": non-null Utf8, "value": List(Utf8)), unsorted)) AS Map([NULL]), CAST(t3._tags AS Map("entries": non-null Struct("key": non-null Utf8, "value": List(Utf8)), unsorted)) AS Map([NULL])__temp__0+ Inner Join: t2.id = t3.id+ TableScan: t2 projection=[id, _tags]+ TableScan: t3 projection=[id, _tags]+ "#+ );+ Ok(())+}+#[tokio::test]async fn roundtrip_union_all() -> Result<()> {roundtrip("SELECT a, e FROM data UNION ALL SELECT a, e FROM data").await