Skip to content

Apache DataFusion SQL Query Engine

fix: preserve column qualifiers during type coercion (#27188)

alexanderbianchi3b87602337eb
diff --git a/datafusion/expr/src/expr_rewriter/mod.rs b/datafusion/expr/src/expr_rewriter/mod.rs
index f02b15d04f..e19a03b5af 100644
--- a/datafusion/expr/src/expr_rewriter/mod.rs
+++ b/datafusion/expr/src/expr_rewriter/mod.rs
@@ -282,8 +282,9 @@ fn coerce_exprs_for_schema(
let new_type = dst_schema.field(idx).data_type();
if new_type != &expr.get_type(src_schema)? {
match expr {
- Expr::Alias(Alias { expr, name, .. }) => {
- Ok(expr.cast_to(new_type, src_schema)?.alias(name))
+ Expr::Alias(mut alias) => {
+ alias.expr = Box::new(alias.expr.cast_to(new_type, src_schema)?);
+ Ok(Expr::Alias(alias))
}
#[expect(deprecated)]
Expr::Wildcard { .. } => Ok(expr),
@@ -293,8 +294,12 @@ fn coerce_exprs_for_schema(
// tablename being added to it when not explicitly set by the query
// (see: https://github.com/apache/datafusion/issues/18818)
Expr::Column(ref column) => {
+ // Keep the qualifier too: joined columns may share a name.
+ let relation = column.relation.clone();
let name = column.name().to_owned();
- Ok(expr.cast_to(new_type, src_schema)?.alias(name))
+ Ok(expr
+ .cast_to(new_type, src_schema)?
+ .alias_qualified(relation, name))
}
_ => Ok(expr.cast_to(new_type, src_schema)?),
}
@@ -515,6 +520,51 @@ mod test {
assert_eq!(error, expected);
}
+ #[test]
+ fn coerce_plan_preserves_qualified_column_names() -> Result<()> {
+ let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
+ let right =
+ crate::logical_plan::table_scan(Some("right"), &schema, None)?.build()?;
+ let join = crate::logical_plan::table_scan(Some("left"), &schema, None)?
+ .cross_join(right)?
+ .build()?;
+ let target = DFSchema::try_from(Schema::new(vec![
+ Field::new("x", DataType::Int64, false),
+ Field::new("y", DataType::Int64, false),
+ ]))?;
+
+ // Exercise both adding a projection and coercing an existing projection,
+ // including qualified aliases produced by earlier rewrites.
+ let columns = vec![col("left.a"), col("right.a")];
+ let projected = LogicalPlanBuilder::from(join.clone())
+ .project(columns.clone())?
+ .build()?;
+ let aliased = LogicalPlanBuilder::from(join.clone())
+ .project([
+ col("left.a").alias_qualified(Some("left"), "a"),
+ col("right.a").alias_qualified(Some("right"), "a"),
+ ])?
+ .build()?;
+ for plan in [join, projected, aliased] {
+ let coerced = coerce_plan_expr_for_schema(plan, &target)?;
+ assert_eq!(
+ coerced.schema().columns(),
+ vec![
+ Column::from_qualified_name("left.a"),
+ Column::from_qualified_name("right.a"),
+ ]
+ );
+ assert!(coerced.schema().fields().iter().all(|field| {
+ field.name() == "a" && field.data_type() == &DataType::Int64
+ }));
+ // Both columns remain independently addressable after casting.
+ LogicalPlanBuilder::from(coerced)
+ .project(columns.clone())?
+ .build()?;
+ }
+ Ok(())
+ }
+
#[test]
fn normalize_batch_schema_precedence() -> Result<()> {
let schema = Schema::new(vec![
diff --git a/datafusion/sqllogictest/test_files/union.slt b/datafusion/sqllogictest/test_files/union.slt
index c44b657ab2..6d1962dbd1 100644
--- a/datafusion/sqllogictest/test_files/union.slt
+++ b/datafusion/sqllogictest/test_files/union.slt
@@ -1145,6 +1145,16 @@ UNION ALL
----
2
+# Coercing same-named columns from different join inputs must retain their qualifiers.
+query II rowsort
+SELECT 1::BIGINT AS a, 2::BIGINT AS b
+UNION ALL
+SELECT x.a, y.a
+FROM (VALUES (3::INT)) AS x(a) CROSS JOIN (VALUES (4::INT)) AS y(a);
+----
+1 2
+3 4
+
# Config reset
statement ok
RESET datafusion.execution.batch_size;
diff --git a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs
index 5ebf2364ab..fe2a8d2a37 100644
--- a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs
+++ b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs
@@ -1463,6 +1463,63 @@ async fn roundtrip_union2() -> Result<()> {
.await
}
+#[tokio::test]
+async fn roundtrip_union_join_duplicate_column_names() -> Result<()> {
+ use datafusion::arrow::array::{Int64Array, ListBuilder, MapBuilder, StringBuilder};
+ use datafusion::arrow::record_batch::RecordBatch;
+
+ // Non-nullable list items expose type coercion after consumption widens the
+ // typed NULL's list items to nullable. Both joined columns need that cast.
+ let mut tags = MapBuilder::new(
+ None,
+ StringBuilder::new(),
+ ListBuilder::new(StringBuilder::new()).with_field(Field::new(
+ "item",
+ DataType::Utf8,
+ false,
+ )),
+ );
+ tags.keys().append_value("env");
+ tags.values().values().append_value("test");
+ tags.values().append(true);
+ tags.append(true)?;
+ let batch = RecordBatch::try_from_iter(vec![
+ ("id", Arc::new(Int64Array::from(vec![1])) as ArrayRef),
+ ("_tags", Arc::new(tags.finish()) as ArrayRef),
+ ])?;
+ let ctx = SessionContext::new();
+ for table in ["t1", "t2", "t3"] {
+ ctx.register_batch(table, batch.clone())?;
+ }
+
+ // List-item nullability changes during the round trip, so the schemas differ.
+ let plan = generate_plan_from_sql_with_ctx(
+ "SELECT NULL AS a, NULL AS b FROM t1 \
+ UNION ALL \
+ SELECT x._tags AS a, y._tags AS b \
+ FROM t2 AS x INNER JOIN t3 AS y ON x.id = y.id",
+ false,
+ true,
+ &ctx,
+ )
+ .await?;
+
+ assert_snapshot!(
+ plan,
+ @r#"
+ Projection: Map([NULL]) AS a, Map([NULL])__temp__0 AS b
+ Union
+ Projection: Map([NULL]), Map([NULL]) AS Map([NULL])__temp__0
+ TableScan: t1 projection=[]
+ Projection: CAST(t2._tags AS Map("entries": non-null Struct("key": non-null Utf8, "value": List(Utf8)), unsorted)) AS Map([NULL]), CAST(t3._tags AS Map("entries": non-null Struct("key": non-null Utf8, "value": List(Utf8)), unsorted)) AS Map([NULL])__temp__0
+ Inner Join: t2.id = t3.id
+ TableScan: t2 projection=[id, _tags]
+ TableScan: t3 projection=[id, _tags]
+ "#
+ );
+ Ok(())
+}
+
#[tokio::test]
async fn roundtrip_union_all() -> Result<()> {
roundtrip("SELECT a, e FROM data UNION ALL SELECT a, e FROM data").await