Skip to content

Apache DataFusion SQL Query Engine

Improve struct type coercion (#27251)

Matthew Turnerc8265b68723c
diff --git a/datafusion/expr-common/src/type_coercion/binary.rs b/datafusion/expr-common/src/type_coercion/binary.rs
index 8f23d27195..df5d3dc90b 100644
--- a/datafusion/expr-common/src/type_coercion/binary.rs
+++ b/datafusion/expr-common/src/type_coercion/binary.rs
@@ -969,7 +969,8 @@ pub fn try_type_union_resolution_with_struct(
/// [`comparison_coercion`], which prefers numeric types so that ordering and
/// equality follow numeric semantics.
pub fn type_union_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataType> {
- if lhs_type.equals_datatype(rhs_type) {
+ if lhs_type.equals_datatype(rhs_type) && struct_field_names_match(lhs_type, rhs_type)
+ {
return Some(lhs_type.clone());
}
binary_numeric_coercion(lhs_type, rhs_type)
@@ -1018,7 +1019,8 @@ fn comparison_coercion_inner<F>(
where
F: Fn(&DataType, &DataType) -> Option<DataType> + Copy,
{
- if lhs_type.equals_datatype(rhs_type) {
+ if lhs_type.equals_datatype(rhs_type) && struct_field_names_match(lhs_type, rhs_type)
+ {
// same type => equality is possible
return Some(lhs_type.clone());
}
@@ -1433,29 +1435,73 @@ where
return None;
}
- // If the two structs have exactly the same set of field names (possibly in
- // different order), prefer name-based coercion. Otherwise fall back to
- // positional coercion which preserves backward compatibility.
- //
- // Name-based coercion is used in:
- // 1. Array construction: [s1, s2] where s1 and s2 have reordered fields
- // 2. UNION operations: different field orders unified by name
- // 3. VALUES clauses: heterogeneous struct rows unified by field name
- // 4. JOIN conditions: structs with matching field names
- // 5. Window functions: partitions/orders by struct fields
- // 6. Aggregate functions: collecting structs with reordered fields
- //
- // See docs/source/user-guide/sql/struct_coercion.md for detailed examples.
+ // Structs coerce only when they have the same set of field names
+ // (any order). Field names are part of the struct type, and the
+ // cast layer matches fields by name, so no other common type is
+ // executable. See docs/source/user-guide/sql/struct_coercion.md.
if fields_have_same_names(lhs_fields, rhs_fields) {
return coerce_struct_by_name(lhs_fields, rhs_fields, coerce_fn);
}
- coerce_struct_by_position(lhs_fields, rhs_fields, coerce_fn)
+ None
}
_ => None,
}
}
+/// Return true if the field names of every nested struct agree.
+///
+/// Guards the `equals_datatype` fast paths: `equals_datatype` ignores struct
+/// field names, but struct casts match fields by name, so types that differ
+/// only in struct field names must not count as equal. Container labels
+/// (list items, map entries and their key/value fields, union variants) stay
+/// ignored because casts do not depend on them.
+///
+/// Call this only after `equals_datatype` returned true, so the two type
+/// trees have the same shape.
+fn struct_field_names_match(lhs_type: &DataType, rhs_type: &DataType) -> bool {
+ use arrow::datatypes::DataType::*;
+ match (lhs_type, rhs_type) {
+ (Struct(lhs), Struct(rhs)) => {
+ lhs.len() == rhs.len()
+ && lhs.iter().zip(rhs.iter()).all(|(l, r)| {
+ l.name() == r.name()
+ && struct_field_names_match(l.data_type(), r.data_type())
+ })
+ }
+ (List(l), List(r))
+ | (LargeList(l), LargeList(r))
+ | (ListView(l), ListView(r))
+ | (LargeListView(l), LargeListView(r))
+ | (FixedSizeList(l, _), FixedSizeList(r, _)) => {
+ struct_field_names_match(l.data_type(), r.data_type())
+ }
+ // Map entry key/value fields are positional; check only their types.
+ (Map(l, _), Map(r, _)) => match (l.data_type(), r.data_type()) {
+ (Struct(le), Struct(re)) => {
+ le.len() == re.len()
+ && le.iter().zip(re.iter()).all(|(lf, rf)| {
+ struct_field_names_match(lf.data_type(), rf.data_type())
+ })
+ }
+ _ => true,
+ },
+ (Dictionary(_, l), Dictionary(_, r)) => struct_field_names_match(l, r),
+ (RunEndEncoded(_, l), RunEndEncoded(_, r)) => {
+ struct_field_names_match(l.data_type(), r.data_type())
+ }
+ // Union variants are identified by type id, not by listing order.
+ (Union(l, _), Union(r, _)) => l.iter().all(|(lid, lf)| {
+ r.iter()
+ .find(|(rid, _)| *rid == lid)
+ .is_some_and(|(_, rf)| {
+ struct_field_names_match(lf.data_type(), rf.data_type())
+ })
+ }),
+ _ => true,
+ }
+}
+
/// Return true if every left-field name exists in the right fields (and lengths are equal).
///
/// # Assumptions
@@ -1524,36 +1570,6 @@ where
Some(Struct(coerced.into()))
}
-/// Coerce two structs positionally (left-to-right) using `coerce_fn`.
-/// Preserves field names from the left struct and uses combined nullability.
-fn coerce_struct_by_position<F>(
- lhs_fields: &Fields,
- rhs_fields: &Fields,
- coerce_fn: F,
-) -> Option<DataType>
-where
- F: Fn(&DataType, &DataType) -> Option<DataType> + Copy,
-{
- use arrow::datatypes::DataType::*;
-
- // First coerce individual types; fail early if any pair cannot be coerced.
- let coerced_types: Vec<DataType> = lhs_fields
- .iter()
- .zip(rhs_fields.iter())
- .map(|(l, r)| coerce_fn(l.data_type(), r.data_type()))
- .collect::<Option<Vec<DataType>>>()?;
-
- // Build final fields preserving left-side names and combined nullability.
- let orig_pairs = lhs_fields.iter().zip(rhs_fields.iter());
- let fields: Vec<FieldRef> = coerced_types
- .into_iter()
- .zip(orig_pairs)
- .map(|(datatype, (lhs, rhs))| coerce_fields(datatype, lhs, rhs))
- .collect();
-
- Some(Struct(fields.into()))
-}
-
/// returns the result of coercing two fields to a common type
fn coerce_fields(common_type: DataType, lhs: &FieldRef, rhs: &FieldRef) -> FieldRef {
let is_nullable = lhs.is_nullable() || rhs.is_nullable();
@@ -1574,14 +1590,33 @@ where
use arrow::datatypes::DataType::*;
match (lhs_type, rhs_type) {
(Map(lhs_field, lhs_ordered), Map(rhs_field, rhs_ordered)) => {
- struct_coercion(lhs_field.data_type(), rhs_field.data_type(), coerce_fn).map(
- |key_value_type| {
- Map(
- Arc::new((**lhs_field).clone().with_data_type(key_value_type)),
- *lhs_ordered && *rhs_ordered,
- )
- },
- )
+ // A map entry struct has exactly two positional fields (key,
+ // value) with arbitrary names; coerce their types positionally
+ // and keep the left side's names.
+ let (Struct(lhs_entry), Struct(rhs_entry)) =
+ (lhs_field.data_type(), rhs_field.data_type())
+ else {
+ return None;
+ };
+ if lhs_entry.len() != 2 || rhs_entry.len() != 2 {
+ return None;
+ }
+ let entry_fields: Vec<FieldRef> = lhs_entry
+ .iter()
+ .zip(rhs_entry.iter())
+ .map(|(lhs, rhs)| {
+ coerce_fn(lhs.data_type(), rhs.data_type())
+ .map(|common_type| coerce_fields(common_type, lhs, rhs))
+ })
+ .collect::<Option<Vec<_>>>()?;
+ Some(Map(
+ Arc::new(
+ (**lhs_field)
+ .clone()
+ .with_data_type(Struct(entry_fields.into())),
+ ),
+ *lhs_ordered && *rhs_ordered,
+ ))
}
_ => None,
}
@@ -1858,7 +1893,8 @@ fn string_concat_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<Si
}
fn array_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataType> {
- if lhs_type.equals_datatype(rhs_type) {
+ if lhs_type.equals_datatype(rhs_type) && struct_field_names_match(lhs_type, rhs_type)
+ {
Some(lhs_type.to_owned())
} else {
None
diff --git a/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rs b/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rs
index 72f8459fd7..e3af04f840 100644
--- a/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rs
+++ b/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rs
@@ -1232,3 +1232,42 @@ fn test_comparison_coercion_with_session_timezone() {
Some(DataType::Utf8)
);
}
+
+#[test]
+fn union_variants_match_by_type_id_not_position() {
+ use arrow::datatypes::{UnionFields, UnionMode};
+
+ // Exercises `struct_field_names_match`, the guard on the
+ // `equals_datatype` fast paths. Union variants are identified by type id,
+ // not by listing order.
+ fn struct_type(name: &str) -> DataType {
+ DataType::Struct(Fields::from(vec![Field::new(name, DataType::Utf8, true)]))
+ }
+ fn union_type(variants: &[(i8, &str, DataType)]) -> DataType {
+ DataType::Union(
+ UnionFields::try_new(
+ variants.iter().map(|(id, _, _)| *id),
+ variants.iter().map(|(_, name, data_type)| {
+ Field::new(*name, data_type.clone(), true)
+ }),
+ )
+ .unwrap(),
+ UnionMode::Dense,
+ )
+ }
+
+ // Same child types throughout, so `equals_datatype` holds for every pair
+ // below and the guard is what decides.
+ let foo = struct_type("foo");
+ let bar = struct_type("bar");
+
+ let ordered = union_type(&[(0, "v0", foo.clone()), (1, "v1", bar.clone())]);
+ let reordered = union_type(&[(1, "v1", bar.clone()), (0, "v0", foo.clone())]);
+ // Same id -> variant mapping, different listing order: a match.
+ assert!(struct_field_names_match(&ordered, &reordered));
+
+ // Positions line up by name, but the id-matched variants have swapped
+ // struct names: not a match.
+ let swapped_ids = union_type(&[(1, "v0", foo.clone()), (0, "v1", bar)]);
+ assert!(!struct_field_names_match(&ordered, &swapped_ids));
+}
diff --git a/datafusion/sqllogictest/test_files/struct.slt b/datafusion/sqllogictest/test_files/struct.slt
index c0fb9be386..2149ca2563 100644
--- a/datafusion/sqllogictest/test_files/struct.slt
+++ b/datafusion/sqllogictest/test_files/struct.slt
@@ -1481,12 +1481,86 @@ drop table target_struct_insert;
# Test 10: CASE expression with different struct field orders
query II
-select
+select
(case when true then {a: 1, b: 2} else {b: 3, a: 4} end)['a'] as a_val,
(case when true then {a: 1, b: 2} else {b: 3, a: 4} end)['b'] as b_val;
----
1 2
+############################
+# Mismatched Field Names
+############################
+
+# Structs with different field names have no common type, so implicit
+# coercion must fail at plan time.
+# See https://github.com/apache/datafusion/issues/14396
+
+statement ok
+create table mismatched_struct_names as values
+(
+ { 'foo': 'baz' },
+ { 'xxx': arrow_cast('blarg', 'Utf8View') }
+);
+
+# UNION of structs with different field names fails at plan time
+statement error Error during planning: Incompatible inputs for Union: Previous inputs were of type Struct\("foo": Utf8\), but got incompatible type Struct\("xxx": Utf8View\) on column 'column2'
+select column1 from mismatched_struct_names union all select column2 from mismatched_struct_names;
+
+# CASE with mismatched struct branches fails at plan time, even though the
+# mismatched branch is dead code
+statement error Error during planning: Failed to coerce then \(Struct\("foo": Utf8\)\) and else \(Struct\("xxx": Utf8View\)\) to common types in CASE WHEN expression
+select case when 1=2 then column1 else column2 end from mismatched_struct_names;
+
+# Comparison of structs with different field names fails at plan time
+statement error Error during planning: Cannot infer common argument type for comparison operation Struct\("foo": Utf8\) = Struct\("xxx": Utf8View\)
+select column1 = column2 from mismatched_struct_names;
+
+# Comparison with an anonymous struct literal (fields c0, c1) also fails
+statement error Error during planning: Cannot infer common argument type for comparison operation Struct\("foo": Utf8\) = Struct\("c0": Utf8\)
+select column1 = struct('baz') from mismatched_struct_names;
+
+statement ok
+drop table mismatched_struct_names;
+
+# Structs with identical field types but different field names must also
+# fail. These take the "types are already equal" fast path, which ignores
+# struct field names unless guarded.
+
+statement error Error during planning: Failed to coerce then \(Struct\("foo": Utf8\)\) and else \(Struct\("xxx": Utf8\)\) to common types in CASE WHEN expression
+select case when 1=2 then { 'foo': 'baz' } else { 'xxx': 'blarg' } end;
+
+statement error Error during planning: Cannot infer common argument type for comparison operation Struct\("foo": Utf8\) = Struct\("xxx": Utf8\)
+select { 'foo': 'baz' } = { 'xxx': 'blarg' };
+
+# Array containment over structs with different field names also fails.
+# The error differs from the ones above because the analyzer rewrites `@>`
+# to the `array_has_all` function before binary operator coercion runs, so
+# the function signature coercion reports the failure.
+statement error Error during planning: Failed to coerce arguments to satisfy a call to 'array_has_all' function
+select [{ 'foo': 1 }] @> [{ 'xxx': 1 }];
+
+# Same field names in a different order, with a field type that needs
+# coercion (Utf8 vs Utf8View), still unify by name.
+
+statement ok
+create table reordered_struct_names as values
+ ({ 'foo': 'baz', 'bar': 'bux' }),
+ ({ 'bar': 'b', 'foo': arrow_cast('bt', 'Utf8View') });
+
+query ?T
+select column1, arrow_typeof(column1) from reordered_struct_names order by column1['foo'];
+----
+{bar: bux, foo: baz} Struct("bar": Utf8, "foo": Utf8View)
+{bar: b, foo: bt} Struct("bar": Utf8, "foo": Utf8View)
+
+query ?
+select case when 1=2 then { 'foo': 'baz', 'bar': 'bux' } else { 'bar': 'b', 'foo': arrow_cast('bt', 'Utf8View') } end;
+----
+{bar: b, foo: bt}
+
+statement ok
+drop table reordered_struct_names;
+
############################
# JOIN Coercion Tests
############################
diff --git a/docs/source/user-guide/sql/struct_coercion.md b/docs/source/user-guide/sql/struct_coercion.md
index c9cf84040e..b69f066b43 100644
--- a/docs/source/user-guide/sql/struct_coercion.md
+++ b/docs/source/user-guide/sql/struct_coercion.md
@@ -175,6 +175,21 @@ SELECT [
### Limitations
+**Implicit coercion requires identical field name sets.** Two structs coerce to a common type only when they have the same field names (in any order). Structs with different field names have no common type, and the query fails at plan time:
+
+```sql
+-- Different field names: foo vs xxx
+SELECT {foo: 'baz'} UNION ALL SELECT {xxx: 'blarg'};
+-- Error: Incompatible inputs for Union
+
+SELECT CASE WHEN a = 1 THEN {foo: 'baz'} ELSE {xxx: 'blarg'} END FROM t;
+-- Error: Failed to coerce then (...) and else (...) to common types
+-- in CASE WHEN expression
+
+SELECT {foo: 'baz'} = {xxx: 'blarg'};
+-- Error: Cannot infer common argument type for comparison operation
+```
+
**Field count must match exactly.** If structs have different numbers of fields and their field names don't completely overlap, the query will fail:
```sql