Apache DataFusion SQL Query Engine
Improve struct type coercion (#27251)
Matthew Turnerc8265b68723c
diff --git a/datafusion/expr-common/src/type_coercion/binary.rs b/datafusion/expr-common/src/type_coercion/binary.rsindex 8f23d27195..df5d3dc90b 100644--- a/datafusion/expr-common/src/type_coercion/binary.rs+++ b/datafusion/expr-common/src/type_coercion/binary.rs@@ -969,7 +969,8 @@ pub fn try_type_union_resolution_with_struct(/// [`comparison_coercion`], which prefers numeric types so that ordering and/// equality follow numeric semantics.pub fn type_union_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataType> {- if lhs_type.equals_datatype(rhs_type) {+ if lhs_type.equals_datatype(rhs_type) && struct_field_names_match(lhs_type, rhs_type)+ {return Some(lhs_type.clone());}binary_numeric_coercion(lhs_type, rhs_type)@@ -1018,7 +1019,8 @@ fn comparison_coercion_inner<F>(whereF: Fn(&DataType, &DataType) -> Option<DataType> + Copy,{- if lhs_type.equals_datatype(rhs_type) {+ if lhs_type.equals_datatype(rhs_type) && struct_field_names_match(lhs_type, rhs_type)+ {// same type => equality is possiblereturn Some(lhs_type.clone());}@@ -1433,29 +1435,73 @@ wherereturn None;}- // If the two structs have exactly the same set of field names (possibly in- // different order), prefer name-based coercion. Otherwise fall back to- // positional coercion which preserves backward compatibility.- //- // Name-based coercion is used in:- // 1. Array construction: [s1, s2] where s1 and s2 have reordered fields- // 2. UNION operations: different field orders unified by name- // 3. VALUES clauses: heterogeneous struct rows unified by field name- // 4. JOIN conditions: structs with matching field names- // 5. Window functions: partitions/orders by struct fields- // 6. Aggregate functions: collecting structs with reordered fields- //- // See docs/source/user-guide/sql/struct_coercion.md for detailed examples.+ // Structs coerce only when they have the same set of field names+ // (any order). Field names are part of the struct type, and the+ // cast layer matches fields by name, so no other common type is+ // executable. See docs/source/user-guide/sql/struct_coercion.md.if fields_have_same_names(lhs_fields, rhs_fields) {return coerce_struct_by_name(lhs_fields, rhs_fields, coerce_fn);}- coerce_struct_by_position(lhs_fields, rhs_fields, coerce_fn)+ None}_ => None,}}+/// Return true if the field names of every nested struct agree.+///+/// Guards the `equals_datatype` fast paths: `equals_datatype` ignores struct+/// field names, but struct casts match fields by name, so types that differ+/// only in struct field names must not count as equal. Container labels+/// (list items, map entries and their key/value fields, union variants) stay+/// ignored because casts do not depend on them.+///+/// Call this only after `equals_datatype` returned true, so the two type+/// trees have the same shape.+fn struct_field_names_match(lhs_type: &DataType, rhs_type: &DataType) -> bool {+ use arrow::datatypes::DataType::*;+ match (lhs_type, rhs_type) {+ (Struct(lhs), Struct(rhs)) => {+ lhs.len() == rhs.len()+ && lhs.iter().zip(rhs.iter()).all(|(l, r)| {+ l.name() == r.name()+ && struct_field_names_match(l.data_type(), r.data_type())+ })+ }+ (List(l), List(r))+ | (LargeList(l), LargeList(r))+ | (ListView(l), ListView(r))+ | (LargeListView(l), LargeListView(r))+ | (FixedSizeList(l, _), FixedSizeList(r, _)) => {+ struct_field_names_match(l.data_type(), r.data_type())+ }+ // Map entry key/value fields are positional; check only their types.+ (Map(l, _), Map(r, _)) => match (l.data_type(), r.data_type()) {+ (Struct(le), Struct(re)) => {+ le.len() == re.len()+ && le.iter().zip(re.iter()).all(|(lf, rf)| {+ struct_field_names_match(lf.data_type(), rf.data_type())+ })+ }+ _ => true,+ },+ (Dictionary(_, l), Dictionary(_, r)) => struct_field_names_match(l, r),+ (RunEndEncoded(_, l), RunEndEncoded(_, r)) => {+ struct_field_names_match(l.data_type(), r.data_type())+ }+ // Union variants are identified by type id, not by listing order.+ (Union(l, _), Union(r, _)) => l.iter().all(|(lid, lf)| {+ r.iter()+ .find(|(rid, _)| *rid == lid)+ .is_some_and(|(_, rf)| {+ struct_field_names_match(lf.data_type(), rf.data_type())+ })+ }),+ _ => true,+ }+}+/// Return true if every left-field name exists in the right fields (and lengths are equal).////// # Assumptions@@ -1524,36 +1570,6 @@ whereSome(Struct(coerced.into()))}-/// Coerce two structs positionally (left-to-right) using `coerce_fn`.-/// Preserves field names from the left struct and uses combined nullability.-fn coerce_struct_by_position<F>(- lhs_fields: &Fields,- rhs_fields: &Fields,- coerce_fn: F,-) -> Option<DataType>-where- F: Fn(&DataType, &DataType) -> Option<DataType> + Copy,-{- use arrow::datatypes::DataType::*;-- // First coerce individual types; fail early if any pair cannot be coerced.- let coerced_types: Vec<DataType> = lhs_fields- .iter()- .zip(rhs_fields.iter())- .map(|(l, r)| coerce_fn(l.data_type(), r.data_type()))- .collect::<Option<Vec<DataType>>>()?;-- // Build final fields preserving left-side names and combined nullability.- let orig_pairs = lhs_fields.iter().zip(rhs_fields.iter());- let fields: Vec<FieldRef> = coerced_types- .into_iter()- .zip(orig_pairs)- .map(|(datatype, (lhs, rhs))| coerce_fields(datatype, lhs, rhs))- .collect();-- Some(Struct(fields.into()))-}-/// returns the result of coercing two fields to a common typefn coerce_fields(common_type: DataType, lhs: &FieldRef, rhs: &FieldRef) -> FieldRef {let is_nullable = lhs.is_nullable() || rhs.is_nullable();@@ -1574,14 +1590,33 @@ whereuse arrow::datatypes::DataType::*;match (lhs_type, rhs_type) {(Map(lhs_field, lhs_ordered), Map(rhs_field, rhs_ordered)) => {- struct_coercion(lhs_field.data_type(), rhs_field.data_type(), coerce_fn).map(- |key_value_type| {- Map(- Arc::new((**lhs_field).clone().with_data_type(key_value_type)),- *lhs_ordered && *rhs_ordered,- )- },- )+ // A map entry struct has exactly two positional fields (key,+ // value) with arbitrary names; coerce their types positionally+ // and keep the left side's names.+ let (Struct(lhs_entry), Struct(rhs_entry)) =+ (lhs_field.data_type(), rhs_field.data_type())+ else {+ return None;+ };+ if lhs_entry.len() != 2 || rhs_entry.len() != 2 {+ return None;+ }+ let entry_fields: Vec<FieldRef> = lhs_entry+ .iter()+ .zip(rhs_entry.iter())+ .map(|(lhs, rhs)| {+ coerce_fn(lhs.data_type(), rhs.data_type())+ .map(|common_type| coerce_fields(common_type, lhs, rhs))+ })+ .collect::<Option<Vec<_>>>()?;+ Some(Map(+ Arc::new(+ (**lhs_field)+ .clone()+ .with_data_type(Struct(entry_fields.into())),+ ),+ *lhs_ordered && *rhs_ordered,+ ))}_ => None,}@@ -1858,7 +1893,8 @@ fn string_concat_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<Si}fn array_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataType> {- if lhs_type.equals_datatype(rhs_type) {+ if lhs_type.equals_datatype(rhs_type) && struct_field_names_match(lhs_type, rhs_type)+ {Some(lhs_type.to_owned())} else {Nonediff --git a/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rs b/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rsindex 72f8459fd7..e3af04f840 100644--- a/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rs+++ b/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rs@@ -1232,3 +1232,42 @@ fn test_comparison_coercion_with_session_timezone() {Some(DataType::Utf8));}++#[test]+fn union_variants_match_by_type_id_not_position() {+ use arrow::datatypes::{UnionFields, UnionMode};++ // Exercises `struct_field_names_match`, the guard on the+ // `equals_datatype` fast paths. Union variants are identified by type id,+ // not by listing order.+ fn struct_type(name: &str) -> DataType {+ DataType::Struct(Fields::from(vec![Field::new(name, DataType::Utf8, true)]))+ }+ fn union_type(variants: &[(i8, &str, DataType)]) -> DataType {+ DataType::Union(+ UnionFields::try_new(+ variants.iter().map(|(id, _, _)| *id),+ variants.iter().map(|(_, name, data_type)| {+ Field::new(*name, data_type.clone(), true)+ }),+ )+ .unwrap(),+ UnionMode::Dense,+ )+ }++ // Same child types throughout, so `equals_datatype` holds for every pair+ // below and the guard is what decides.+ let foo = struct_type("foo");+ let bar = struct_type("bar");++ let ordered = union_type(&[(0, "v0", foo.clone()), (1, "v1", bar.clone())]);+ let reordered = union_type(&[(1, "v1", bar.clone()), (0, "v0", foo.clone())]);+ // Same id -> variant mapping, different listing order: a match.+ assert!(struct_field_names_match(&ordered, &reordered));++ // Positions line up by name, but the id-matched variants have swapped+ // struct names: not a match.+ let swapped_ids = union_type(&[(1, "v0", foo.clone()), (0, "v1", bar)]);+ assert!(!struct_field_names_match(&ordered, &swapped_ids));+}diff --git a/datafusion/sqllogictest/test_files/struct.slt b/datafusion/sqllogictest/test_files/struct.sltindex c0fb9be386..2149ca2563 100644--- a/datafusion/sqllogictest/test_files/struct.slt+++ b/datafusion/sqllogictest/test_files/struct.slt@@ -1481,12 +1481,86 @@ drop table target_struct_insert;# Test 10: CASE expression with different struct field ordersquery II-select+select(case when true then {a: 1, b: 2} else {b: 3, a: 4} end)['a'] as a_val,(case when true then {a: 1, b: 2} else {b: 3, a: 4} end)['b'] as b_val;----1 2+############################+# Mismatched Field Names+############################++# Structs with different field names have no common type, so implicit+# coercion must fail at plan time.+# See https://github.com/apache/datafusion/issues/14396++statement ok+create table mismatched_struct_names as values+(+ { 'foo': 'baz' },+ { 'xxx': arrow_cast('blarg', 'Utf8View') }+);++# UNION of structs with different field names fails at plan time+statement error Error during planning: Incompatible inputs for Union: Previous inputs were of type Struct\("foo": Utf8\), but got incompatible type Struct\("xxx": Utf8View\) on column 'column2'+select column1 from mismatched_struct_names union all select column2 from mismatched_struct_names;++# CASE with mismatched struct branches fails at plan time, even though the+# mismatched branch is dead code+statement error Error during planning: Failed to coerce then \(Struct\("foo": Utf8\)\) and else \(Struct\("xxx": Utf8View\)\) to common types in CASE WHEN expression+select case when 1=2 then column1 else column2 end from mismatched_struct_names;++# Comparison of structs with different field names fails at plan time+statement error Error during planning: Cannot infer common argument type for comparison operation Struct\("foo": Utf8\) = Struct\("xxx": Utf8View\)+select column1 = column2 from mismatched_struct_names;++# Comparison with an anonymous struct literal (fields c0, c1) also fails+statement error Error during planning: Cannot infer common argument type for comparison operation Struct\("foo": Utf8\) = Struct\("c0": Utf8\)+select column1 = struct('baz') from mismatched_struct_names;++statement ok+drop table mismatched_struct_names;++# Structs with identical field types but different field names must also+# fail. These take the "types are already equal" fast path, which ignores+# struct field names unless guarded.++statement error Error during planning: Failed to coerce then \(Struct\("foo": Utf8\)\) and else \(Struct\("xxx": Utf8\)\) to common types in CASE WHEN expression+select case when 1=2 then { 'foo': 'baz' } else { 'xxx': 'blarg' } end;++statement error Error during planning: Cannot infer common argument type for comparison operation Struct\("foo": Utf8\) = Struct\("xxx": Utf8\)+select { 'foo': 'baz' } = { 'xxx': 'blarg' };++# Array containment over structs with different field names also fails.+# The error differs from the ones above because the analyzer rewrites `@>`+# to the `array_has_all` function before binary operator coercion runs, so+# the function signature coercion reports the failure.+statement error Error during planning: Failed to coerce arguments to satisfy a call to 'array_has_all' function+select [{ 'foo': 1 }] @> [{ 'xxx': 1 }];++# Same field names in a different order, with a field type that needs+# coercion (Utf8 vs Utf8View), still unify by name.++statement ok+create table reordered_struct_names as values+ ({ 'foo': 'baz', 'bar': 'bux' }),+ ({ 'bar': 'b', 'foo': arrow_cast('bt', 'Utf8View') });++query ?T+select column1, arrow_typeof(column1) from reordered_struct_names order by column1['foo'];+----+{bar: bux, foo: baz} Struct("bar": Utf8, "foo": Utf8View)+{bar: b, foo: bt} Struct("bar": Utf8, "foo": Utf8View)++query ?+select case when 1=2 then { 'foo': 'baz', 'bar': 'bux' } else { 'bar': 'b', 'foo': arrow_cast('bt', 'Utf8View') } end;+----+{bar: b, foo: bt}++statement ok+drop table reordered_struct_names;+############################# JOIN Coercion Tests############################diff --git a/docs/source/user-guide/sql/struct_coercion.md b/docs/source/user-guide/sql/struct_coercion.mdindex c9cf84040e..b69f066b43 100644--- a/docs/source/user-guide/sql/struct_coercion.md+++ b/docs/source/user-guide/sql/struct_coercion.md@@ -175,6 +175,21 @@ SELECT [### Limitations+**Implicit coercion requires identical field name sets.** Two structs coerce to a common type only when they have the same field names (in any order). Structs with different field names have no common type, and the query fails at plan time:++```sql+-- Different field names: foo vs xxx+SELECT {foo: 'baz'} UNION ALL SELECT {xxx: 'blarg'};+-- Error: Incompatible inputs for Union++SELECT CASE WHEN a = 1 THEN {foo: 'baz'} ELSE {xxx: 'blarg'} END FROM t;+-- Error: Failed to coerce then (...) and else (...) to common types+-- in CASE WHEN expression++SELECT {foo: 'baz'} = {xxx: 'blarg'};+-- Error: Cannot infer common argument type for comparison operation+```+**Field count must match exactly.** If structs have different numbers of fields and their field names don't completely overlap, the query will fail:```sql