Apache DataFusion SQL Query Engine
Disallow positional struct casting when field names don’t overlap (#19955)
kosiew8c478e945274
diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rsindex f3f45cfa44..bf2558f313 100644--- a/datafusion/common/src/nested_struct.rs+++ b/datafusion/common/src/nested_struct.rs@@ -31,7 +31,7 @@ use std::{collections::HashSet, sync::Arc};////// ## Field Matching Strategy/// - **By Name**: Source struct fields are matched to target fields by name (case-sensitive)-/// - **By Position**: When there is no name overlap and the field counts match, fields are cast by index+/// - **No Positional Mapping**: Structs with no overlapping field names are rejected/// - **Type Adaptation**: When a matching field is found, it is recursively cast to the target field's type/// - **Missing Fields**: Target fields not present in the source are filled with null values/// - **Extra Fields**: Source fields not present in the target are ignored@@ -67,24 +67,16 @@ fn cast_struct_column(if let Some(source_struct) = source_col.as_any().downcast_ref::<StructArray>() {let source_fields = source_struct.fields();validate_struct_compatibility(source_fields, target_fields)?;- let has_overlap = has_one_of_more_common_fields(source_fields, target_fields);-let mut fields: Vec<Arc<Field>> = Vec::with_capacity(target_fields.len());let mut arrays: Vec<ArrayRef> = Vec::with_capacity(target_fields.len());let num_rows = source_col.len();- // Iterate target fields and pick source child either by name (when fields overlap)- // or by position (when there is no name overlap).- for (index, target_child_field) in target_fields.iter().enumerate() {+ // Iterate target fields and pick source child by name when present.+ for target_child_field in target_fields.iter() {fields.push(Arc::clone(target_child_field));- // Determine the source child column: by name when overlapping names exist,- // otherwise by position.- let source_child_opt: Option<&ArrayRef> = if has_overlap {- source_struct.column_by_name(target_child_field.name())- } else {- Some(source_struct.column(index))- };+ let source_child_opt =+ source_struct.column_by_name(target_child_field.name());match source_child_opt {Some(source_child_col) => {@@ -230,20 +222,11 @@ pub fn validate_struct_compatibility() -> Result<()> {let has_overlap = has_one_of_more_common_fields(source_fields, target_fields);if !has_overlap {- if source_fields.len() != target_fields.len() {- return _plan_err!(- "Cannot cast struct with {} fields to {} fields without name overlap; positional mapping is ambiguous",- source_fields.len(),- target_fields.len()- );- }-- for (source_field, target_field) in source_fields.iter().zip(target_fields.iter())- {- validate_field_compatibility(source_field, target_field)?;- }-- return Ok(());+ return _plan_err!(+ "Cannot cast struct with {} fields to {} fields because there is no field name overlap",+ source_fields.len(),+ target_fields.len()+ );}// Check compatibility for each target field@@ -323,7 +306,11 @@ fn validate_field_compatibility(Ok(())}-fn has_one_of_more_common_fields(+/// Check if two field lists have at least one common field by name.+///+/// This is useful for validating struct compatibility when casting between structs,+/// ensuring that source and target fields have overlapping names.+pub fn has_one_of_more_common_fields(source_fields: &[FieldRef],target_fields: &[FieldRef],) -> bool {@@ -546,7 +533,7 @@ mod tests {}#[test]- fn test_validate_struct_compatibility_positional_no_overlap_mismatch_len() {+ fn test_validate_struct_compatibility_no_overlap_mismatch_len() {let source_fields = vec![arc_field("left", DataType::Int32),arc_field("right", DataType::Int32),@@ -556,7 +543,7 @@ mod tests {let result = validate_struct_compatibility(&source_fields, &target_fields);assert!(result.is_err());let error_msg = result.unwrap_err().to_string();- assert!(error_msg.contains("positional mapping is ambiguous"));+ assert_contains!(error_msg, "no field name overlap");}#[test]@@ -665,21 +652,21 @@ mod tests {}#[test]- fn test_validate_struct_compatibility_positional_with_type_mismatch() {- // Source struct: {left: Struct} - nested struct- let source_fields =- vec![arc_struct_field("left", vec![field("x", DataType::Int32)])];+ fn test_validate_struct_compatibility_no_overlap_equal_len() {+ let source_fields = vec![+ arc_field("left", DataType::Int32),+ arc_field("right", DataType::Utf8),+ ];- // Target struct: {alpha: Int32} (no name overlap, incompatible type at position 0)- let target_fields = vec![arc_field("alpha", DataType::Int32)];+ let target_fields = vec![+ arc_field("alpha", DataType::Int32),+ arc_field("beta", DataType::Utf8),+ ];let result = validate_struct_compatibility(&source_fields, &target_fields);assert!(result.is_err());let error_msg = result.unwrap_err().to_string();- assert_contains!(- error_msg,- "Cannot cast struct field 'alpha' from type Struct(\"x\": Int32) to type Int32"- );+ assert_contains!(error_msg, "no field name overlap");}#[test]@@ -948,7 +935,7 @@ mod tests {}#[test]- fn test_cast_struct_positional_when_no_overlap() {+ fn test_cast_struct_no_overlap_rejected() {let first = Arc::new(Int32Array::from(vec![Some(10), Some(20)])) as ArrayRef;let second =Arc::new(StringArray::from(vec![Some("alpha"), Some("beta")])) as ArrayRef;@@ -964,17 +951,10 @@ mod tests {vec![field("a", DataType::Int64), field("b", DataType::Utf8)],);- let result =- cast_column(&source_col, &target_field, &DEFAULT_CAST_OPTIONS).unwrap();- let struct_array = result.as_any().downcast_ref::<StructArray>().unwrap();-- let a_col = get_column_as!(&struct_array, "a", Int64Array);- assert_eq!(a_col.value(0), 10);- assert_eq!(a_col.value(1), 20);-- let b_col = get_column_as!(&struct_array, "b", StringArray);- assert_eq!(b_col.value(0), "alpha");- assert_eq!(b_col.value(1), "beta");+ let result = cast_column(&source_col, &target_field, &DEFAULT_CAST_OPTIONS);+ assert!(result.is_err());+ let error_msg = result.unwrap_err().to_string();+ assert_contains!(error_msg, "no field name overlap");}#[test]diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rsindex c47316bccc..ce563378a2 100644--- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs+++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs@@ -28,6 +28,7 @@ use std::ops::Not;use std::sync::Arc;use datafusion_common::config::ConfigOptions;+use datafusion_common::nested_struct::has_one_of_more_common_fields;use datafusion_common::{DFSchema, DataFusionError, Result, ScalarValue, exec_datafusion_err, internal_err,};@@ -657,6 +658,11 @@ impl ConstEvaluator {return false;}+ // Skip const-folding when there is no field name overlap+ if !has_one_of_more_common_fields(&source_fields, target_fields) {+ return false;+ }+// Don't const-fold struct casts with empty (0-row) literals// The simplifier uses a 1-row input batch, which causes dimension mismatches// when evaluating 0-row struct literals@@ -5220,7 +5226,7 @@ mod tests {#[test]fn test_struct_cast_different_names_same_count() {// Test struct cast with same field count but different names- // Field count matches; simplification should succeed+ // Field count matches; simplification should be skipped because names do not overlaplet source_fields = Fields::from(vec![Arc::new(Field::new("a", DataType::Int32, true)),@@ -5237,14 +5243,11 @@ mod tests {let simplifier =ExprSimplifier::new(SimplifyContext::default().with_schema(test_schema()));- // The cast should be simplified since field counts match+ // The cast should remain unchanged because there is no name overlaplet result = simplifier.simplify(expr.clone()).unwrap();- // Struct casts with same field count are const-folded to literals- assert!(matches!(result, Expr::Literal(_, _)));- // Ensure the simplifier made a change (not identical to original)- assert_ne!(+ assert_eq!(result, expr,- "Struct cast with different names but same field count should be simplified"+ "Struct cast with different names but same field count should not be simplified");}diff --git a/datafusion/sqllogictest/test_files/joins.slt b/datafusion/sqllogictest/test_files/joins.sltindex df3cad1a14..dd7f4710d9 100644--- a/datafusion/sqllogictest/test_files/joins.slt+++ b/datafusion/sqllogictest/test_files/joins.slt@@ -57,15 +57,15 @@ statement okCREATE TABLE join_t3(s3 struct<id INT>)AS VALUES(NULL),- (struct(1)),- (struct(2));+ ({id: 1}),+ ({id: 2});statement okCREATE TABLE join_t4(s4 struct<id INT>)AS VALUES(NULL),- (struct(2)),- (struct(3));+ ({id: 2}),+ ({id: 3});# Left semi anti joindiff --git a/datafusion/sqllogictest/test_files/struct.slt b/datafusion/sqllogictest/test_files/struct.sltindex 9b1668e58f..e20815a58c 100644--- a/datafusion/sqllogictest/test_files/struct.slt+++ b/datafusion/sqllogictest/test_files/struct.slt@@ -38,9 +38,9 @@ CREATE TABLE struct_values (s1 struct<INT>,s2 struct<a INT,b VARCHAR>) AS VALUES- (struct(1), struct(1, 'string1')),- (struct(2), struct(2, 'string2')),- (struct(3), struct(3, 'string3'))+ (struct(1), struct(1 AS a, 'string1' AS b)),+ (struct(2), struct(2 AS a, 'string2' AS b)),+ (struct(3), struct(3 AS a, 'string3' AS b));query ??@@ -397,7 +397,8 @@ drop view complex_view;# struct with different keys r1 and r2 is not validstatement ok-create table t(a struct<r1 varchar, c int>, b struct<r2 varchar, c float>) as values (struct('red', 1), struct('blue', 2.3));+create table t(a struct<r1 varchar, c int>, b struct<r2 varchar, c float>) as values+ (struct('red' AS r1, 1 AS c), struct('blue' AS r2, 2.3 AS c));# Expect same keys for struct type but got mismatched pair r1,c and r2,cquery error@@ -408,7 +409,8 @@ drop table t;# struct with the same keystatement ok-create table t(a struct<r varchar, c int>, b struct<r varchar, c float>) as values (struct('red', 1), struct('blue', 2.3));+create table t(a struct<r varchar, c int>, b struct<r varchar, c float>) as values+ (struct('red' AS r, 1 AS c), struct('blue' AS r, 2.3 AS c));query Tselect arrow_typeof([a, b]) from t;@@ -442,9 +444,9 @@ CREATE TABLE struct_values (s1 struct(a int, b varchar),s2 struct(a int, b varchar)) AS VALUES- (row(1, 'red'), row(1, 'string1')),- (row(2, 'blue'), row(2, 'string2')),- (row(3, 'green'), row(3, 'string3'))+ ({a: 1, b: 'red'}, {a: 1, b: 'string1'}),+ ({a: 2, b: 'blue'}, {a: 2, b: 'string2'}),+ ({a: 3, b: 'green'}, {a: 3, b: 'string3'});statement ok@@ -452,8 +454,8 @@ drop table struct_values;statement okcreate table t (c1 struct(r varchar, b int), c2 struct(r varchar, b float)) as values (- row('red', 2),- row('blue', 2.3)+ {r: 'red', b: 2},+ {r: 'blue', b: 2.3});query ??@@ -501,9 +503,9 @@ CREATE TABLE t (s1 struct(a int, b varchar),s2 struct(a float, b varchar)) AS VALUES- (row(1, 'red'), row(1.1, 'string1')),- (row(2, 'blue'), row(2.2, 'string2')),- (row(3, 'green'), row(33.2, 'string3'))+ ({a: 1, b: 'red'}, {a: 1.1, b: 'string1'}),+ ({a: 2, b: 'blue'}, {a: 2.2, b: 'string2'}),+ ({a: 3, b: 'green'}, {a: 33.2, b: 'string3'});query ?@@ -528,9 +530,9 @@ CREATE TABLE t (s1 struct(a int, b varchar),s2 struct(a float, b varchar)) AS VALUES- (row(1, 'red'), row(1.1, 'string1')),- (null, row(2.2, 'string2')),- (row(3, 'green'), row(33.2, 'string3'))+ ({a: 1, b: 'red'}, {a: 1.1, b: 'string1'}),+ (null, {a: 2.2, b: 'string2'}),+ ({a: 3, b: 'green'}, {a: 33.2, b: 'string3'});query ?@@ -553,8 +555,8 @@ drop table t;# row() with incorrect order - row() is positional, not name-basedstatement error DataFusion error: Optimizer rule 'simplify_expressions' failed[\s\S]*Arrow error: Cast error: Cannot cast string 'blue' to value of Float32 typecreate table t(a struct(r varchar, c int), b struct(r varchar, c float)) as values- (row('red', 1), row(2.3, 'blue')),- (row('purple', 1), row('green', 2.3));+ ({r: 'red', c: 1}, {r: 2.3, c: 'blue'}),+ ({r: 'purple', c: 1}, {r: 'green', c: 2.3});##################################@@ -568,7 +570,7 @@ select [{r: 'a', c: 1}, {r: 'b', c: 2}];statement ok-create table t(a struct(r varchar, c int), b struct(r varchar, c float)) as values (row('a', 1), row('b', 2.3));+create table t(a struct(r varchar, c int), b struct(r varchar, c float)) as values ({r: 'a', c: 1}, {r: 'b', c: 2.3});query Tselect arrow_typeof([a, b]) from t;@@ -580,7 +582,7 @@ drop table t;statement ok-create table t(a struct(r varchar, c int, g float), b struct(r varchar, c float, g int)) as values (row('a', 1, 2.3), row('b', 2.3, 2));+create table t(a struct(r varchar, c int, g float), b struct(r varchar, c float, g int)) as values ({r: 'a', c: 1, g: 2.3}, {r: 'b', c: 2.3, g: 2});# type of each column should not coerced but preserve as it isquery T@@ -602,7 +604,7 @@ drop table t;# This tests accessing struct fields using the subscript notation with string literalsstatement ok-create table test (struct_field struct(substruct int)) as values (struct(1));+create table test (struct_field struct(substruct int)) as values ({substruct: 1});query ??select *@@ -615,7 +617,7 @@ statement okDROP TABLE test;statement ok-create table test (struct_field struct(substruct struct(subsubstruct int))) as values (struct(struct(1)));+create table test (struct_field struct(substruct struct(subsubstruct int))) as values ({substruct: {subsubstruct: 1}});query ??select *@@ -823,9 +825,9 @@ SELECT CAST({b: 3, a: 4} AS STRUCT(a BIGINT, b INT));----{a: 4, b: 3}-# Test positional casting when there is no name overlap+# Test casting with explicit field namesquery ?-SELECT CAST(struct(1, 'x') AS STRUCT(a INT, b VARCHAR));+SELECT CAST({a: 1, b: 'x'} AS STRUCT(a INT, b VARCHAR));----{a: 1, b: x}@@ -859,9 +861,9 @@ statement okCREATE TABLE struct_reorder_test (data STRUCT(b INT, a VARCHAR)) AS VALUES- (struct(100, 'first')),- (struct(200, 'second')),- (struct(300, 'third'))+ ({b: 100, a: 'first'}),+ ({b: 200, a: 'second'}),+ ({b: 300, a: 'third'});query ?diff --git a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rsindex f78b255526..386ef9dc55 100644--- a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs+++ b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs@@ -1377,7 +1377,7 @@ async fn roundtrip_literal_named_struct() -> Result<()> {assert_snapshot!(plan,@r#"- Projection: Struct({int_field:1,boolean_field:true,string_field:}) AS named_struct(Utf8("int_field"),Int64(1),Utf8("boolean_field"),Boolean(true),Utf8("string_field"),NULL)+ Projection: CAST(Struct({c0:1,c1:true,c2:}) AS Struct("int_field": Int64, "boolean_field": Boolean, "string_field": Utf8View)) AS named_struct(Utf8("int_field"),Int64(1),Utf8("boolean_field"),Boolean(true),Utf8("string_field"),NULL)TableScan: data projection=[]"#);@@ -1397,10 +1397,10 @@ async fn roundtrip_literal_renamed_struct() -> Result<()> {assert_snapshot!(plan,- @r"- Projection: Struct({int_field:1}) AS Struct({c0:1})+ @r#"+ Projection: CAST(Struct({c0:1}) AS Struct("int_field": Int32))TableScan: data projection=[]- "+ "#);Ok(())}diff --git a/docs/source/library-user-guide/upgrading.md b/docs/source/library-user-guide/upgrading.mdindex f5c4fc0921..182f2f0ef9 100644--- a/docs/source/library-user-guide/upgrading.md+++ b/docs/source/library-user-guide/upgrading.md@@ -156,6 +156,97 @@ let context = SimplifyContext::default()See [`SimplifyContext` documentation](https://docs.rs/datafusion-expr/latest/datafusion_expr/simplify/struct.SimplifyContext.html) for more details.+### Struct Casting Now Requires Field Name Overlap++DataFusion's struct casting mechanism previously allowed casting between structs with differing field names if the field counts matched. This "positional fallback" behavior could silently misalign fields and cause data corruption.++**Breaking Change:**++Starting with DataFusion 53.0.0, struct casts now require **at least one overlapping field name** between the source and target structs. Casts without field name overlap are rejected at plan time with a clear error message.++**Who is affected:**++- Applications that cast between structs with no overlapping field names+- Queries that rely on positional struct field mapping (e.g., casting `struct(x, y)` to `struct(a, b)` based solely on position)+- Code that constructs or transforms struct columns programmatically++**Migration guide:**++If you encounter an error like:++```text+Cannot cast struct with 2 fields to 2 fields because there is no field name overlap+```++You must explicitly rename or map fields to ensure at least one field name matches. Here are common patterns:++**Example 1: Source and target field names already match (Name-based casting)**++**Success case (field names align):**++```sql+-- source_col has schema: STRUCT<x INT, y INT>+-- Casting to the same field names succeeds (no-op or type validation only)+SELECT CAST(source_col AS STRUCT<x INT, y INT>) FROM table1;+```++**Example 2: Source and target field names differ (Migration scenario)**++**What fails now (no field name overlap):**++```sql+-- source_col has schema: STRUCT<a INT, b INT>+-- This FAILS because there is no field name overlap:+-- ❌ SELECT CAST(source_col AS STRUCT<x INT, y INT>) FROM table1;+-- Error: Cannot cast struct with 2 fields to 2 fields because there is no field name overlap+```++**Migration options (must align names):**++**Option A: Use struct constructor for explicit field mapping**++```sql+-- source_col has schema: STRUCT<a INT, b INT>+-- Use STRUCT_CONSTRUCT with explicit field names+SELECT STRUCT_CONSTRUCT(+ 'x', source_col.a,+ 'y', source_col.b+) AS renamed_struct FROM table1;+```++**Option B: Rename in the cast target to match source names**++```sql+-- source_col has schema: STRUCT<a INT, b INT>+-- Cast to target with matching field names+SELECT CAST(source_col AS STRUCT<a INT, b INT>) FROM table1;+```++**Example 3: Using struct constructors in Rust API**++If you need to map fields programmatically, build the target struct explicitly:++```rust,ignore+// Build the target struct with explicit field names+let target_struct_type = DataType::Struct(vec![+ FieldRef::new("x", DataType::Int32),+ FieldRef::new("y", DataType::Utf8),+]);++// Use struct constructors rather than casting for field mapping+// This makes the field mapping explicit and unambiguous+// Use struct builders or row constructors that preserve your mapping logic+```++**Why this change:**++1. **Safety:** Field names are now the primary contract for struct compatibility+2. **Explicitness:** Prevents silent data misalignment caused by positional assumptions+3. **Consistency:** Matches DuckDB's behavior and aligns with other SQL engines that enforce name-based matching+4. **Debuggability:** Errors now appear at plan time rather than as silent data corruption++See [Issue #19841](https://github.com/apache/datafusion/issues/19841) and [PR #19955](https://github.com/apache/datafusion/pull/19955) for more details.+### `FilterExec` builder methods deprecatedThe following methods on `FilterExec` have been deprecated in favor of using `FilterExecBuilder`: