Apache DataFusion SQL Query Engine
Fix struct casts to align fields by name (prevent positional mis-casts) (#19674)
kosiew0c82adec71b9
diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rsindex 87344914d2..4860393cab 100644--- a/datafusion/common/src/config.rs+++ b/datafusion/common/src/config.rs@@ -2248,7 +2248,7 @@ impl TableOptions {/// Options that control how Parquet files are read, including global options/// that apply to all columns and optional column-specific overrides///-/// Closely tied to [`ParquetWriterOptions`](crate::file_options::parquet_writer::ParquetWriterOptions).+/// Closely tied to `ParquetWriterOptions` (see `crate::file_options::parquet_writer::ParquetWriterOptions` when the "parquet" feature is enabled)./// Properties not included in [`TableParquetOptions`] may not be configurable at the external API/// (e.g. sorting_columns).#[derive(Clone, Default, Debug, PartialEq)]diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rsindex 086d96e852..f3f45cfa44 100644--- a/datafusion/common/src/nested_struct.rs+++ b/datafusion/common/src/nested_struct.rs@@ -19,9 +19,9 @@ use crate::error::{_plan_err, Result};use arrow::{array::{Array, ArrayRef, StructArray, new_null_array},compute::{CastOptions, cast_with_options},- datatypes::{DataType::Struct, Field, FieldRef},+ datatypes::{DataType, DataType::Struct, Field, FieldRef},};-use std::sync::Arc;+use std::{collections::HashSet, sync::Arc};/// Cast a struct column to match target struct fields, handling nested structs recursively.///@@ -31,6 +31,7 @@ use std::sync::Arc;////// ## Field Matching Strategy/// - **By Name**: Source struct fields are matched to target fields by name (case-sensitive)+/// - **By Position**: When there is no name overlap and the field counts match, fields are cast by index/// - **Type Adaptation**: When a matching field is found, it is recursively cast to the target field's type/// - **Missing Fields**: Target fields not present in the source are filled with null values/// - **Extra Fields**: Source fields not present in the target are ignored@@ -54,16 +55,38 @@ fn cast_struct_column(target_fields: &[Arc<Field>],cast_options: &CastOptions,) -> Result<ArrayRef> {+ if source_col.data_type() == &DataType::Null+ || (!source_col.is_empty() && source_col.null_count() == source_col.len())+ {+ return Ok(new_null_array(+ &Struct(target_fields.to_vec().into()),+ source_col.len(),+ ));+ }+if let Some(source_struct) = source_col.as_any().downcast_ref::<StructArray>() {- validate_struct_compatibility(source_struct.fields(), target_fields)?;+ let source_fields = source_struct.fields();+ validate_struct_compatibility(source_fields, target_fields)?;+ let has_overlap = has_one_of_more_common_fields(source_fields, target_fields);let mut fields: Vec<Arc<Field>> = Vec::with_capacity(target_fields.len());let mut arrays: Vec<ArrayRef> = Vec::with_capacity(target_fields.len());let num_rows = source_col.len();- for target_child_field in target_fields {+ // Iterate target fields and pick source child either by name (when fields overlap)+ // or by position (when there is no name overlap).+ for (index, target_child_field) in target_fields.iter().enumerate() {fields.push(Arc::clone(target_child_field));- match source_struct.column_by_name(target_child_field.name()) {++ // Determine the source child column: by name when overlapping names exist,+ // otherwise by position.+ let source_child_opt: Option<&ArrayRef> = if has_overlap {+ source_struct.column_by_name(target_child_field.name())+ } else {+ Some(source_struct.column(index))+ };++ match source_child_opt {Some(source_child_col) => {let adapted_child =cast_column(source_child_col, target_child_field, cast_options)@@ -200,10 +223,29 @@ pub fn cast_column(/// // Target: {a: binary}/// // Result: Err(...) - string cannot cast to binary/// ```+///pub fn validate_struct_compatibility(source_fields: &[FieldRef],target_fields: &[FieldRef],) -> Result<()> {+ let has_overlap = has_one_of_more_common_fields(source_fields, target_fields);+ if !has_overlap {+ if source_fields.len() != target_fields.len() {+ return _plan_err!(+ "Cannot cast struct with {} fields to {} fields without name overlap; positional mapping is ambiguous",+ source_fields.len(),+ target_fields.len()+ );+ }++ for (source_field, target_field) in source_fields.iter().zip(target_fields.iter())+ {+ validate_field_compatibility(source_field, target_field)?;+ }++ return Ok(());+ }+// Check compatibility for each target fieldfor target_field in target_fields {// Look for matching field in source by name@@ -211,53 +253,98 @@ pub fn validate_struct_compatibility(.iter().find(|f| f.name() == target_field.name()){- // Ensure nullability is compatible. It is invalid to cast a nullable- // source field to a non-nullable target field as this may discard- // null values.- if source_field.is_nullable() && !target_field.is_nullable() {+ validate_field_compatibility(source_field, target_field)?;+ } else {+ // Target field is missing from source+ // If it's non-nullable, we cannot fill it with NULL+ if !target_field.is_nullable() {return _plan_err!(- "Cannot cast nullable struct field '{}' to non-nullable field",+ "Cannot cast struct: target field '{}' is non-nullable but missing from source. \+ Cannot fill with NULL.",target_field.name());}- // Check if the matching field types are compatible- match (source_field.data_type(), target_field.data_type()) {- // Recursively validate nested structs- (Struct(source_nested), Struct(target_nested)) => {- validate_struct_compatibility(source_nested, target_nested)?;- }- // For non-struct types, use the existing castability check- _ => {- if !arrow::compute::can_cast_types(- source_field.data_type(),- target_field.data_type(),- ) {- return _plan_err!(- "Cannot cast struct field '{}' from type {} to type {}",- target_field.name(),- source_field.data_type(),- target_field.data_type()- );- }- }- }}- // Missing fields in source are OK - they'll be filled with nulls}// Extra fields in source are OK - they'll be ignoredOk(())}+fn validate_field_compatibility(+ source_field: &Field,+ target_field: &Field,+) -> Result<()> {+ if source_field.data_type() == &DataType::Null {+ // Validate that target allows nulls before returning early.+ // It is invalid to cast a NULL source field to a non-nullable target field.+ if !target_field.is_nullable() {+ return _plan_err!(+ "Cannot cast NULL struct field '{}' to non-nullable field '{}'",+ source_field.name(),+ target_field.name()+ );+ }+ return Ok(());+ }++ // Ensure nullability is compatible. It is invalid to cast a nullable+ // source field to a non-nullable target field as this may discard+ // null values.+ if source_field.is_nullable() && !target_field.is_nullable() {+ return _plan_err!(+ "Cannot cast nullable struct field '{}' to non-nullable field",+ target_field.name()+ );+ }++ // Check if the matching field types are compatible+ match (source_field.data_type(), target_field.data_type()) {+ // Recursively validate nested structs+ (Struct(source_nested), Struct(target_nested)) => {+ validate_struct_compatibility(source_nested, target_nested)?;+ }+ // For non-struct types, use the existing castability check+ _ => {+ if !arrow::compute::can_cast_types(+ source_field.data_type(),+ target_field.data_type(),+ ) {+ return _plan_err!(+ "Cannot cast struct field '{}' from type {} to type {}",+ target_field.name(),+ source_field.data_type(),+ target_field.data_type()+ );+ }+ }+ }++ Ok(())+}++fn has_one_of_more_common_fields(+ source_fields: &[FieldRef],+ target_fields: &[FieldRef],+) -> bool {+ let source_names: HashSet<&str> = source_fields+ .iter()+ .map(|field| field.name().as_str())+ .collect();+ target_fields+ .iter()+ .any(|field| source_names.contains(field.name().as_str()))+}+#[cfg(test)]mod tests {use super::*;- use crate::format::DEFAULT_CAST_OPTIONS;+ use crate::{assert_contains, format::DEFAULT_CAST_OPTIONS};use arrow::{array::{BinaryArray, Int32Array, Int32Builder, Int64Array, ListArray, MapArray,- MapBuilder, StringArray, StringBuilder,+ MapBuilder, NullArray, StringArray, StringBuilder,},buffer::NullBuffer,datatypes::{DataType, Field, FieldRef, Int32Type},@@ -428,11 +515,14 @@ mod tests {#[test]fn test_validate_struct_compatibility_missing_field_in_source() {- // Source struct: {field2: String} (missing field1)- let source_fields = vec![arc_field("field2", DataType::Utf8)];+ // Source struct: {field1: Int32} (missing field2)+ let source_fields = vec![arc_field("field1", DataType::Int32)];- // Target struct: {field1: Int32}- let target_fields = vec![arc_field("field1", DataType::Int32)];+ // Target struct: {field1: Int32, field2: Utf8}+ let target_fields = vec![+ arc_field("field1", DataType::Int32),+ arc_field("field2", DataType::Utf8),+ ];// Should be OK - missing fields will be filled with nullslet result = validate_struct_compatibility(&source_fields, &target_fields);@@ -455,6 +545,20 @@ mod tests {assert!(result.is_ok());}+ #[test]+ fn test_validate_struct_compatibility_positional_no_overlap_mismatch_len() {+ let source_fields = vec![+ arc_field("left", DataType::Int32),+ arc_field("right", DataType::Int32),+ ];+ let target_fields = vec![arc_field("alpha", DataType::Int32)];++ let result = validate_struct_compatibility(&source_fields, &target_fields);+ assert!(result.is_err());+ let error_msg = result.unwrap_err().to_string();+ assert!(error_msg.contains("positional mapping is ambiguous"));+ }+#[test]fn test_cast_struct_parent_nulls_retained() {let a_array = Arc::new(Int32Array::from(vec![Some(1), Some(2)])) as ArrayRef;@@ -525,6 +629,117 @@ mod tests {assert!(error_msg.contains("non-nullable"));}+ #[test]+ fn test_validate_struct_compatibility_by_name() {+ // Source struct: {field1: Int32, field2: String}+ let source_fields = vec![+ arc_field("field1", DataType::Int32),+ arc_field("field2", DataType::Utf8),+ ];++ // Target struct: {field2: String, field1: Int64}+ let target_fields = vec![+ arc_field("field2", DataType::Utf8),+ arc_field("field1", DataType::Int64),+ ];++ let result = validate_struct_compatibility(&source_fields, &target_fields);+ assert!(result.is_ok());+ }++ #[test]+ fn test_validate_struct_compatibility_by_name_with_type_mismatch() {+ // Source struct: {field1: Binary}+ let source_fields = vec![arc_field("field1", DataType::Binary)];++ // Target struct: {field1: Int32} (incompatible type)+ let target_fields = vec![arc_field("field1", DataType::Int32)];++ let result = validate_struct_compatibility(&source_fields, &target_fields);+ assert!(result.is_err());+ let error_msg = result.unwrap_err().to_string();+ assert_contains!(+ error_msg,+ "Cannot cast struct field 'field1' from type Binary to type Int32"+ );+ }++ #[test]+ fn test_validate_struct_compatibility_positional_with_type_mismatch() {+ // Source struct: {left: Struct} - nested struct+ let source_fields =+ vec![arc_struct_field("left", vec![field("x", DataType::Int32)])];++ // Target struct: {alpha: Int32} (no name overlap, incompatible type at position 0)+ let target_fields = vec![arc_field("alpha", DataType::Int32)];++ let result = validate_struct_compatibility(&source_fields, &target_fields);+ assert!(result.is_err());+ let error_msg = result.unwrap_err().to_string();+ assert_contains!(+ error_msg,+ "Cannot cast struct field 'alpha' from type Struct(\"x\": Int32) to type Int32"+ );+ }++ #[test]+ fn test_validate_struct_compatibility_mixed_name_overlap() {+ // Source struct: {a: Int32, b: String, extra: Boolean}+ let source_fields = vec![+ arc_field("a", DataType::Int32),+ arc_field("b", DataType::Utf8),+ arc_field("extra", DataType::Boolean),+ ];++ // Target struct: {b: String, a: Int64, c: Float32}+ // Name overlap with a and b, missing c (nullable)+ let target_fields = vec![+ arc_field("b", DataType::Utf8),+ arc_field("a", DataType::Int64),+ arc_field("c", DataType::Float32),+ ];++ let result = validate_struct_compatibility(&source_fields, &target_fields);+ assert!(result.is_ok());+ }++ #[test]+ fn test_validate_struct_compatibility_by_name_missing_required_field() {+ // Source struct: {field1: Int32} (missing field2)+ let source_fields = vec![arc_field("field1", DataType::Int32)];++ // Target struct: {field1: Int32, field2: Int32 non-nullable}+ let target_fields = vec![+ arc_field("field1", DataType::Int32),+ Arc::new(non_null_field("field2", DataType::Int32)),+ ];++ let result = validate_struct_compatibility(&source_fields, &target_fields);+ assert!(result.is_err());+ let error_msg = result.unwrap_err().to_string();+ assert_contains!(+ error_msg,+ "Cannot cast struct: target field 'field2' is non-nullable but missing from source. Cannot fill with NULL."+ );+ }++ #[test]+ fn test_validate_struct_compatibility_partial_name_overlap_with_count_mismatch() {+ // Source struct: {a: Int32} (only one field)+ let source_fields = vec![arc_field("a", DataType::Int32)];++ // Target struct: {a: Int32, b: String} (two fields, but 'a' overlaps)+ let target_fields = vec![+ arc_field("a", DataType::Int32),+ arc_field("b", DataType::Utf8),+ ];++ // This should succeed - partial overlap means by-name mapping+ // and missing field 'b' is nullable+ let result = validate_struct_compatibility(&source_fields, &target_fields);+ assert!(result.is_ok());+ }+#[test]fn test_cast_nested_struct_with_extra_and_missing_fields() {// Source inner struct has fields a, b, extra@@ -585,6 +800,33 @@ mod tests {assert!(missing.is_null(1));}+ #[test]+ fn test_cast_null_struct_field_to_nested_struct() {+ let null_inner = Arc::new(NullArray::new(2)) as ArrayRef;+ let source_struct = StructArray::from(vec![(+ arc_field("inner", DataType::Null),+ Arc::clone(&null_inner),+ )]);+ let source_col = Arc::new(source_struct) as ArrayRef;++ let target_field = struct_field(+ "outer",+ vec![struct_field("inner", vec![field("a", DataType::Int32)])],+ );++ let result =+ cast_column(&source_col, &target_field, &DEFAULT_CAST_OPTIONS).unwrap();+ let outer = result.as_any().downcast_ref::<StructArray>().unwrap();+ let inner = get_column_as!(&outer, "inner", StructArray);+ assert_eq!(inner.len(), 2);+ assert!(inner.is_null(0));+ assert!(inner.is_null(1));++ let inner_a = get_column_as!(inner, "a", Int32Array);+ assert!(inner_a.is_null(0));+ assert!(inner_a.is_null(1));+ }+#[test]fn test_cast_struct_with_array_and_map_fields() {// Array field with second row null@@ -704,4 +946,88 @@ mod tests {assert_eq!(a_col.value(0), 1);assert_eq!(a_col.value(1), 2);}++ #[test]+ fn test_cast_struct_positional_when_no_overlap() {+ let first = Arc::new(Int32Array::from(vec![Some(10), Some(20)])) as ArrayRef;+ let second =+ Arc::new(StringArray::from(vec![Some("alpha"), Some("beta")])) as ArrayRef;++ let source_struct = StructArray::from(vec![+ (arc_field("left", DataType::Int32), first),+ (arc_field("right", DataType::Utf8), second),+ ]);+ let source_col = Arc::new(source_struct) as ArrayRef;++ let target_field = struct_field(+ "s",+ vec![field("a", DataType::Int64), field("b", DataType::Utf8)],+ );++ let result =+ cast_column(&source_col, &target_field, &DEFAULT_CAST_OPTIONS).unwrap();+ let struct_array = result.as_any().downcast_ref::<StructArray>().unwrap();++ let a_col = get_column_as!(&struct_array, "a", Int64Array);+ assert_eq!(a_col.value(0), 10);+ assert_eq!(a_col.value(1), 20);++ let b_col = get_column_as!(&struct_array, "b", StringArray);+ assert_eq!(b_col.value(0), "alpha");+ assert_eq!(b_col.value(1), "beta");+ }++ #[test]+ fn test_cast_struct_missing_non_nullable_field_fails() {+ // Source has only field 'a'+ let a = Arc::new(Int32Array::from(vec![Some(1), Some(2)])) as ArrayRef;+ let source_struct = StructArray::from(vec![(arc_field("a", DataType::Int32), a)]);+ let source_col = Arc::new(source_struct) as ArrayRef;++ // Target has fields 'a' (nullable) and 'b' (non-nullable)+ let target_field = struct_field(+ "s",+ vec![+ field("a", DataType::Int32),+ non_null_field("b", DataType::Int32),+ ],+ );++ // Should fail because 'b' is non-nullable but missing from source+ let result = cast_column(&source_col, &target_field, &DEFAULT_CAST_OPTIONS);+ assert!(result.is_err());+ let err = result.unwrap_err();+ assert!(+ err.to_string()+ .contains("target field 'b' is non-nullable but missing from source"),+ "Unexpected error: {err}"+ );+ }++ #[test]+ fn test_cast_struct_missing_nullable_field_succeeds() {+ // Source has only field 'a'+ let a = Arc::new(Int32Array::from(vec![Some(1), Some(2)])) as ArrayRef;+ let source_struct = StructArray::from(vec![(arc_field("a", DataType::Int32), a)]);+ let source_col = Arc::new(source_struct) as ArrayRef;++ // Target has fields 'a' and 'b' (both nullable)+ let target_field = struct_field(+ "s",+ vec![field("a", DataType::Int32), field("b", DataType::Int32)],+ );++ // Should succeed - 'b' is nullable so can be filled with NULL+ let result =+ cast_column(&source_col, &target_field, &DEFAULT_CAST_OPTIONS).unwrap();+ let struct_array = result.as_any().downcast_ref::<StructArray>().unwrap();++ let a_col = get_column_as!(&struct_array, "a", Int32Array);+ assert_eq!(a_col.value(0), 1);+ assert_eq!(a_col.value(1), 2);++ let b_col = get_column_as!(&struct_array, "b", Int32Array);+ assert!(b_col.is_null(0));+ assert!(b_col.is_null(1));+ }}diff --git a/datafusion/common/src/scalar/mod.rs b/datafusion/common/src/scalar/mod.rsindex 495f8c3b3f..064091971c 100644--- a/datafusion/common/src/scalar/mod.rs+++ b/datafusion/common/src/scalar/mod.rs@@ -3688,7 +3688,23 @@ impl ScalarValue {}let scalar_array = self.to_array()?;- let cast_arr = cast_with_options(&scalar_array, target_type, cast_options)?;++ // For struct types, use name-based casting logic that matches fields by name+ // and recursively casts nested structs. The field name wrapper is arbitrary+ // since cast_column only uses the DataType::Struct field definitions inside.+ let cast_arr = match target_type {+ DataType::Struct(_) => {+ // Field name is unused; only the struct's inner field names matter+ let target_field = Field::new("_", target_type.clone(), true);+ crate::nested_struct::cast_column(+ &scalar_array,+ &target_field,+ cast_options,+ )?+ }+ _ => cast_with_options(&scalar_array, target_type, cast_options)?,+ };+ScalarValue::try_from_array(&cast_arr, 0)}diff --git a/datafusion/expr-common/src/columnar_value.rs b/datafusion/expr-common/src/columnar_value.rsindex 99c21d4abd..1aa42470a1 100644--- a/datafusion/expr-common/src/columnar_value.rs+++ b/datafusion/expr-common/src/columnar_value.rs@@ -20,7 +20,7 @@use arrow::{array::{Array, ArrayRef, Date32Array, Date64Array, NullArray},compute::{CastOptions, kernels, max, min},- datatypes::DataType,+ datatypes::{DataType, Field},util::pretty::pretty_format_columns,};use datafusion_common::internal_datafusion_err;@@ -274,7 +274,17 @@ impl ColumnarValue {Ok(args)}- /// Cast's this [ColumnarValue] to the specified `DataType`+ /// Cast this [ColumnarValue] to the specified `DataType`+ ///+ /// # Struct Casting Behavior+ ///+ /// When casting struct types, fields are matched **by name** rather than position:+ /// - Source fields are matched to target fields using case-sensitive name comparison+ /// - Fields are reordered to match the target schema+ /// - Missing target fields are filled with null arrays+ /// - Extra source fields are ignored+ ///+ /// For non-struct types, uses Arrow's standard positional casting.pub fn cast_to(&self,cast_type: &DataType,@@ -283,12 +293,8 @@ impl ColumnarValue {let cast_options = cast_options.cloned().unwrap_or(DEFAULT_CAST_OPTIONS);match self {ColumnarValue::Array(array) => {- ensure_date_array_timestamp_bounds(array, cast_type)?;- Ok(ColumnarValue::Array(kernels::cast::cast_with_options(- array,- cast_type,- &cast_options,- )?))+ let casted = cast_array_by_name(array, cast_type, &cast_options)?;+ Ok(ColumnarValue::Array(casted))}ColumnarValue::Scalar(scalar) => Ok(ColumnarValue::Scalar(scalar.cast_to_with_options(cast_type, &cast_options)?,@@ -297,6 +303,37 @@ impl ColumnarValue {}}+fn cast_array_by_name(+ array: &ArrayRef,+ cast_type: &DataType,+ cast_options: &CastOptions<'static>,+) -> Result<ArrayRef> {+ // If types are already equal, no cast needed+ if array.data_type() == cast_type {+ return Ok(Arc::clone(array));+ }++ match cast_type {+ DataType::Struct(_) => {+ // Field name is unused; only the struct's inner field names matter+ let target_field = Field::new("_", cast_type.clone(), true);+ datafusion_common::nested_struct::cast_column(+ array,+ &target_field,+ cast_options,+ )+ }+ _ => {+ ensure_date_array_timestamp_bounds(array, cast_type)?;+ Ok(kernels::cast::cast_with_options(+ array,+ cast_type,+ cast_options,+ )?)+ }+ }+}+fn ensure_date_array_timestamp_bounds(array: &ArrayRef,cast_type: &DataType,@@ -378,8 +415,8 @@ impl fmt::Display for ColumnarValue {mod tests {use super::*;use arrow::{- array::{Date64Array, Int32Array},- datatypes::TimeUnit,+ array::{Date64Array, Int32Array, StructArray},+ datatypes::{Field, Fields, TimeUnit},};#[test]@@ -553,6 +590,102 @@ mod tests {);}+ #[test]+ fn cast_struct_by_field_name() {+ let source_fields = Fields::from(vec![+ Field::new("b", DataType::Int32, true),+ Field::new("a", DataType::Int32, true),+ ]);++ let target_fields = Fields::from(vec![+ Field::new("a", DataType::Int32, true),+ Field::new("b", DataType::Int32, true),+ ]);++ let struct_array = StructArray::new(+ source_fields,+ vec![+ Arc::new(Int32Array::from(vec![Some(3)])),+ Arc::new(Int32Array::from(vec![Some(4)])),+ ],+ None,+ );++ let value = ColumnarValue::Array(Arc::new(struct_array));+ let casted = value+ .cast_to(&DataType::Struct(target_fields.clone()), None)+ .expect("struct cast should succeed");++ let ColumnarValue::Array(arr) = casted else {+ panic!("expected array after cast");+ };++ let struct_array = arr+ .as_any()+ .downcast_ref::<StructArray>()+ .expect("expected StructArray");++ let field_a = struct_array+ .column_by_name("a")+ .expect("expected field a in cast result");+ let field_b = struct_array+ .column_by_name("b")+ .expect("expected field b in cast result");++ assert_eq!(+ field_a+ .as_any()+ .downcast_ref::<Int32Array>()+ .expect("expected Int32 array")+ .value(0),+ 4+ );+ assert_eq!(+ field_b+ .as_any()+ .downcast_ref::<Int32Array>()+ .expect("expected Int32 array")+ .value(0),+ 3+ );+ }++ #[test]+ fn cast_struct_missing_field_inserts_nulls() {+ let source_fields = Fields::from(vec![Field::new("a", DataType::Int32, true)]);++ let target_fields = Fields::from(vec![+ Field::new("a", DataType::Int32, true),+ Field::new("b", DataType::Int32, true),+ ]);++ let struct_array = StructArray::new(+ source_fields,+ vec![Arc::new(Int32Array::from(vec![Some(5)]))],+ None,+ );++ let value = ColumnarValue::Array(Arc::new(struct_array));+ let casted = value+ .cast_to(&DataType::Struct(target_fields.clone()), None)+ .expect("struct cast should succeed");++ let ColumnarValue::Array(arr) = casted else {+ panic!("expected array after cast");+ };++ let struct_array = arr+ .as_any()+ .downcast_ref::<StructArray>()+ .expect("expected StructArray");++ let field_b = struct_array+ .column_by_name("b")+ .expect("expected missing field to be added");++ assert!(field_b.is_null(0));+ }+#[test]fn cast_date64_array_to_timestamp_overflow() {let overflow_value = i64::MAX / 1_000_000 + 1;diff --git a/datafusion/expr-common/src/type_coercion/binary.rs b/datafusion/expr-common/src/type_coercion/binary.rsindex c9b39eacef..427ebc5980 100644--- a/datafusion/expr-common/src/type_coercion/binary.rs+++ b/datafusion/expr-common/src/type_coercion/binary.rs@@ -17,6 +17,7 @@//! Coercion rules for matching argument types for binary operators+use std::collections::HashMap;use std::collections::HashSet;use std::sync::Arc;@@ -1236,30 +1237,123 @@ fn coerce_numeric_type_to_decimal256(numeric_type: &DataType) -> Option<DataTypefn struct_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataType> {use arrow::datatypes::DataType::*;+match (lhs_type, rhs_type) {(Struct(lhs_fields), Struct(rhs_fields)) => {+ // Field count must match for coercionif lhs_fields.len() != rhs_fields.len() {return None;}- let coerced_types = std::iter::zip(lhs_fields.iter(), rhs_fields.iter())- .map(|(lhs, rhs)| comparison_coercion(lhs.data_type(), rhs.data_type()))- .collect::<Option<Vec<DataType>>>()?;-- // preserve the field name and nullability- let orig_fields = std::iter::zip(lhs_fields.iter(), rhs_fields.iter());+ // If the two structs have exactly the same set of field names (possibly in+ // different order), prefer name-based coercion. Otherwise fall back to+ // positional coercion which preserves backward compatibility.+ //+ // Name-based coercion is used in:+ // 1. Array construction: [s1, s2] where s1 and s2 have reordered fields+ // 2. UNION operations: different field orders unified by name+ // 3. VALUES clauses: heterogeneous struct rows unified by field name+ // 4. JOIN conditions: structs with matching field names+ // 5. Window functions: partitions/orders by struct fields+ // 6. Aggregate functions: collecting structs with reordered fields+ //+ // See docs/source/user-guide/sql/struct_coercion.md for detailed examples.+ if fields_have_same_names(lhs_fields, rhs_fields) {+ return coerce_struct_by_name(lhs_fields, rhs_fields);+ }- let fields: Vec<FieldRef> = coerced_types- .into_iter()- .zip(orig_fields)- .map(|(datatype, (lhs, rhs))| coerce_fields(datatype, lhs, rhs))- .collect();- Some(Struct(fields.into()))+ coerce_struct_by_position(lhs_fields, rhs_fields)}_ => None,}}+/// Return true if every left-field name exists in the right fields (and lengths are equal).+///+/// # Assumptions+/// **This function assumes field names within each struct are unique.** This assumption is safe+/// because field name uniqueness is enforced at multiple levels:+/// - **Arrow level:** `StructType` construction enforces unique field names at the schema level+/// - **DataFusion level:** SQL parser rejects duplicate field names in `CREATE TABLE` and struct type definitions+/// - **Runtime level:** `StructArray::try_new()` validates field uniqueness+///+/// Therefore, we don't need to handle degenerate cases like:+/// - `struct<c1 int> -> struct<c1 int, c1 int>` (target has duplicate field names)+/// - `struct<c1 int, c1 int> -> struct<c1 int>` (source has duplicate field names)+fn fields_have_same_names(lhs_fields: &Fields, rhs_fields: &Fields) -> bool {+ // Debug assertions: field names should be unique within each struct+ #[cfg(debug_assertions)]+ {+ let lhs_names: HashSet<_> = lhs_fields.iter().map(|f| f.name()).collect();+ assert_eq!(+ lhs_names.len(),+ lhs_fields.len(),+ "Struct has duplicate field names (should be caught by Arrow schema validation)"+ );++ let rhs_names_check: HashSet<_> = rhs_fields.iter().map(|f| f.name()).collect();+ assert_eq!(+ rhs_names_check.len(),+ rhs_fields.len(),+ "Struct has duplicate field names (should be caught by Arrow schema validation)"+ );+ }++ let rhs_names: HashSet<&str> = rhs_fields.iter().map(|f| f.name().as_str()).collect();+ lhs_fields+ .iter()+ .all(|lf| rhs_names.contains(lf.name().as_str()))+}++/// Coerce two structs by matching fields by name. Assumes the name-sets match.+fn coerce_struct_by_name(lhs_fields: &Fields, rhs_fields: &Fields) -> Option<DataType> {+ use arrow::datatypes::DataType::*;++ let rhs_by_name: HashMap<&str, &FieldRef> =+ rhs_fields.iter().map(|f| (f.name().as_str(), f)).collect();++ let mut coerced: Vec<FieldRef> = Vec::with_capacity(lhs_fields.len());++ for lhs in lhs_fields.iter() {+ let rhs = rhs_by_name.get(lhs.name().as_str()).unwrap(); // safe: caller ensured names match+ let coerced_type = comparison_coercion(lhs.data_type(), rhs.data_type())?;+ let is_nullable = lhs.is_nullable() || rhs.is_nullable();+ coerced.push(Arc::new(Field::new(+ lhs.name().clone(),+ coerced_type,+ is_nullable,+ )));+ }++ Some(Struct(coerced.into()))+}++/// Coerce two structs positionally (left-to-right). This preserves field names from+/// the left struct and uses the combined nullability.+fn coerce_struct_by_position(+ lhs_fields: &Fields,+ rhs_fields: &Fields,+) -> Option<DataType> {+ use arrow::datatypes::DataType::*;++ // First coerce individual types; fail early if any pair cannot be coerced.+ let coerced_types: Vec<DataType> = lhs_fields+ .iter()+ .zip(rhs_fields.iter())+ .map(|(l, r)| comparison_coercion(l.data_type(), r.data_type()))+ .collect::<Option<Vec<DataType>>>()?;++ // Build final fields preserving left-side names and combined nullability.+ let orig_pairs = lhs_fields.iter().zip(rhs_fields.iter());+ let fields: Vec<FieldRef> = coerced_types+ .into_iter()+ .zip(orig_pairs)+ .map(|(datatype, (lhs, rhs))| coerce_fields(datatype, lhs, rhs))+ .collect();++ Some(Struct(fields.into()))+}+/// returns the result of coercing two fields to a common typefn coerce_fields(common_type: DataType, lhs: &FieldRef, rhs: &FieldRef) -> FieldRef {let is_nullable = lhs.is_nullable() || rhs.is_nullable();diff --git a/datafusion/expr/src/expr_schema.rs b/datafusion/expr/src/expr_schema.rsindex f1c77eb525..76cbf7b4ac 100644--- a/datafusion/expr/src/expr_schema.rs+++ b/datafusion/expr/src/expr_schema.rs@@ -662,7 +662,16 @@ impl ExprSchemable for Expr {// like all of the binary expressions below. Perhaps Expr should track the// type of the expression?- if can_cast_types(&this_type, cast_to_type) {+ // Special handling for struct-to-struct casts with name-based field matching+ let can_cast = match (&this_type, cast_to_type) {+ (DataType::Struct(_), DataType::Struct(_)) => {+ // Always allow struct-to-struct casts; field matching happens at runtime+ true+ }+ _ => can_cast_types(&this_type, cast_to_type),+ };++ if can_cast {match self {Expr::ScalarSubquery(subquery) => {Ok(Expr::ScalarSubquery(cast_subquery(subquery, cast_to_type)?))diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rsindex 7bbb7e79d1..c47316bccc 100644--- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs+++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs@@ -18,7 +18,7 @@//! Expression simplification APIuse arrow::{- array::{AsArray, new_null_array},+ array::{Array, AsArray, new_null_array},datatypes::{DataType, Field, Schema},record_batch::RecordBatch,};@@ -38,8 +38,8 @@ use datafusion_common::{tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRewriter},};use datafusion_expr::{- BinaryExpr, Case, ColumnarValue, Expr, Like, Operator, Volatility, and,- binary::BinaryTypeCoercer, lit, or, preimage::PreimageResult,+ BinaryExpr, Case, ColumnarValue, Expr, ExprSchemable, Like, Operator, Volatility,+ and, binary::BinaryTypeCoercer, lit, or, preimage::PreimageResult,};use datafusion_expr::{Cast, TryCast, simplify::ExprSimplifyResult};use datafusion_expr::{expr::ScalarFunction, interval_arithmetic::NullableInterval};@@ -645,6 +645,30 @@ impl ConstEvaluator {Expr::ScalarFunction(ScalarFunction { func, .. }) => {Self::volatility_ok(func.signature().volatility)}+ Expr::Cast(Cast { expr, data_type })+ | Expr::TryCast(TryCast { expr, data_type }) => {+ if let (+ Ok(DataType::Struct(source_fields)),+ DataType::Struct(target_fields),+ ) = (expr.get_type(&DFSchema::empty()), data_type)+ {+ // Don't const-fold struct casts with different field counts+ if source_fields.len() != target_fields.len() {+ return false;+ }++ // Don't const-fold struct casts with empty (0-row) literals+ // The simplifier uses a 1-row input batch, which causes dimension mismatches+ // when evaluating 0-row struct literals+ if let Expr::Literal(ScalarValue::Struct(struct_array), _) =+ expr.as_ref()+ && struct_array.len() == 0+ {+ return false;+ }+ }+ true+ }Expr::Literal(_, _)| Expr::Alias(..)| Expr::Unnest(_)@@ -663,8 +687,6 @@ impl ConstEvaluator {| Expr::Like { .. }| Expr::SimilarTo { .. }| Expr::Case(_)- | Expr::Cast { .. }- | Expr::TryCast { .. }| Expr::InList { .. } => true,}}@@ -2243,7 +2265,10 @@ mod tests {use super::*;use crate::simplify_expressions::SimplifyContext;use crate::test::test_table_scan_with_name;- use arrow::datatypes::FieldRef;+ use arrow::{+ array::{Int32Array, StructArray},+ datatypes::{FieldRef, Fields},+ };use datafusion_common::{DFSchemaRef, ToDFSchema, assert_contains};use datafusion_expr::{expr::WindowFunction,@@ -5109,4 +5134,159 @@ mod tests {else_expr: None,})}++ // --------------------------------+ // --- Struct Cast Tests -----+ // --------------------------------++ /// Helper to create a `Struct` literal cast expression from `source_fields` and `target_fields`.+ fn make_struct_cast_expr(source_fields: Fields, target_fields: Fields) -> Expr {+ // Create 1-row struct array (not 0-row) so it can be evaluated by simplifier+ let arrays: Vec<Arc<dyn Array>> = vec![+ Arc::new(Int32Array::from(vec![Some(1)])),+ Arc::new(Int32Array::from(vec![Some(2)])),+ ];+ let struct_array = StructArray::try_new(source_fields, arrays, None).unwrap();++ Expr::Cast(Cast::new(+ Box::new(Expr::Literal(+ ScalarValue::Struct(Arc::new(struct_array)),+ None,+ )),+ DataType::Struct(target_fields),+ ))+ }++ #[test]+ fn test_struct_cast_different_field_counts_not_foldable() {+ // Test that struct casts with different field counts are NOT marked as foldable+ // When field counts differ, const-folding should not be attempted++ let source_fields = Fields::from(vec![+ Arc::new(Field::new("a", DataType::Int32, true)),+ Arc::new(Field::new("b", DataType::Int32, true)),+ ]);++ let target_fields = Fields::from(vec![+ Arc::new(Field::new("x", DataType::Int32, true)),+ Arc::new(Field::new("y", DataType::Int32, true)),+ Arc::new(Field::new("z", DataType::Int32, true)),+ ]);++ let expr = make_struct_cast_expr(source_fields, target_fields);++ let simplifier =+ ExprSimplifier::new(SimplifyContext::default().with_schema(test_schema()));++ // The cast should remain unchanged since field counts differ+ let result = simplifier.simplify(expr.clone()).unwrap();+ // Ensure const-folding was not attempted (the expression remains exactly the same)+ assert_eq!(+ result, expr,+ "Struct cast with different field counts should remain unchanged (no const-folding)"+ );+ }++ #[test]+ fn test_struct_cast_same_field_count_foldable() {+ // Test that struct casts with same field counts can be considered for const-folding++ let source_fields = Fields::from(vec![+ Arc::new(Field::new("a", DataType::Int32, true)),+ Arc::new(Field::new("b", DataType::Int32, true)),+ ]);++ let target_fields = Fields::from(vec![+ Arc::new(Field::new("a", DataType::Int32, true)),+ Arc::new(Field::new("b", DataType::Int32, true)),+ ]);++ let expr = make_struct_cast_expr(source_fields, target_fields);++ let simplifier =+ ExprSimplifier::new(SimplifyContext::default().with_schema(test_schema()));++ // The cast should be simplified+ let result = simplifier.simplify(expr.clone()).unwrap();+ // Struct casts with same field count should be const-folded to a literal+ assert!(matches!(result, Expr::Literal(_, _)));+ // Ensure the simplifier made a change (not identical to original)+ assert_ne!(+ result, expr,+ "Struct cast with same field count should be simplified (not identical to input)"+ );+ }++ #[test]+ fn test_struct_cast_different_names_same_count() {+ // Test struct cast with same field count but different names+ // Field count matches; simplification should succeed++ let source_fields = Fields::from(vec![+ Arc::new(Field::new("a", DataType::Int32, true)),+ Arc::new(Field::new("b", DataType::Int32, true)),+ ]);++ let target_fields = Fields::from(vec![+ Arc::new(Field::new("x", DataType::Int32, true)),+ Arc::new(Field::new("y", DataType::Int32, true)),+ ]);++ let expr = make_struct_cast_expr(source_fields, target_fields);++ let simplifier =+ ExprSimplifier::new(SimplifyContext::default().with_schema(test_schema()));++ // The cast should be simplified since field counts match+ let result = simplifier.simplify(expr.clone()).unwrap();+ // Struct casts with same field count are const-folded to literals+ assert!(matches!(result, Expr::Literal(_, _)));+ // Ensure the simplifier made a change (not identical to original)+ assert_ne!(+ result, expr,+ "Struct cast with different names but same field count should be simplified"+ );+ }++ #[test]+ fn test_struct_cast_empty_array_not_foldable() {+ // Test that struct casts with 0-row (empty) struct arrays are NOT const-folded+ // The simplifier uses a 1-row input batch, which causes dimension mismatches+ // when evaluating 0-row struct literals++ let source_fields = Fields::from(vec![+ Arc::new(Field::new("a", DataType::Int32, true)),+ Arc::new(Field::new("b", DataType::Int32, true)),+ ]);++ let target_fields = Fields::from(vec![+ Arc::new(Field::new("a", DataType::Int32, true)),+ Arc::new(Field::new("b", DataType::Int32, true)),+ ]);++ // Create a 0-row (empty) struct array+ let arrays: Vec<Arc<dyn Array>> = vec![+ Arc::new(Int32Array::new(vec![].into(), None)),+ Arc::new(Int32Array::new(vec![].into(), None)),+ ];+ let struct_array = StructArray::try_new(source_fields, arrays, None).unwrap();++ let expr = Expr::Cast(Cast::new(+ Box::new(Expr::Literal(+ ScalarValue::Struct(Arc::new(struct_array)),+ None,+ )),+ DataType::Struct(target_fields),+ ));++ let simplifier =+ ExprSimplifier::new(SimplifyContext::default().with_schema(test_schema()));++ // The cast should remain unchanged since the struct array is empty (0-row)+ let result = simplifier.simplify(expr.clone()).unwrap();+ assert_eq!(+ result, expr,+ "Struct cast with empty (0-row) array should remain unchanged"+ );+ }}diff --git a/datafusion/physical-expr/src/expressions/cast.rs b/datafusion/physical-expr/src/expressions/cast.rsindex bd5c63a699..f679a9587c 100644--- a/datafusion/physical-expr/src/expressions/cast.rs+++ b/datafusion/physical-expr/src/expressions/cast.rs@@ -26,6 +26,7 @@ use arrow::compute::{CastOptions, can_cast_types};use arrow::datatypes::{DataType, DataType::*, FieldRef, Schema};use arrow::record_batch::RecordBatch;use datafusion_common::format::DEFAULT_FORMAT_OPTIONS;+use datafusion_common::nested_struct::validate_struct_compatibility;use datafusion_common::{Result, not_impl_err};use datafusion_expr_common::columnar_value::ColumnarValue;use datafusion_expr_common::interval_arithmetic::Interval;@@ -41,6 +42,22 @@ const DEFAULT_SAFE_CAST_OPTIONS: CastOptions<'static> = CastOptions {format_options: DEFAULT_FORMAT_OPTIONS,};+/// Check if struct-to-struct casting is allowed by validating field compatibility.+///+/// This function applies the same validation rules as execution time to ensure+/// planning-time validation matches runtime validation, enabling fail-fast behavior+/// instead of deferring errors to execution.+fn can_cast_struct_types(source: &DataType, target: &DataType) -> bool {+ match (source, target) {+ (Struct(source_fields), Struct(target_fields)) => {+ // Apply the same struct compatibility rules as at execution time.+ // This ensures planning-time validation matches execution-time validation.+ validate_struct_compatibility(source_fields, target_fields).is_ok()+ }+ _ => false,+ }+}+/// CAST expression casts an expression to a specific data type and returns a runtime error on invalid cast#[derive(Debug, Clone, Eq)]pub struct CastExpr {@@ -237,6 +254,12 @@ pub fn cast_with_options(Ok(Arc::clone(&expr))} else if can_cast_types(&expr_type, &cast_type) {Ok(Arc::new(CastExpr::new(expr, cast_type, cast_options)))+ } else if can_cast_struct_types(&expr_type, &cast_type) {+ // Allow struct-to-struct casts that pass name-based compatibility validation.+ // This validation is applied at planning time (now) to fail fast, rather than+ // deferring errors to execution time. The name-based casting logic will be+ // executed at runtime via ColumnarValue::cast_to.+ Ok(Arc::new(CastExpr::new(expr, cast_type, cast_options)))} else {not_impl_err!("Unsupported CAST from {expr_type} to {cast_type}")}diff --git a/datafusion/sqllogictest/test_files/case.slt b/datafusion/sqllogictest/test_files/case.sltindex 481dde5be9..8e0ee08d99 100644--- a/datafusion/sqllogictest/test_files/case.slt+++ b/datafusion/sqllogictest/test_files/case.slt@@ -383,8 +383,10 @@ SELECT column2, column3, column4 FROM t;----{foo: a, xxx: b} {xxx: c, foo: d} {xxx: e}-# coerce structs with different field orders,-# should keep the same field values+# coerce structs with different field orders+# With name-based struct coercion, matching fields by name:+# column2={foo:a, xxx:b} unified with column3={xxx:c, foo:d}+# Result uses the THEN branch's field order (when executed): {xxx: b, foo: a}query ?SELECTcase@@ -396,6 +398,7 @@ FROM t;{xxx: b, foo: a}# coerce structs with different field orders+# When ELSE branch executes, uses its field order: {xxx: c, foo: d}query ?SELECTcase@@ -406,8 +409,9 @@ FROM t;----{xxx: c, foo: d}-# coerce structs with subset of fields-query error Failed to coerce then+# coerce structs with subset of fields - field count mismatch causes type coercion failure+# column3 has 2 fields but column4 has only 1 field+query error DataFusion error: type_coercion\ncaused by\nError during planning: Failed to coerce then .* and else .* to common types in CASE WHEN expressionSELECTcasewhen column1 > 0 then column3diff --git a/datafusion/sqllogictest/test_files/struct.slt b/datafusion/sqllogictest/test_files/struct.sltindex a91a5e7f87..9b1668e58f 100644--- a/datafusion/sqllogictest/test_files/struct.slt+++ b/datafusion/sqllogictest/test_files/struct.slt@@ -492,18 +492,6 @@ Struct("r": Utf8, "c": Float64)statement okdrop table t;-statement ok-create table t as values({r: 'a', c: 1}), ({c: 2.3, r: 'b'});--query ?-select * from t;------{c: 1.0, r: a}-{c: 2.3, r: b}--statement ok-drop table t;-#################################### Test Coalesce with Struct##################################@@ -562,24 +550,12 @@ Struct("a": Float32, "b": Utf8View)statement okdrop table t;-# row() with incorrect order+# row() with incorrect order - row() is positional, not name-basedstatement error DataFusion error: Optimizer rule 'simplify_expressions' failed[\s\S]*Arrow error: Cast error: Cannot cast string 'blue' to value of Float32 typecreate table t(a struct(r varchar, c int), b struct(r varchar, c float)) as values(row('red', 1), row(2.3, 'blue')),(row('purple', 1), row('green', 2.3));-# out of order struct literal-statement ok-create table t(a struct(r varchar, c int)) as values ({r: 'a', c: 1}), ({c: 2, r: 'b'});--query ?-select * from t;------{r: a, c: 1}-{r: b, c: 2}--statement ok-drop table t;#################################### Test Array of Struct@@ -590,11 +566,6 @@ select [{r: 'a', c: 1}, {r: 'b', c: 2}];----[{r: a, c: 1}, {r: b, c: 2}]-# Create a list of struct with different field types-query ?-select [{r: 'a', c: 1}, {c: 2, r: 'b'}];------[{c: 1, r: a}, {c: 2, r: b}]statement okcreate table t(a struct(r varchar, c int), b struct(r varchar, c float)) as values (row('a', 1), row('b', 2.3));@@ -607,18 +578,6 @@ List(Struct("r": Utf8View, "c": Float32))statement okdrop table t;-# create table with different struct type is fine-statement ok-create table t(a struct(r varchar, c int), b struct(c float, r varchar)) as values (row('a', 1), row(2.3, 'b'));--# create array with different struct type should be cast-query T-select arrow_typeof([a, b]) from t;------List(Struct("c": Float32, "r": Utf8View))--statement ok-drop table t;statement okcreate table t(a struct(r varchar, c int, g float), b struct(r varchar, c float, g int)) as values (row('a', 1, 2.3), row('b', 2.3, 2));@@ -845,3 +804,864 @@ NULLstatement okdrop table nullable_parent_test;++# Test struct casting with field reordering - string fields+query ?+SELECT CAST({b: 'b_value', a: 'a_value'} AS STRUCT(a VARCHAR, b VARCHAR));+----+{a: a_value, b: b_value}++# Test struct casting with field reordering - integer fields+query ?+SELECT CAST({b: 3, a: 4} AS STRUCT(a INT, b INT));+----+{a: 4, b: 3}++# Test with type casting AND field reordering+query ?+SELECT CAST({b: 3, a: 4} AS STRUCT(a BIGINT, b INT));+----+{a: 4, b: 3}++# Test positional casting when there is no name overlap+query ?+SELECT CAST(struct(1, 'x') AS STRUCT(a INT, b VARCHAR));+----+{a: 1, b: x}++# Test with missing field - should insert nulls+query ?+SELECT CAST({a: 1} AS STRUCT(a INT, b INT));+----+{a: 1, b: NULL}++# Test with extra source field - should be ignored+query ?+SELECT CAST({a: 1, b: 2, extra: 3} AS STRUCT(a INT, b INT));+----+{a: 1, b: 2}++# Test no overlap with mismatched field count - should fail because no field names match+statement error DataFusion error: (Plan error|Error during planning|This feature is not implemented): (Cannot cast struct: at least one field name must match between source and target|Cannot cast struct with 3 fields to 2 fields without name overlap|Unsupported CAST from Struct)+SELECT CAST(struct(1, 'x', 'y') AS STRUCT(a INT, b VARCHAR));++# Test nested struct with field reordering+query ?+SELECT CAST(+ {inner: {y: 2, x: 1}}+ AS STRUCT(inner STRUCT(x INT, y INT))+);+----+{inner: {x: 1, y: 2}}++# Test field reordering with table data+statement ok+CREATE TABLE struct_reorder_test (+ data STRUCT(b INT, a VARCHAR)+) AS VALUES+ (struct(100, 'first')),+ (struct(200, 'second')),+ (struct(300, 'third'))+;++query ?+SELECT CAST(data AS STRUCT(a VARCHAR, b INT)) AS casted_data FROM struct_reorder_test ORDER BY data['b'];+----+{a: first, b: 100}+{a: second, b: 200}+{a: third, b: 300}++statement ok+drop table struct_reorder_test;++# Test casting struct with multiple levels of nesting and reordering+query ?+SELECT CAST(+ {level1: {z: 100, y: 'inner', x: 1}}+ AS STRUCT(level1 STRUCT(x INT, y VARCHAR, z INT))+);+----+{level1: {x: 1, y: inner, z: 100}}++# Test field reordering with nulls in source+query ?+SELECT CAST(+ {b: NULL::INT, a: 42}+ AS STRUCT(a INT, b INT)+);+----+{a: 42, b: NULL}++# Test casting preserves struct-level nulls+query ?+SELECT CAST(NULL::STRUCT(b INT, a INT) AS STRUCT(a INT, b INT));+----+NULL++############################+# Implicit Coercion Tests with CREATE TABLE AS VALUES+############################++# Test implicit coercion with same field order, different types+statement ok+create table t as values({r: 'a', c: 1}), ({r: 'b', c: 2.3});++query T+select arrow_typeof(column1) from t limit 1;+----+Struct("r": Utf8, "c": Float64)++query ?+select * from t order by column1.r;+----+{r: a, c: 1.0}+{r: b, c: 2.3}++statement ok+drop table t;++# Test implicit coercion with nullable fields (same order)+statement ok+create table t as values({a: 1, b: 'x'}), ({a: 2, b: 'y'});++query T+select arrow_typeof(column1) from t limit 1;+----+Struct("a": Int64, "b": Utf8)++query ?+select * from t order by column1.a;+----+{a: 1, b: x}+{a: 2, b: y}++statement ok+drop table t;++# Test implicit coercion with nested structs (same field order)+statement ok+create table t as+ select {outer: {x: 1, y: 2}} as column1+ union all+ select {outer: {x: 3, y: 4}};++query T+select arrow_typeof(column1) from t limit 1;+----+Struct("outer": Struct("x": Int64, "y": Int64))++query ?+select column1 from t order by column1.outer.x;+----+{outer: {x: 1, y: 2}}+{outer: {x: 3, y: 4}}++statement ok+drop table t;++# Test implicit coercion with type widening (Int32 -> Int64)+statement ok+create table t as values({id: 1, val: 100}), ({id: 2, val: 9223372036854775807});++query T+select arrow_typeof(column1) from t limit 1;+----+Struct("id": Int64, "val": Int64)++query ?+select * from t order by column1.id;+----+{id: 1, val: 100}+{id: 2, val: 9223372036854775807}++statement ok+drop table t;++# Test implicit coercion with nested struct and type coercion+statement ok+create table t as+ select {name: 'Alice', data: {score: 100, active: true}} as column1+ union all+ select {name: 'Bob', data: {score: 200, active: false}};++query T+select arrow_typeof(column1) from t limit 1;+----+Struct("name": Utf8, "data": Struct("score": Int64, "active": Boolean))++query ?+select column1 from t order by column1.name;+----+{name: Alice, data: {score: 100, active: true}}+{name: Bob, data: {score: 200, active: false}}++statement ok+drop table t;++############################+# Field Reordering Tests (using explicit CAST)+############################++# Test explicit cast with field reordering in VALUES - basic case+query ?+select CAST({c: 2.3, r: 'b'} AS STRUCT(r VARCHAR, c FLOAT));+----+{r: b, c: 2.3}++# Test explicit cast with field reordering - multiple rows+query ?+select * from (values+ (CAST({c: 1, r: 'a'} AS STRUCT(r VARCHAR, c FLOAT))),+ (CAST({c: 2.3, r: 'b'} AS STRUCT(r VARCHAR, c FLOAT)))+) order by column1.r;+----+{r: a, c: 1.0}+{r: b, c: 2.3}++# Test table with explicit cast for field reordering+statement ok+create table t as select CAST({c: 1, r: 'a'} AS STRUCT(r VARCHAR, c FLOAT)) as s+union all+select CAST({c: 2.3, r: 'b'} AS STRUCT(r VARCHAR, c FLOAT));++query T+select arrow_typeof(s) from t limit 1;+----+Struct("r": Utf8View, "c": Float32)++query ?+select * from t order by s.r;+----+{r: a, c: 1.0}+{r: b, c: 2.3}++statement ok+drop table t;++# Test field reordering with nullable fields using CAST+query ?+select CAST({b: NULL, a: 42} AS STRUCT(a INT, b INT));+----+{a: 42, b: NULL}++# Test field reordering with nested structs using CAST+query ?+select CAST({outer: {y: 4, x: 3}} AS STRUCT(outer STRUCT(x INT, y INT)));+----+{outer: {x: 3, y: 4}}++# Test complex nested field reordering+query ?+select CAST(+ {data: {active: false, score: 200}, name: 'Bob'}+ AS STRUCT(name VARCHAR, data STRUCT(score INT, active BOOLEAN))+);+----+{name: Bob, data: {score: 200, active: false}}++############################+# Array Literal Tests with Struct Field Reordering (Implicit Coercion)+############################++# Test array literal with reordered struct fields - implicit coercion by name+# Field order in unified schema is determined during type coercion+query ?+select [{r: 'a', c: 1}, {c: 2.3, r: 'b'}];+----+[{c: 1.0, r: a}, {c: 2.3, r: b}]++# Test array literal with same-named fields but different order+# Fields are reordered during coercion+query ?+select [{a: 1, b: 2}, {b: 3, a: 4}];+----+[{b: 2, a: 1}, {b: 3, a: 4}]++# Test array literal with explicit cast to unify struct schemas with partial overlap+# Use CAST to explicitly unify schemas when fields don't match completely+query ?+select [+ CAST({a: 1, b: 2} AS STRUCT(a INT, b INT, c INT)),+ CAST({b: 3, c: 4} AS STRUCT(a INT, b INT, c INT))+];+----+[{a: 1, b: 2, c: NULL}, {a: NULL, b: 3, c: 4}]++# Test NULL handling in array literals with reordered but matching fields+query ?+select [{a: NULL, b: 1}, {b: 2, a: NULL}];+----+[{b: 1, a: NULL}, {b: 2, a: NULL}]++# Verify arrow_typeof for array with reordered struct fields+# The unified schema type follows the coercion order+query T+select arrow_typeof([{x: 1, y: 2}, {y: 3, x: 4}]);+----+List(Struct("y": Int64, "x": Int64))++# Test array of structs with matching nested fields in different order+# Inner nested fields are also reordered during coercion+query ?+select [+ {id: 1, info: {name: 'Alice', age: 30}},+ {info: {age: 25, name: 'Bob'}, id: 2}+];+----+[{info: {age: 30, name: Alice}, id: 1}, {info: {age: 25, name: Bob}, id: 2}]++# Test nested arrays with matching struct fields (different order)+query ?+select [[{x: 1, y: 2}], [{y: 4, x: 3}]];+----+[[{x: 1, y: 2}], [{x: 3, y: 4}]]++# Test array literal with float type coercion across elements+query ?+select [{val: 1}, {val: 2.5}];+----+[{val: 1.0}, {val: 2.5}]++############################+# Dynamic Array Construction Tests (from Table Columns)+############################++# Setup test table with struct columns for dynamic array construction+statement ok+create table t_complete_overlap (+ s1 struct(x int, y int),+ s2 struct(y int, x int)+) as values+ ({x: 1, y: 2}, {y: 3, x: 4}),+ ({x: 5, y: 6}, {y: 7, x: 8});++# Test 1: Complete overlap - same fields, different order+# Verify arrow_typeof for dynamically created array+query T+select arrow_typeof([s1, s2]) from t_complete_overlap limit 1;+----+List(Struct("y": Int32, "x": Int32))++# Verify values are correctly mapped by name in the array+# Field order follows the second column's field order+query ?+select [s1, s2] from t_complete_overlap order by s1.x;+----+[{y: 2, x: 1}, {y: 3, x: 4}]+[{y: 6, x: 5}, {y: 7, x: 8}]++statement ok+drop table t_complete_overlap;++# Test 2: Partial overlap - some shared fields between columns+# Note: Columns must have the exact same field set for array construction to work+# Test with identical field set (all fields present in both columns)+statement ok+create table t_partial_overlap (+ col_a struct(name VARCHAR, age int, active boolean),+ col_b struct(age int, name VARCHAR, active boolean)+) as values+ ({name: 'Alice', age: 30, active: true}, {age: 25, name: 'Bob', active: false}),+ ({name: 'Charlie', age: 35, active: true}, {age: 40, name: 'Diana', active: false});++# Verify unified type includes all fields from both structs+query T+select arrow_typeof([col_a, col_b]) from t_partial_overlap limit 1;+----+List(Struct("age": Int32, "name": Utf8View, "active": Boolean))++# Verify values are correctly mapped by name in the array+# Field order follows the second column's field order+query ?+select [col_a, col_b] from t_partial_overlap order by col_a.name;+----+[{age: 30, name: Alice, active: true}, {age: 25, name: Bob, active: false}]+[{age: 35, name: Charlie, active: true}, {age: 40, name: Diana, active: false}]++statement ok+drop table t_partial_overlap;++# Test 3: Complete field set matching (no CAST needed)+# Schemas already align; confirm unified type and values+statement ok+create table t_with_cast (+ col_x struct(id int, description VARCHAR),+ col_y struct(id int, description VARCHAR)+) as values+ ({id: 1, description: 'First'}, {id: 10, description: 'First Value'}),+ ({id: 2, description: 'Second'}, {id: 20, description: 'Second Value'});++# Verify type unification with all fields+query T+select arrow_typeof([col_x, col_y]) from t_with_cast limit 1;+----+List(Struct("id": Int32, "description": Utf8View))++# Verify values remain aligned by name+query ?+select [col_x, col_y] from t_with_cast order by col_x.id;+----+[{id: 1, description: First}, {id: 10, description: First Value}]+[{id: 2, description: Second}, {id: 20, description: Second Value}]++statement ok+drop table t_with_cast;++# Test 4: Explicit CAST for partial field overlap scenarios+# When columns have different field sets, use explicit CAST to unify schemas+query ?+select [+ CAST({id: 1} AS STRUCT(id INT, description VARCHAR)),+ CAST({id: 10, description: 'Value'} AS STRUCT(id INT, description VARCHAR))+];+----+[{id: 1, description: NULL}, {id: 10, description: Value}]++# Test 5: Complex nested structs with field reordering+# Nested fields must have the exact same field set for array construction+statement ok+create table t_nested (+ col_1 struct(id int, outer struct(x int, y int)),+ col_2 struct(id int, outer struct(x int, y int))+) as values+ ({id: 100, outer: {x: 1, y: 2}}, {id: 101, outer: {x: 4, y: 3}}),+ ({id: 200, outer: {x: 5, y: 6}}, {id: 201, outer: {x: 8, y: 7}});++# Verify nested struct in unified schema+query T+select arrow_typeof([col_1, col_2]) from t_nested limit 1;+----+List(Struct("id": Int32, "outer": Struct("x": Int32, "y": Int32)))++# Verify nested field values are correctly mapped+query ?+select [col_1, col_2] from t_nested order by col_1.id;+----+[{id: 100, outer: {x: 1, y: 2}}, {id: 101, outer: {x: 4, y: 3}}]+[{id: 200, outer: {x: 5, y: 6}}, {id: 201, outer: {x: 8, y: 7}}]++statement ok+drop table t_nested;++# Test 6: NULL handling with matching field sets+statement ok+create table t_nulls (+ col_a struct(val int, flag boolean),+ col_b struct(val int, flag boolean)+) as values+ ({val: 1, flag: true}, {val: 10, flag: false}),+ ({val: NULL, flag: false}, {val: NULL, flag: true});++# Verify NULL values are preserved+query ?+select [col_a, col_b] from t_nulls order by col_a.val;+----+[{val: 1, flag: true}, {val: 10, flag: false}]+[{val: NULL, flag: false}, {val: NULL, flag: true}]++statement ok+drop table t_nulls;++# Test 7: Multiple columns with complete field matching+statement ok+create table t_multi (+ col1 struct(a int, b int, c int),+ col2 struct(a int, b int, c int)+) as values+ ({a: 1, b: 2, c: 3}, {a: 10, b: 20, c: 30}),+ ({a: 4, b: 5, c: 6}, {a: 40, b: 50, c: 60});++# Verify array with complete field matching+query T+select arrow_typeof([col1, col2]) from t_multi limit 1;+----+List(Struct("a": Int32, "b": Int32, "c": Int32))++# Verify values are correctly unified+query ?+select [col1, col2] from t_multi order by col1.a;+----+[{a: 1, b: 2, c: 3}, {a: 10, b: 20, c: 30}]+[{a: 4, b: 5, c: 6}, {a: 40, b: 50, c: 60}]++statement ok+drop table t_multi;++############################+# Comprehensive Implicit Struct Coercion Suite+############################++# Test 1: VALUES clause with field reordering coerced by name into declared schema+statement ok+create table implicit_values_reorder (+ s struct(a int, b int)+) as values+ ({a: 1, b: 2}),+ ({b: 3, a: 4});++query T+select arrow_typeof(s) from implicit_values_reorder limit 1;+----+Struct("a": Int32, "b": Int32)++query ?+select s from implicit_values_reorder order by s.a;+----+{a: 1, b: 2}+{a: 4, b: 3}++statement ok+drop table implicit_values_reorder;++# Test 2: Array literal coercion with reordered struct fields+query IIII+select+ [{a: 1, b: 2}, {b: 3, a: 4}][1]['a'],+ [{a: 1, b: 2}, {b: 3, a: 4}][1]['b'],+ [{a: 1, b: 2}, {b: 3, a: 4}][2]['a'],+ [{a: 1, b: 2}, {b: 3, a: 4}][2]['b'];+----+1 2 4 3++# Test 3: Array construction from columns with reordered struct fields+statement ok+create table struct_columns_order (+ s1 struct(a int, b int),+ s2 struct(b int, a int)+) as values+ ({a: 1, b: 2}, {b: 3, a: 4}),+ ({a: 5, b: 6}, {b: 7, a: 8});++query IIII+select+ [s1, s2][1]['a'],+ [s1, s2][1]['b'],+ [s1, s2][2]['a'],+ [s1, s2][2]['b']+from struct_columns_order+order by s1['a'];+----+1 2 4 3+5 6 8 7++statement ok+drop table struct_columns_order;++# Test 4: UNION with struct field reordering+query II+select s['a'], s['b']+from (+ select {a: 1, b: 2} as s+ union all+ select {b: 3, a: 4} as s+) t+order by s['a'];+----+1 2+4 3++# Test 5: CTE with struct coercion across branches+query II+with+ t1 as (select {a: 1, b: 2} as s),+ t2 as (select {b: 3, a: 4} as s)+select t1.s['a'], t1.s['b'] from t1+union all+select t2.s['a'], t2.s['b'] from t2+order by 1;+----+1 2+4 3++# Test 6: Struct aggregation retains name-based mapping+statement ok+create table agg_structs_reorder (+ k int,+ s struct(x int, y int)+) as values+ (1, {x: 1, y: 2}),+ (1, {y: 3, x: 4}),+ (2, {x: 5, y: 6});++query I?+select k, array_agg(s) from agg_structs_reorder group by k order by k;+----+1 [{x: 1, y: 2}, {x: 4, y: 3}]+2 [{x: 5, y: 6}]++statement ok+drop table agg_structs_reorder;++# Test 7: Nested struct coercion with reordered inner fields+query IIII+with nested as (+ select [{outer: {inner: 1, value: 2}}, {outer: {value: 3, inner: 4}}] as arr+)+select+ arr[1]['outer']['inner'],+ arr[1]['outer']['value'],+ arr[2]['outer']['inner'],+ arr[2]['outer']['value']+from nested;+----+1 2 4 3++# Test 8: Partial name overlap - currently errors (field count mismatch detected)+# This is a documented limitation: structs must have exactly same field set for coercion+query error DataFusion error: Error during planning: Inconsistent data type across values list+select column1 from (values ({a: 1, b: 2}), ({b: 3, c: 4})) order by column1['a'];++# Negative test: mismatched struct field counts are rejected (documented limitation)+query error DataFusion error: .*+select [{a: 1}, {a: 2, b: 3}];++# Test 9: INSERT with name-based struct coercion into target schema+statement ok+create table target_struct_insert (s struct(a int, b int));++statement ok+insert into target_struct_insert values ({b: 1, a: 2});++query ?+select s from target_struct_insert;+----+{a: 2, b: 1}++statement ok+drop table target_struct_insert;++# Test 10: CASE expression with different struct field orders+query II+select+ (case when true then {a: 1, b: 2} else {b: 3, a: 4} end)['a'] as a_val,+ (case when true then {a: 1, b: 2} else {b: 3, a: 4} end)['b'] as b_val;+----+1 2++############################+# JOIN Coercion Tests+############################++# Test: Struct coercion in JOIN ON condition+statement ok+create table t_left (+ id int,+ s struct(x int, y int)+) as values+ (1, {x: 1, y: 2}),+ (2, {x: 3, y: 4});++statement ok+create table t_right (+ id int,+ s struct(y int, x int)+) as values+ (1, {y: 2, x: 1}),+ (2, {y: 4, x: 3});++# JOIN on reordered struct fields - matched by name+query IIII+select t_left.id, t_left.s['x'], t_left.s['y'], t_right.id+from t_left+join t_right on t_left.s = t_right.s+order by t_left.id;+----+1 1 2 1+2 3 4 2++statement ok+drop table t_left;++statement ok+drop table t_right;++# Test: Struct coercion with filtered JOIN+statement ok+create table orders (+ order_id int,+ customer struct(name varchar, id int)+) as values+ (1, {name: 'Alice', id: 100}),+ (2, {name: 'Bob', id: 101}),+ (3, {name: 'Charlie', id: 102});++statement ok+create table customers (+ customer_id int,+ info struct(id int, name varchar)+) as values+ (100, {id: 100, name: 'Alice'}),+ (101, {id: 101, name: 'Bob'}),+ (103, {id: 103, name: 'Diana'});++# Join with struct field reordering - names matched, not positions+query I+select count(*) from orders+join customers on orders.customer = customers.info+where orders.order_id <= 2;+----+2++statement ok+drop table orders;++statement ok+drop table customers;++############################+# WHERE Predicate Coercion Tests+############################++# Test: Struct equality in WHERE clause with field reordering+statement ok+create table t_where (+ id int,+ s struct(x int, y int)+) as values+ (1, {x: 1, y: 2}),+ (2, {x: 3, y: 4}),+ (3, {x: 1, y: 2});++# WHERE clause with struct comparison - coerced by name+query I+select id from t_where+where s = {y: 2, x: 1}+order by id;+----+1+3++statement ok+drop table t_where;++# Test: Struct IN clause with reordering+statement ok+create table t_in (+ id int,+ s struct(a int, b varchar)+) as values+ (1, {a: 1, b: 'x'}),+ (2, {a: 2, b: 'y'}),+ (3, {a: 1, b: 'x'});++# IN clause with reordered struct literals+query I+select id from t_in+where s in ({b: 'x', a: 1}, {b: 'y', a: 2})+order by id;+----+1+2+3++statement ok+drop table t_in;++# Test: Struct BETWEEN (not supported, but documents limitation)+# Structs don't support BETWEEN, but can use comparison operators++statement ok+create table t_between (+ id int,+ s struct(val int)+) as values+ (1, {val: 10}),+ (2, {val: 20}),+ (3, {val: 30});++# Comparison via field extraction works+query I+select id from t_between+where s['val'] >= 20+order by id;+----+2+3++statement ok+drop table t_between;++############################+# Window Function Coercion Tests+############################++# Test: Struct in window function PARTITION BY+statement ok+create table t_window (+ id int,+ s struct(category int, value int)+) as values+ (1, {category: 1, value: 10}),+ (2, {category: 1, value: 20}),+ (3, {category: 2, value: 30}),+ (4, {category: 2, value: 40});++# Window partition on struct field via extraction+query III+select+ id,+ s['value'],+ row_number() over (partition by s['category'] order by s['value'])+from t_window+order by id;+----+1 10 1+2 20 2+3 30 1+4 40 2++statement ok+drop table t_window;++# Test: Struct in window function ORDER BY with coercion+statement ok+create table t_rank (+ id int,+ s struct(rank_val int, group_id int)+) as values+ (1, {rank_val: 10, group_id: 1}),+ (2, {rank_val: 20, group_id: 1}),+ (3, {rank_val: 15, group_id: 2});++# Window ranking with struct field extraction+query III+select+ id,+ s['rank_val'],+ rank() over (partition by s['group_id'] order by s['rank_val'])+from t_rank+order by id;+----+1 10 1+2 20 2+3 15 1++statement ok+drop table t_rank;++# Test: Aggregate function with struct coercion across window partitions+statement ok+create table t_agg_window (+ id int,+ partition_id int,+ s struct(amount int)+) as values+ (1, 1, {amount: 100}),+ (2, 1, {amount: 200}),+ (3, 2, {amount: 150});++# Running sum via extracted struct field+query III+select+ id,+ partition_id,+ sum(s['amount']) over (partition by partition_id order by id)+from t_agg_window+order by id;+----+1 1 100+2 1 300+3 2 150++statement ok+drop table t_agg_window;\ No newline at end of filediff --git a/docs/source/user-guide/sql/index.rst b/docs/source/user-guide/sql/index.rstindex a13d40334b..f1fef45f70 100644--- a/docs/source/user-guide/sql/index.rst+++ b/docs/source/user-guide/sql/index.rst@@ -22,6 +22,7 @@ SQL Reference:maxdepth: 2data_types+ struct_coercionselectsubqueriesddldiff --git a/docs/source/user-guide/sql/struct_coercion.md b/docs/source/user-guide/sql/struct_coercion.mdnew file mode 100644index 0000000000..d2a32fcee2--- /dev/null+++ b/docs/source/user-guide/sql/struct_coercion.md@@ -0,0 +1,354 @@+<!---+ Licensed to the Apache Software Foundation (ASF) under one+ or more contributor license agreements. See the NOTICE file+ distributed with this work for additional information+ regarding copyright ownership. The ASF licenses this file+ to you under the Apache License, Version 2.0 (the+ "License"); you may not use this file except in compliance+ with the License. You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing,+ software distributed under the License is distributed on an+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+ KIND, either express or implied. See the License for the+ specific language governing permissions and limitations+ under the License.+-->++# Struct Type Coercion and Field Mapping++DataFusion uses **name-based field mapping** when coercing struct types across different operations. This document explains how struct coercion works, when it applies, and how to handle NULL fields.++## Overview: Name-Based vs Positional Mapping++When combining structs from different sources (e.g., in UNION, array construction, or JOINs), DataFusion matches struct fields by **name** rather than by **position**. This provides more robust and predictable behavior compared to positional matching.++### Example: Field Reordering is Handled Transparently++```sql+-- These two structs have the same fields in different order+SELECT [{a: 1, b: 2}, {b: 3, a: 4}];++-- Result: Field names matched, values unified+-- [{"a": 1, "b": 2}, {"a": 4, "b": 3}]+```++## Coercion Paths Using Name-Based Matching++The following query operations use name-based field mapping for struct coercion:++### 1. Array Literal Construction++When creating array literals with struct elements that have different field orders:++```sql+-- Structs with reordered fields in array literal+SELECT [{x: 1, y: 2}, {y: 3, x: 4}];++-- Unified type: List(Struct("x": Int32, "y": Int32))+-- Values: [{"x": 1, "y": 2}, {"x": 4, "y": 3}]+```++**When it applies:**++- Array literals with struct elements: `[{...}, {...}]`+- Nested arrays with structs: `[[{x: 1}, {x: 2}]]`++### 2. Array Construction from Columns++When constructing arrays from table columns with different struct schemas:++```sql+CREATE TABLE t_left (s struct(x int, y int)) AS VALUES ({x: 1, y: 2});+CREATE TABLE t_right (s struct(y int, x int)) AS VALUES ({y: 3, x: 4});++-- Dynamically constructs unified array schema+SELECT [t_left.s, t_right.s] FROM t_left JOIN t_right;++-- Result: [{"x": 1, "y": 2}, {"x": 4, "y": 3}]+```++**When it applies:**++- Array construction with column references: `[col1, col2]`+- Array construction in joins with matching field names++### 3. UNION Operations++When combining query results with different struct field orders:++```sql+SELECT {a: 1, b: 2} as s+UNION ALL+SELECT {b: 3, a: 4} as s;++-- Result: {"a": 1, "b": 2} and {"a": 4, "b": 3}+```++**When it applies:**++- UNION ALL with structs: field names matched across branches+- UNION (deduplicated) with structs++### 4. Common Table Expressions (CTEs)++When multiple CTEs produce structs with different field orders that are combined:++```sql+WITH+ t1 AS (SELECT {a: 1, b: 2} as s),+ t2 AS (SELECT {b: 3, a: 4} as s)+SELECT s FROM t1+UNION ALL+SELECT s FROM t2;++-- Result: Field names matched across CTEs+```++### 5. VALUES Clauses++When creating tables or temporary results with struct values in different field orders:++```sql+CREATE TABLE t AS VALUES ({a: 1, b: 2}), ({b: 3, a: 4});++-- Table schema unified: struct(a: int, b: int)+-- Values: {a: 1, b: 2} and {a: 4, b: 3}+```++### 6. JOIN Operations++When joining tables where the JOIN condition involves structs with different field orders:++```sql+CREATE TABLE orders (customer struct(name varchar, id int));+CREATE TABLE customers (info struct(id int, name varchar));++-- Join matches struct fields by name+SELECT * FROM orders+JOIN customers ON orders.customer = customers.info;+```++### 7. Aggregate Functions++When collecting structs with different field orders using aggregate functions like `array_agg`:++```sql+SELECT array_agg(s) FROM (+ SELECT {x: 1, y: 2} as s+ UNION ALL+ SELECT {y: 3, x: 4} as s+) t+GROUP BY category;++-- Result: Array of structs with unified field order+```++### 8. Window Functions++When using window functions with struct expressions having different field orders:++```sql+SELECT+ id,+ row_number() over (partition by s order by id) as rn+FROM (+ SELECT {category: 1, value: 10} as s, 1 as id+ UNION ALL+ SELECT {value: 20, category: 1} as s, 2 as id+);++-- Fields matched by name in PARTITION BY clause+```++## NULL Handling for Missing Fields++When structs have different field sets, missing fields are filled with **NULL** values during coercion.++### Example: Partial Field Overlap++```sql+-- Struct in first position has fields: a, b+-- Struct in second position has fields: b, c+-- Unified schema includes all fields: a, b, c++SELECT [+ CAST({a: 1, b: 2} AS STRUCT(a INT, b INT, c INT)),+ CAST({b: 3, c: 4} AS STRUCT(a INT, b INT, c INT))+];++-- Result:+-- [+-- {"a": 1, "b": 2, "c": NULL},+-- {"a": NULL, "b": 3, "c": 4}+-- ]+```++### Limitations++**Field count must match exactly.** If structs have different numbers of fields and their field names don't completely overlap, the query will fail:++```sql+-- This fails because field sets don't match:+-- t_left has {x, y} but t_right has {x, y, z}+SELECT [t_left.s, t_right.s] FROM t_left JOIN t_right;+-- Error: Cannot coerce struct with mismatched field counts+```++**Workaround: Use explicit CAST**++To handle partial field overlap, explicitly cast structs to a unified schema:++```sql+SELECT [+ CAST(t_left.s AS STRUCT(x INT, y INT, z INT)),+ CAST(t_right.s AS STRUCT(x INT, y INT, z INT))+] FROM t_left JOIN t_right;+```++## Migration Guide: From Positional to Name-Based Matching++If you have existing code that relied on **positional** struct field matching, you may need to update it.++### Example: Query That Changes Behavior++**Old behavior (positional):**++```sql+-- These would have been positionally mapped (left-to-right)+SELECT [{x: 1, y: 2}, {y: 3, x: 4}];+-- Old result (positional): [{"x": 1, "y": 2}, {"y": 3, "x": 4}]+```++**New behavior (name-based):**++```sql+-- Now uses name-based matching+SELECT [{x: 1, y: 2}, {y: 3, x: 4}];+-- New result (by name): [{"x": 1, "y": 2}, {"x": 4, "y": 3}]+```++### Migration Steps++1. **Review struct operations** - Look for queries that combine structs from different sources+2. **Check field names** - Verify that field names match as expected (not positions)+3. **Test with new coercion** - Run queries and verify the results match your expectations+4. **Handle field reordering** - If you need specific field orders, use explicit CAST operations++### Using Explicit CAST for Compatibility++If you need precise control over struct field order and types, use explicit `CAST`:++```sql+-- Guarantee specific field order and types+SELECT CAST({b: 3, a: 4} AS STRUCT(a INT, b INT));+-- Result: {"a": 4, "b": 3}+```++## Best Practices++### 1. Be Explicit with Schema Definitions++When joining or combining structs, define target schemas explicitly:++```sql+-- Good: explicit schema definition+SELECT CAST(data AS STRUCT(id INT, name VARCHAR, active BOOLEAN))+FROM external_source;+```++### 2. Use Named Struct Constructors++Prefer named struct constructors for clarity:++```sql+-- Good: field names are explicit+SELECT named_struct('id', 1, 'name', 'Alice', 'active', true);++-- Or using struct literal syntax+SELECT {id: 1, name: 'Alice', active: true};+```++### 3. Test Field Mappings++Always verify that field mappings work as expected:++```sql+-- Use arrow_typeof to verify unified schema+SELECT arrow_typeof([{x: 1, y: 2}, {y: 3, x: 4}]);+-- Result: List(Struct("x": Int32, "y": Int32))+```++### 4. Handle Partial Field Overlap Explicitly++When combining structs with partial field overlap, use explicit CAST:++```sql+-- Instead of relying on implicit coercion+SELECT [+ CAST(left_struct AS STRUCT(x INT, y INT, z INT)),+ CAST(right_struct AS STRUCT(x INT, y INT, z INT))+];+```++### 5. Document Struct Schemas++In complex queries, document the expected struct schemas:++```sql+-- Expected schema: {customer_id: INT, name: VARCHAR, age: INT}+SELECT {+ customer_id: c.id,+ name: c.name,+ age: c.age+} as customer_info+FROM customers c;+```++## Error Messages and Troubleshooting++### "Cannot coerce struct with different field counts"++**Cause:** Trying to combine structs with different numbers of fields.++**Solution:**++```sql+-- Use explicit CAST to handle missing fields+SELECT [+ CAST(struct1 AS STRUCT(a INT, b INT, c INT)),+ CAST(struct2 AS STRUCT(a INT, b INT, c INT))+];+```++### "Field X not found in struct"++**Cause:** Referencing a field name that doesn't exist in the struct.++**Solution:**++```sql+-- Verify field names match exactly (case-sensitive)+SELECT s['field_name'] FROM my_table; -- Use bracket notation for access+-- Or use get_field function+SELECT get_field(s, 'field_name') FROM my_table;+```++### Unexpected NULL values after coercion++**Cause:** Struct coercion added NULL for missing fields.++**Solution:** Check that all structs have the required fields, or explicitly handle NULLs:++```sql+SELECT COALESCE(s['field'], default_value) FROM my_table;+```++## Related Functions++- `arrow_typeof()` - Returns the Arrow type of an expression+- `struct()` / `named_struct()` - Creates struct values+- `get_field()` - Extracts field values from structs+- `CAST()` - Explicitly casts structs to specific schemas