Apache DataFusion SQL Query Engine
Add special case in validate_struct_compatibility for empty structs (#25023)
Pepijn Van Eeckhoudt84ccbb15be15
diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rsindex 3604126075..2f046b92c3 100644--- a/datafusion/common/src/nested_struct.rs+++ b/datafusion/common/src/nested_struct.rs@@ -36,13 +36,13 @@ use std::{/// Cast a struct column to match target struct fields, handling nested structs recursively.///-/// This function implements struct-to-struct casting with the assumption that **structs should-/// always be allowed to cast to other structs**. However, the source column must already be-/// a struct type - non-struct sources will result in an error.+/// Structs can be cast to other structs as long as they share at least one field. The source column+/// must already be a struct type - non-struct sources will result in an error.////// ## Field Matching Strategy/// - **By Name**: Source struct fields are matched to target fields by name (case-sensitive)-/// - **No Positional Mapping**: Structs with no overlapping field names are rejected+/// - **At Least One Mapped Field**: The source and target struct must share at least one field by name+/// (unless the source struct has no fields at all)/// - **Type Adaptation**: When a matching field is found, it is recursively cast to the target field's type/// - **Missing Fields**: Target fields not present in the source are filled with null values/// - **Extra Fields**: Source fields not present in the target are ignored@@ -75,7 +75,13 @@ fn cast_struct_column(if let Some(source_struct) = source_col.as_any().downcast_ref::<StructArray>() {let source_fields = source_struct.fields();- validate_struct_compatibility(source_fields, target_fields)?;+ let mut mapped_source_fields: Vec<Option<usize>> =+ Vec::with_capacity(target_fields.len());+ map_struct_fields(+ source_fields,+ target_fields,+ Some(&mut mapped_source_fields),+ )?;if !source_col.is_empty() && source_col.null_count() == source_col.len() {return Ok(new_null_array(@@ -89,14 +95,15 @@ fn cast_struct_column(let num_rows = source_col.len();// Iterate target fields and pick source child by name when present.- for target_child_field in target_fields.iter() {+ for (source_child_opt, target_child_field) in+ mapped_source_fields.iter().zip(target_fields.iter())+ {fields.push(Arc::clone(target_child_field));- let source_child_opt =- source_struct.column_by_name(target_child_field.name());-match source_child_opt {- Some(source_child_col) => {+ Some(source_child_col_idx) => {+ let source_child_col = source_struct.column(*source_child_col_idx);+let adapted_child = cast_column(source_child_col,target_child_field.data_type(),@@ -111,13 +118,19 @@ fn cast_struct_column(arrays.push(adapted_child);}None => {+ // No need to check if `target_child_field` is nullable here+ // The call to `map_struct_fields` will have done that for usarrays.push(new_null_array(target_child_field.data_type(), num_rows));}}}- let struct_array =- StructArray::try_new(fields.into(), arrays, source_struct.nulls().cloned())?;+ let struct_array = StructArray::try_new_with_length(+ fields.into(),+ arrays,+ source_struct.nulls().cloned(),+ source_struct.len(),+ )?;Ok(Arc::new(struct_array))} else {// Return error if source is not a struct type@@ -670,6 +683,8 @@ fn cast_dictionary_column(/// - **Field Matching**: Fields are matched by name (case-sensitive)/// - **Missing Target Fields**: Allowed - will be filled with null values during casting/// - **Extra Source Fields**: Allowed - will be ignored during casting+/// - **At Least One Mapped Field**: source and target must share at least one field by name+/// (unless source is empty)/// - **Type Compatibility**: Each matching field must be castable using Arrow's type system/// - **Nested Structs**: Recursively validates nested struct compatibility///@@ -698,23 +713,49 @@ pub fn validate_struct_compatibility(source_fields: &[FieldRef],target_fields: &[FieldRef],) -> Result<()> {- let has_overlap = has_one_of_more_common_fields(source_fields, target_fields);- if !has_overlap {- return _plan_err!(- "Cannot cast struct with {} fields to {} fields because there is no field name overlap",- source_fields.len(),- target_fields.len()- );+ map_struct_fields(source_fields, target_fields, None)+}++/// Performs the mapping of struct fields, checking compatibility and handling missing or extra fields.+///+/// # Arguments+/// * `source_fields` - Fields from the source struct type+/// * `target_fields` - Fields from the target struct type+/// * `mapped_source_fields` - An optional `Vec` into which the indexes of the matching source fields+/// are written in the order of the target fields+///+/// # Returns+/// * `Ok(())` if the structs are compatible for casting+/// * `Err(DataFusionError)` with detailed error message if incompatible+fn map_struct_fields(+ source_fields: &[FieldRef],+ target_fields: &[FieldRef],+ mut mapped_source_fields: Option<&mut Vec<Option<usize>>>,+) -> Result<()> {+ if !source_fields.is_empty() {+ let has_overlap = has_one_of_more_common_fields(source_fields, target_fields);+ if !has_overlap {+ return _plan_err!(+ "Cannot cast struct with {} fields to {} fields because there is no field name overlap",+ source_fields.len(),+ target_fields.len()+ );+ }}// Check compatibility for each target fieldfor target_field in target_fields {// Look for matching field in source by name- if let Some(source_field) = source_fields+ let source_field_opt = source_fields.iter()- .find(|f| f.name() == target_field.name())+ .enumerate()+ .find(|(_, f)| f.name() == target_field.name());+ let source_field_idx = if let Some((source_field_idx, source_field)) =+ source_field_opt{validate_field_compatibility(source_field, target_field)?;++ Some(source_field_idx)} else {// Target field is missing from source// If it's non-nullable, we cannot fill it with NULL@@ -725,6 +766,12 @@ pub fn validate_struct_compatibility(target_field.name());}++ None+ };++ if let Some(mapped_fields) = mapped_source_fields.as_mut() {+ mapped_fields.push(source_field_idx);}}@@ -1153,6 +1200,8 @@ mod tests {buffer::{NullBuffer, OffsetBuffer, ScalarBuffer},datatypes::{DataType, Field, FieldRef, Int32Type},};+ use arrow_schema::Fields;+/// Macro to extract and downcast a column from a StructArraymacro_rules! get_column_as {($struct_array:expr, $column_name:expr, $array_type:ty) => {@@ -1289,6 +1338,65 @@ mod tests {assert!(error_msg.contains("Cannot cast struct field 'a'"));}+ #[test]+ fn test_cast_struct_empty_source_and_target() {+ // Source and target struct: {} (no fields)+ let source =+ StructArray::try_new_with_length(Fields::empty(), vec![], None, 1024)+ .unwrap();+ let source = Arc::new(source) as ArrayRef;++ // Target struct: {}+ let target_field = struct_field("s", vec![]);++ // This should succeed - an empty struct is compatible with itself+ let result =+ cast_column(&source, target_field.data_type(), &DEFAULT_CAST_OPTIONS)+ .unwrap();+ let result = result.as_any().downcast_ref::<StructArray>().unwrap();++ assert_eq!(result.len(), source.len());+ assert_eq!(result.data_type(), target_field.data_type());+ }++ #[test]+ fn test_cast_struct_compatibility_empty_source_nullable_target() {+ // Source and target struct: {} (no fields)+ let source =+ StructArray::try_new_with_length(Fields::empty(), vec![], None, 1024)+ .unwrap();+ let source = Arc::new(source) as ArrayRef;++ // Target struct: {a: Int32}+ let target_field = struct_field("s", vec![field("a", DataType::Int32)]);++ // This should succeed - all target fields are nullable+ let result =+ cast_column(&source, target_field.data_type(), &DEFAULT_CAST_OPTIONS)+ .unwrap();+ let result = result.as_any().downcast_ref::<StructArray>().unwrap();++ assert_eq!(result.len(), source.len());+ assert_eq!(result.data_type(), target_field.data_type());+ }++ #[test]+ fn test_cast_struct_compatibility_empty_source_non_nullable_target() {+ // Source and target struct: {} (no fields)+ let source =+ StructArray::try_new_with_length(Fields::empty(), vec![], None, 1024)+ .unwrap();+ let source = Arc::new(source) as ArrayRef;++ // Target struct: {a: Int32}+ let target_field = struct_field("s", vec![non_null_field("a", DataType::Int32)]);++ // This should succeed - all target fields are nullable+ let result =+ cast_column(&source, target_field.data_type(), &DEFAULT_CAST_OPTIONS);+ assert!(result.is_err());+ }+#[test]fn test_validate_struct_compatibility_incompatible_types() {// Source struct: {field1: Binary, field2: String}@@ -1551,6 +1659,46 @@ mod tests {assert!(result.is_ok());}+ #[test]+ fn test_validate_struct_compatibility_empty_source_and_target() {+ // Source and target struct: {} (no fields)+ let source_fields = vec![];+ let target_fields = vec![];++ // This should succeed - an empty struct is compatible with itself+ let result = validate_struct_compatibility(&source_fields, &target_fields);+ assert!(result.is_ok());+ }++ #[test]+ fn test_validate_struct_compatibility_empty_source_nullable_target() {+ // Source and target struct: {} (no fields)+ let source_fields = vec![];++ // Target struct: {a: Int32}+ let target_fields = vec![arc_field("a", DataType::Int32)];++ // This should succeed - all target fields are nullable+ let result = validate_struct_compatibility(&source_fields, &target_fields);+ assert!(result.is_ok());+ }++ #[test]+ fn test_validate_struct_compatibility_empty_source_non_nullable_target() {+ // Source and target struct: {} (no fields)+ let source_fields = vec![];++ // Target struct: {a: Int32 NOT NULL}+ let target_fields = vec![+ arc_field("a", DataType::Int32),+ Arc::new(non_null_field("b", DataType::Utf8)),+ ];++ // This should fail - not all target fields are nullable+ let result = validate_struct_compatibility(&source_fields, &target_fields);+ assert!(result.is_err());+ }+#[test]fn test_cast_nested_struct_with_extra_and_missing_fields() {// Source inner struct has fields a, b, extra