Skip to content

Apache DataFusion SQL Query Engine

Add special case in validate_struct_compatibility for empty structs (#25023)

Pepijn Van Eeckhoudt84ccbb15be15
diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rs
index 3604126075..2f046b92c3 100644
--- a/datafusion/common/src/nested_struct.rs
+++ b/datafusion/common/src/nested_struct.rs
@@ -36,13 +36,13 @@ use std::{
/// Cast a struct column to match target struct fields, handling nested structs recursively.
///
-/// This function implements struct-to-struct casting with the assumption that **structs should
-/// always be allowed to cast to other structs**. However, the source column must already be
-/// a struct type - non-struct sources will result in an error.
+/// Structs can be cast to other structs as long as they share at least one field. The source column
+/// must already be a struct type - non-struct sources will result in an error.
///
/// ## Field Matching Strategy
/// - **By Name**: Source struct fields are matched to target fields by name (case-sensitive)
-/// - **No Positional Mapping**: Structs with no overlapping field names are rejected
+/// - **At Least One Mapped Field**: The source and target struct must share at least one field by name
+/// (unless the source struct has no fields at all)
/// - **Type Adaptation**: When a matching field is found, it is recursively cast to the target field's type
/// - **Missing Fields**: Target fields not present in the source are filled with null values
/// - **Extra Fields**: Source fields not present in the target are ignored
@@ -75,7 +75,13 @@ fn cast_struct_column(
if let Some(source_struct) = source_col.as_any().downcast_ref::<StructArray>() {
let source_fields = source_struct.fields();
- validate_struct_compatibility(source_fields, target_fields)?;
+ let mut mapped_source_fields: Vec<Option<usize>> =
+ Vec::with_capacity(target_fields.len());
+ map_struct_fields(
+ source_fields,
+ target_fields,
+ Some(&mut mapped_source_fields),
+ )?;
if !source_col.is_empty() && source_col.null_count() == source_col.len() {
return Ok(new_null_array(
@@ -89,14 +95,15 @@ fn cast_struct_column(
let num_rows = source_col.len();
// Iterate target fields and pick source child by name when present.
- for target_child_field in target_fields.iter() {
+ for (source_child_opt, target_child_field) in
+ mapped_source_fields.iter().zip(target_fields.iter())
+ {
fields.push(Arc::clone(target_child_field));
- let source_child_opt =
- source_struct.column_by_name(target_child_field.name());
-
match source_child_opt {
- Some(source_child_col) => {
+ Some(source_child_col_idx) => {
+ let source_child_col = source_struct.column(*source_child_col_idx);
+
let adapted_child = cast_column(
source_child_col,
target_child_field.data_type(),
@@ -111,13 +118,19 @@ fn cast_struct_column(
arrays.push(adapted_child);
}
None => {
+ // No need to check if `target_child_field` is nullable here
+ // The call to `map_struct_fields` will have done that for us
arrays.push(new_null_array(target_child_field.data_type(), num_rows));
}
}
}
- let struct_array =
- StructArray::try_new(fields.into(), arrays, source_struct.nulls().cloned())?;
+ let struct_array = StructArray::try_new_with_length(
+ fields.into(),
+ arrays,
+ source_struct.nulls().cloned(),
+ source_struct.len(),
+ )?;
Ok(Arc::new(struct_array))
} else {
// Return error if source is not a struct type
@@ -670,6 +683,8 @@ fn cast_dictionary_column(
/// - **Field Matching**: Fields are matched by name (case-sensitive)
/// - **Missing Target Fields**: Allowed - will be filled with null values during casting
/// - **Extra Source Fields**: Allowed - will be ignored during casting
+/// - **At Least One Mapped Field**: source and target must share at least one field by name
+/// (unless source is empty)
/// - **Type Compatibility**: Each matching field must be castable using Arrow's type system
/// - **Nested Structs**: Recursively validates nested struct compatibility
///
@@ -698,23 +713,49 @@ pub fn validate_struct_compatibility(
source_fields: &[FieldRef],
target_fields: &[FieldRef],
) -> Result<()> {
- let has_overlap = has_one_of_more_common_fields(source_fields, target_fields);
- if !has_overlap {
- return _plan_err!(
- "Cannot cast struct with {} fields to {} fields because there is no field name overlap",
- source_fields.len(),
- target_fields.len()
- );
+ map_struct_fields(source_fields, target_fields, None)
+}
+
+/// Performs the mapping of struct fields, checking compatibility and handling missing or extra fields.
+///
+/// # Arguments
+/// * `source_fields` - Fields from the source struct type
+/// * `target_fields` - Fields from the target struct type
+/// * `mapped_source_fields` - An optional `Vec` into which the indexes of the matching source fields
+/// are written in the order of the target fields
+///
+/// # Returns
+/// * `Ok(())` if the structs are compatible for casting
+/// * `Err(DataFusionError)` with detailed error message if incompatible
+fn map_struct_fields(
+ source_fields: &[FieldRef],
+ target_fields: &[FieldRef],
+ mut mapped_source_fields: Option<&mut Vec<Option<usize>>>,
+) -> Result<()> {
+ if !source_fields.is_empty() {
+ let has_overlap = has_one_of_more_common_fields(source_fields, target_fields);
+ if !has_overlap {
+ return _plan_err!(
+ "Cannot cast struct with {} fields to {} fields because there is no field name overlap",
+ source_fields.len(),
+ target_fields.len()
+ );
+ }
}
// Check compatibility for each target field
for target_field in target_fields {
// Look for matching field in source by name
- if let Some(source_field) = source_fields
+ let source_field_opt = source_fields
.iter()
- .find(|f| f.name() == target_field.name())
+ .enumerate()
+ .find(|(_, f)| f.name() == target_field.name());
+ let source_field_idx = if let Some((source_field_idx, source_field)) =
+ source_field_opt
{
validate_field_compatibility(source_field, target_field)?;
+
+ Some(source_field_idx)
} else {
// Target field is missing from source
// If it's non-nullable, we cannot fill it with NULL
@@ -725,6 +766,12 @@ pub fn validate_struct_compatibility(
target_field.name()
);
}
+
+ None
+ };
+
+ if let Some(mapped_fields) = mapped_source_fields.as_mut() {
+ mapped_fields.push(source_field_idx);
}
}
@@ -1153,6 +1200,8 @@ mod tests {
buffer::{NullBuffer, OffsetBuffer, ScalarBuffer},
datatypes::{DataType, Field, FieldRef, Int32Type},
};
+ use arrow_schema::Fields;
+
/// Macro to extract and downcast a column from a StructArray
macro_rules! get_column_as {
($struct_array:expr, $column_name:expr, $array_type:ty) => {
@@ -1289,6 +1338,65 @@ mod tests {
assert!(error_msg.contains("Cannot cast struct field 'a'"));
}
+ #[test]
+ fn test_cast_struct_empty_source_and_target() {
+ // Source and target struct: {} (no fields)
+ let source =
+ StructArray::try_new_with_length(Fields::empty(), vec![], None, 1024)
+ .unwrap();
+ let source = Arc::new(source) as ArrayRef;
+
+ // Target struct: {}
+ let target_field = struct_field("s", vec![]);
+
+ // This should succeed - an empty struct is compatible with itself
+ let result =
+ cast_column(&source, target_field.data_type(), &DEFAULT_CAST_OPTIONS)
+ .unwrap();
+ let result = result.as_any().downcast_ref::<StructArray>().unwrap();
+
+ assert_eq!(result.len(), source.len());
+ assert_eq!(result.data_type(), target_field.data_type());
+ }
+
+ #[test]
+ fn test_cast_struct_compatibility_empty_source_nullable_target() {
+ // Source and target struct: {} (no fields)
+ let source =
+ StructArray::try_new_with_length(Fields::empty(), vec![], None, 1024)
+ .unwrap();
+ let source = Arc::new(source) as ArrayRef;
+
+ // Target struct: {a: Int32}
+ let target_field = struct_field("s", vec![field("a", DataType::Int32)]);
+
+ // This should succeed - all target fields are nullable
+ let result =
+ cast_column(&source, target_field.data_type(), &DEFAULT_CAST_OPTIONS)
+ .unwrap();
+ let result = result.as_any().downcast_ref::<StructArray>().unwrap();
+
+ assert_eq!(result.len(), source.len());
+ assert_eq!(result.data_type(), target_field.data_type());
+ }
+
+ #[test]
+ fn test_cast_struct_compatibility_empty_source_non_nullable_target() {
+ // Source and target struct: {} (no fields)
+ let source =
+ StructArray::try_new_with_length(Fields::empty(), vec![], None, 1024)
+ .unwrap();
+ let source = Arc::new(source) as ArrayRef;
+
+ // Target struct: {a: Int32}
+ let target_field = struct_field("s", vec![non_null_field("a", DataType::Int32)]);
+
+ // This should succeed - all target fields are nullable
+ let result =
+ cast_column(&source, target_field.data_type(), &DEFAULT_CAST_OPTIONS);
+ assert!(result.is_err());
+ }
+
#[test]
fn test_validate_struct_compatibility_incompatible_types() {
// Source struct: {field1: Binary, field2: String}
@@ -1551,6 +1659,46 @@ mod tests {
assert!(result.is_ok());
}
+ #[test]
+ fn test_validate_struct_compatibility_empty_source_and_target() {
+ // Source and target struct: {} (no fields)
+ let source_fields = vec![];
+ let target_fields = vec![];
+
+ // This should succeed - an empty struct is compatible with itself
+ let result = validate_struct_compatibility(&source_fields, &target_fields);
+ assert!(result.is_ok());
+ }
+
+ #[test]
+ fn test_validate_struct_compatibility_empty_source_nullable_target() {
+ // Source and target struct: {} (no fields)
+ let source_fields = vec![];
+
+ // Target struct: {a: Int32}
+ let target_fields = vec![arc_field("a", DataType::Int32)];
+
+ // This should succeed - all target fields are nullable
+ let result = validate_struct_compatibility(&source_fields, &target_fields);
+ assert!(result.is_ok());
+ }
+
+ #[test]
+ fn test_validate_struct_compatibility_empty_source_non_nullable_target() {
+ // Source and target struct: {} (no fields)
+ let source_fields = vec![];
+
+ // Target struct: {a: Int32 NOT NULL}
+ let target_fields = vec![
+ arc_field("a", DataType::Int32),
+ Arc::new(non_null_field("b", DataType::Utf8)),
+ ];
+
+ // This should fail - not all target fields are nullable
+ let result = validate_struct_compatibility(&source_fields, &target_fields);
+ assert!(result.is_err());
+ }
+
#[test]
fn test_cast_nested_struct_with_extra_and_missing_fields() {
// Source inner struct has fields a, b, extra