Apache DataFusion SQL Query Engine
Add FixedSizeList support for recursive struct schema adaptation (#22980)
kosiewf1ab86dad406
diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rsindex cdd6215d08..e915b91b91 100644--- a/datafusion/common/src/nested_struct.rs+++ b/datafusion/common/src/nested_struct.rs@@ -18,9 +18,10 @@use crate::error::{_plan_err, Result};use arrow::{array::{- Array, ArrayRef, DictionaryArray, GenericListArray, GenericListViewArray,- StructArray, downcast_integer, new_null_array,+ Array, ArrayRef, AsArray, DictionaryArray, FixedSizeListArray, GenericListArray,+ GenericListViewArray, StructArray, downcast_integer, make_array, new_null_array,},+ buffer::NullBuffer,compute::{CastOptions, can_cast_types, cast_with_options},datatypes::{DataType, DataType::Struct, Field, FieldRef},};@@ -58,9 +59,7 @@ fn cast_struct_column(target_fields: &[Arc<Field>],cast_options: &CastOptions,) -> Result<ArrayRef> {- if source_col.data_type() == &DataType::Null- || (!source_col.is_empty() && source_col.null_count() == source_col.len())- {+ if source_col.data_type() == &DataType::Null {return Ok(new_null_array(&Struct(target_fields.to_vec().into()),source_col.len(),@@ -70,6 +69,14 @@ fn cast_struct_column(if let Some(source_struct) = source_col.as_any().downcast_ref::<StructArray>() {let source_fields = source_struct.fields();validate_struct_compatibility(source_fields, target_fields)?;++ if !source_col.is_empty() && source_col.null_count() == source_col.len() {+ return Ok(new_null_array(+ &Struct(target_fields.to_vec().into()),+ source_col.len(),+ ));+ }+let mut fields: Vec<Arc<Field>> = Vec::with_capacity(target_fields.len());let mut arrays: Vec<ArrayRef> = Vec::with_capacity(target_fields.len());let num_rows = source_col.len();@@ -183,6 +190,15 @@ pub fn cast_column((DataType::LargeList(_), DataType::LargeList(target_inner)) => {cast_list_column::<i64>(source_col, target_inner, cast_options)}+ (+ DataType::FixedSizeList(_, source_list_size),+ DataType::FixedSizeList(target_inner, target_list_size),+ ) if source_list_size == target_list_size => cast_fixed_size_list_column(+ source_col,+ target_inner,+ *target_list_size,+ cast_options,+ ),(DataType::ListView(_), DataType::ListView(target_inner)) => {cast_list_view_column::<i32>(source_col, target_inner, cast_options)}@@ -208,15 +224,7 @@ fn cast_list_column<O: arrow::array::OffsetSizeTrait>(target_inner_field: &FieldRef,cast_options: &CastOptions,) -> Result<ArrayRef> {- let source_list = source_col- .as_any()- .downcast_ref::<GenericListArray<O>>()- .ok_or_else(|| {- crate::error::DataFusionError::Plan(format!(- "Expected list array but got {}",- source_col.data_type()- ))- })?;+ let source_list = source_col.as_list::<O>();let cast_values = cast_column(source_list.values(),@@ -238,15 +246,7 @@ fn cast_list_view_column<O: arrow::array::OffsetSizeTrait>(target_inner_field: &FieldRef,cast_options: &CastOptions,) -> Result<ArrayRef> {- let source_list = source_col- .as_any()- .downcast_ref::<GenericListViewArray<O>>()- .ok_or_else(|| {- crate::error::DataFusionError::Plan(format!(- "Expected list view array but got {}",- source_col.data_type()- ))- })?;+ let source_list = source_col.as_list_view::<O>();let cast_values = cast_column(source_list.values(),@@ -264,6 +264,82 @@ fn cast_list_view_column<O: arrow::array::OffsetSizeTrait>(Ok(Arc::new(result))}+fn cast_fixed_size_list_column(+ source_col: &ArrayRef,+ target_inner_field: &FieldRef,+ target_list_size: i32,+ cast_options: &CastOptions,+) -> Result<ArrayRef> {+ let source_list = source_col.as_fixed_size_list();++ let source_values = source_list.values();+ let target_type = target_inner_field.data_type();++ let cast_values = match cast_column(source_values, target_type, cast_options) {+ Ok(cast_values) => cast_values,+ Err(error) => match cast_fixed_size_list_values_with_parent_nulls(+ source_values,+ target_type,+ cast_options,+ source_list.nulls(),+ target_list_size,+ ) {+ Some(masked_cast) => masked_cast?,+ None => return Err(error),+ },+ };++ Ok(Arc::new(FixedSizeListArray::try_new(+ Arc::clone(target_inner_field),+ target_list_size,+ cast_values,+ source_list.nulls().cloned(),+ )?))+}++fn cast_fixed_size_list_values_with_parent_nulls(+ source_values: &ArrayRef,+ target_type: &DataType,+ cast_options: &CastOptions,+ parent_nulls: Option<&NullBuffer>,+ list_size: i32,+) -> Option<Result<ArrayRef>> {+ let parent_nulls = parent_nulls.filter(|nulls| nulls.null_count() > 0)?;++ // FixedSizeList stores child slots for null parent lists. Those child+ // values are semantically hidden, but recursive casts still inspect them.+ let hidden_child_nulls = parent_nulls.expand(list_size as usize);+ let masked_values = mask_array_values(source_values, &hidden_child_nulls);+ Some(masked_values.and_then(|values| cast_column(&values, target_type, cast_options)))+}++fn mask_array_values(+ values: &ArrayRef,+ additional_nulls: &NullBuffer,+) -> Result<ArrayRef> {+ let nulls = NullBuffer::union(values.nulls(), Some(additional_nulls));++ if let Some(struct_array) = values.as_any().downcast_ref::<StructArray>() {+ let struct_nulls = nulls+ .as_ref()+ .expect("additional nulls always produce nulls");+ let arrays = struct_array+ .columns()+ .iter()+ .map(|child| mask_array_values(child, struct_nulls))+ .collect::<Result<Vec<_>>>()?;+ return Ok(Arc::new(StructArray::new(+ struct_array.fields().clone(),+ arrays,+ nulls,+ )));+ }++ Ok(make_array(+ values.to_data().into_builder().nulls(nulls).build()?,+ ))+}+fn cast_dictionary_column(source_col: &ArrayRef,source_key_type: &DataType,@@ -425,6 +501,12 @@ pub fn validate_data_type_compatibility((Struct(source_nested), Struct(target_nested)) => {validate_struct_compatibility(source_nested, target_nested)?;}+ (+ DataType::FixedSizeList(s, source_list_size),+ DataType::FixedSizeList(t, target_list_size),+ ) if source_list_size == target_list_size => {+ validate_field_compatibility(s, t)?;+ }(DataType::List(s), DataType::List(t))| (DataType::LargeList(s), DataType::LargeList(t))| (DataType::ListView(s), DataType::ListView(t))@@ -460,8 +542,8 @@ pub fn validate_data_type_compatibility(/// name-based nested struct casting logic, rather than Arrow's standard cast.////// This is the case when both types are struct types, or both are the same-/// container type (List, LargeList, ListView, LargeListView, Dictionary) wrapping-/// types that recursively contain structs.+/// container type (List, LargeList, equal-width FixedSizeList, ListView,+/// LargeListView, Dictionary) wrapping types that recursively contain structs.////// Use this predicate at both planning time (to decide whether to apply struct/// compatibility validation) and execution time (to decide whether to route@@ -472,6 +554,12 @@ pub fn requires_nested_struct_cast() -> bool {match (source_type, target_type) {(Struct(_), Struct(_)) => true,+ (+ DataType::FixedSizeList(s, source_list_size),+ DataType::FixedSizeList(t, target_list_size),+ ) if source_list_size == target_list_size => {+ requires_nested_struct_cast(s.data_type(), t.data_type())+ }(DataType::List(s), DataType::List(t))| (DataType::LargeList(s), DataType::LargeList(t))| (DataType::ListView(s), DataType::ListView(t))@@ -508,8 +596,9 @@ mod tests {use crate::{assert_contains, format::DEFAULT_CAST_OPTIONS};use arrow::{array::{- BinaryArray, Int32Array, Int32Builder, Int64Array, ListArray, ListViewArray,- MapArray, MapBuilder, NullArray, StringArray, StringBuilder,+ BinaryArray, FixedSizeListArray, Int32Array, Int32Builder, Int64Array,+ ListArray, ListViewArray, MapArray, MapBuilder, NullArray, StringArray,+ StringBuilder,},buffer::{NullBuffer, ScalarBuffer},datatypes::{DataType, Field, FieldRef, Int32Type},@@ -1307,6 +1396,275 @@ mod tests {assert!(b_col.iter().all(|v| v.is_none()));}+ fn fixed_size_list_struct_field(fields: Vec<(&str, DataType)>) -> FieldRef {+ arc_field(+ "item",+ struct_type(+ fields+ .into_iter()+ .map(|(name, data_type)| field(name, data_type))+ .collect(),+ ),+ )+ }++ fn create_fixed_size_list_test_fields(+ source_struct_fields: Vec<(&str, DataType)>,+ target_struct_fields: Vec<(&str, DataType)>,+ ) -> (FieldRef, FieldRef) {+ (+ fixed_size_list_struct_field(source_struct_fields),+ fixed_size_list_struct_field(target_struct_fields),+ )+ }++ fn fixed_size_list_struct_values(+ array: &ArrayRef,+ ) -> (&FixedSizeListArray, &StructArray) {+ let list = array.as_any().downcast_ref::<FixedSizeListArray>().unwrap();+ let values = list+ .values()+ .as_any()+ .downcast_ref::<StructArray>()+ .unwrap();+ (list, values)+ }++ #[test]+ fn test_cast_fixed_size_list_struct() {+ let struct_arr = StructArray::from(vec![(+ arc_field("a", DataType::Int32),+ Arc::new(Int32Array::from(vec![1, 2, 3, 4])) as ArrayRef,+ )]);++ let (source_field, target_field) = create_fixed_size_list_test_fields(+ vec![("a", DataType::Int32)],+ vec![("a", DataType::Int64), ("b", DataType::Utf8)],+ );+ let source_col: ArrayRef = Arc::new(FixedSizeListArray::new(+ source_field,+ 2,+ Arc::new(struct_arr),+ Some(NullBuffer::from(vec![true, false])),+ ));+ let target_type = DataType::FixedSizeList(target_field, 2);++ let result =+ cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS).unwrap();+ let (result_list, struct_values) = fixed_size_list_struct_values(&result);+ assert_eq!(result_list.len(), 2);+ assert!(result_list.is_valid(0));+ assert!(result_list.is_null(1));+ let a_col = get_column_as!(&struct_values, "a", Int64Array);+ assert_eq!(a_col.values(), &[1, 2, 3, 4]);+ let b_col = get_column_as!(&struct_values, "b", StringArray);+ assert!(b_col.iter().all(|v| v.is_none()));+ }++ #[test]+ fn test_validate_fixed_size_list_struct_compatibility() {+ let (source_field, target_field) = create_fixed_size_list_test_fields(+ vec![("a", DataType::Int32)],+ vec![("a", DataType::Int64), ("b", DataType::Utf8)],+ );+ let source = DataType::FixedSizeList(source_field, 2);+ let target = DataType::FixedSizeList(target_field, 2);++ assert!(requires_nested_struct_cast(&source, &target));+ assert!(validate_data_type_compatibility("col", &source, &target).is_ok());+ }++ #[test]+ fn test_validate_fixed_size_list_struct_missing_non_nullable_field_rejected() {+ let (source_field, _) = create_fixed_size_list_test_fields(+ vec![("a", DataType::Int32)],+ vec![("a", DataType::Int64), ("b", DataType::Utf8)],+ );+ let source = DataType::FixedSizeList(source_field, 2);+ let target = DataType::FixedSizeList(+ arc_field(+ "item",+ struct_type(vec![+ field("a", DataType::Int32),+ non_null_field("b", DataType::Utf8),+ ]),+ ),+ 2,+ );++ let error = validate_data_type_compatibility("col", &source, &target)+ .unwrap_err()+ .to_string();+ assert_contains!(+ error,+ "target field 'b' is non-nullable but missing from source"+ );+ }++ #[test]+ fn test_fixed_size_list_struct_size_mismatch_rejected() {+ let source_field = fixed_size_list_struct_field(vec![("a", DataType::Int32)]);+ let target_field = Arc::clone(&source_field);+ let source_type = DataType::FixedSizeList(Arc::clone(&source_field), 2);+ let target_type = DataType::FixedSizeList(target_field, 3);++ let validation_error =+ validate_data_type_compatibility("col", &source_type, &target_type)+ .unwrap_err()+ .to_string();+ assert_contains!(validation_error, "Cannot cast struct field 'col'");++ let struct_arr = StructArray::from(vec![(+ arc_field("a", DataType::Int32),+ Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef,+ )]);+ let source_col: ArrayRef = Arc::new(FixedSizeListArray::new(+ source_field,+ 2,+ Arc::new(struct_arr),+ None,+ ));++ let runtime_error = cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS)+ .unwrap_err()+ .to_string();+ assert_contains!(+ runtime_error,+ "cannot cast fixed-size-list to fixed-size-list with different size"+ );+ }++ #[test]+ fn test_cast_fixed_size_list_struct_all_null() {+ let (source_field, target_field) = create_fixed_size_list_test_fields(+ vec![("a", DataType::Int32)],+ vec![("a", DataType::Int64), ("b", DataType::Utf8)],+ );+ let source_col: ArrayRef =+ Arc::new(FixedSizeListArray::new_null(source_field, 2, 2));+ let target_type = DataType::FixedSizeList(target_field, 2);++ let result =+ cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS).unwrap();+ let (result_list, struct_values) = fixed_size_list_struct_values(&result);+ assert_eq!(result_list.null_count(), 2);+ let a_col = get_column_as!(&struct_values, "a", Int64Array);+ let b_col = get_column_as!(&struct_values, "b", StringArray);+ assert!(a_col.iter().all(|v| v.is_none()));+ assert!(b_col.iter().all(|v| v.is_none()));+ }++ #[test]+ fn test_fixed_size_list_struct_planner_runtime_parity_on_incompatible_type() {+ let source_field =+ arc_field("item", struct_type(vec![field("a", DataType::Binary)]));+ let target_field =+ arc_field("item", struct_type(vec![field("a", DataType::Int32)]));+ let source_type = DataType::FixedSizeList(Arc::clone(&source_field), 2);+ let target_type = DataType::FixedSizeList(target_field, 2);+ let validation_error =+ validate_data_type_compatibility("col", &source_type, &target_type)+ .unwrap_err()+ .to_string();+ assert_contains!(validation_error, "Cannot cast struct field 'a'");++ let struct_arr = StructArray::from(vec![(+ arc_field("a", DataType::Binary),+ Arc::new(BinaryArray::from(vec![+ Some(b"x".as_ref()),+ Some(b"y".as_ref()),+ ])) as ArrayRef,+ )]);+ let source_col: ArrayRef = Arc::new(FixedSizeListArray::new(+ source_field,+ 2,+ Arc::new(struct_arr),+ None,+ ));++ let runtime_error = cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS)+ .unwrap_err()+ .to_string();+ assert_contains!(runtime_error, "Cannot cast struct field 'a'");+ }++ #[test]+ fn test_cast_fixed_size_list_struct_missing_non_nullable_field_runtime_rejected() {+ let source_field =+ arc_field("item", struct_type(vec![field("a", DataType::Int32)]));+ let target_field = arc_field(+ "item",+ struct_type(vec![+ field("a", DataType::Int32),+ non_null_field("b", DataType::Utf8),+ ]),+ );+ let source_col: ArrayRef =+ Arc::new(FixedSizeListArray::new_null(source_field, 2, 1));+ let target_type = DataType::FixedSizeList(target_field, 2);++ let error = cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS)+ .unwrap_err()+ .to_string();+ assert_contains!(+ error,+ "target field 'b' is non-nullable but missing from source"+ );+ }++ #[test]+ fn test_cast_fixed_size_list_returns_error_for_non_nullable_child() {+ let source_field = Arc::new(Field::new("item", DataType::Int32, true));+ let target_field = Arc::new(Field::new("item", DataType::Int32, false));+ let source_col: ArrayRef = Arc::new(FixedSizeListArray::new(+ source_field,+ 2,+ Arc::new(Int32Array::from(vec![None, Some(1)])),+ None,+ ));+ let target_type = DataType::FixedSizeList(target_field, 2);++ let error = cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS)+ .unwrap_err()+ .to_string();+ assert_contains!(error, "Found unmasked nulls for non-nullable");+ }++ #[test]+ fn test_cast_sliced_fixed_size_list_struct_ignores_hidden_child_values() {+ let source_field =+ arc_field("item", struct_type(vec![field("a", DataType::Utf8)]));+ let target_field =+ arc_field("item", struct_type(vec![field("a", DataType::Int32)]));+ let struct_arr = StructArray::from(vec![(+ arc_field("a", DataType::Utf8),+ Arc::new(StringArray::from(vec![+ "0", "0", "not_int", "also_bad", "1", "2",+ ])) as ArrayRef,+ )]);+ let source_col: ArrayRef = Arc::new(+ FixedSizeListArray::new(+ source_field,+ 2,+ Arc::new(struct_arr),+ Some(NullBuffer::from(vec![true, false, true])),+ )+ .slice(1, 2),+ );+ let target_type = DataType::FixedSizeList(target_field, 2);++ let result =+ cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS).unwrap();+ let (result_list, struct_values) = fixed_size_list_struct_values(&result);+ assert!(result_list.is_null(0));+ assert!(result_list.is_valid(1));+ let a_col = get_column_as!(&struct_values, "a", Int32Array);+ assert!(a_col.is_null(0));+ assert!(a_col.is_null(1));+ assert_eq!(a_col.value(2), 1);+ assert_eq!(a_col.value(3), 2);+ }+#[test]fn test_requires_nested_struct_cast() {let s1 = struct_type(vec![field("a", DataType::Int32)]);@@ -1322,8 +1680,12 @@ mod tests {&DataType::Dictionary(Box::new(DataType::Int32), Box::new(s2.clone())),));assert!(requires_nested_struct_cast(- &DataType::ListView(arc_field("item", s1)),- &DataType::ListView(arc_field("item", s2)),+ &DataType::ListView(arc_field("item", s1.clone())),+ &DataType::ListView(arc_field("item", s2.clone())),+ ));+ assert!(requires_nested_struct_cast(+ &DataType::FixedSizeList(arc_field("item", s1), 2),+ &DataType::FixedSizeList(arc_field("item", s2), 2),));// Non-struct types should return false.@@ -1335,5 +1697,9 @@ mod tests {&DataType::List(arc_field("item", DataType::Int32)),&DataType::List(arc_field("item", DataType::Int64)),));+ assert!(!requires_nested_struct_cast(+ &DataType::FixedSizeList(arc_field("item", DataType::Int32), 2),+ &DataType::FixedSizeList(arc_field("item", DataType::Int64), 2),+ ));}}diff --git a/datafusion/core/tests/parquet/expr_adapter.rs b/datafusion/core/tests/parquet/expr_adapter.rsindex fd70d74a91..535828fa29 100644--- a/datafusion/core/tests/parquet/expr_adapter.rs+++ b/datafusion/core/tests/parquet/expr_adapter.rs@@ -18,8 +18,8 @@use std::sync::Arc;use arrow::array::{- Array, ArrayRef, BooleanArray, Int32Array, Int64Array, LargeListArray, ListArray,- RecordBatch, StringArray, StructArray, record_batch,+ Array, ArrayRef, BooleanArray, FixedSizeListArray, Int32Array, Int64Array,+ LargeListArray, ListArray, RecordBatch, StringArray, StructArray, record_batch,};use arrow::buffer::OffsetBuffer;use arrow::compute::concat_batches;@@ -60,13 +60,19 @@ async fn write_parquet(batch: RecordBatch, store: Arc<dyn ObjectStore>, path: &senum NestedListKind {List,LargeList,+ FixedSizeList,}+const FIXED_SIZE_LIST_LEN: usize = 2;+impl NestedListKind {fn field_data_type(self, item_field: Arc<Field>) -> DataType {match self {Self::List => DataType::List(item_field),Self::LargeList => DataType::LargeList(item_field),+ Self::FixedSizeList => {+ DataType::FixedSizeList(item_field, FIXED_SIZE_LIST_LEN as i32)+ }}}@@ -89,6 +95,19 @@ impl NestedListKind {values,None,)),+ Self::FixedSizeList => {+ assert_eq!(+ lengths.as_slice(),+ &[FIXED_SIZE_LIST_LEN],+ "FixedSizeList fixtures must contain exactly {FIXED_SIZE_LIST_LEN} elements per row"+ );+ Arc::new(FixedSizeListArray::new(+ item_field,+ FIXED_SIZE_LIST_LEN as i32,+ values,+ None,+ ))+ }}}@@ -96,6 +115,7 @@ impl NestedListKind {match self {Self::List => "list",Self::LargeList => "large_list",+ Self::FixedSizeList => "fixed_size_list",}}}@@ -277,7 +297,8 @@ fn nested_list_table_schema(}// Helper to extract message values from a nested list column.-// Returns the values at indices 0 and 1 from either a ListArray or LargeListArray.+// Returns the values at indices 0 and 1 from either a ListArray, LargeListArray,+// or FixedSizeListArray.fn extract_nested_list_values(kind: NestedListKind,column: &ArrayRef,@@ -297,7 +318,50 @@ fn extract_nested_list_values(.expect("messages should be a LargeListArray");(list.value(0), list.value(1))}+ NestedListKind::FixedSizeList => {+ let list = column+ .as_any()+ .downcast_ref::<FixedSizeListArray>()+ .expect("messages should be a FixedSizeListArray");+ (list.value(0), list.value(1))+ }+ }+}++fn evolved_messages(kind: NestedListKind) -> Vec<NestedMessageRow<'static>> {+ let mut messages = vec![NestedMessageRow {+ id: 30,+ name: "gamma",+ chain: Some("eth"),+ ignored: Some(99),+ }];+ if matches!(kind, NestedListKind::FixedSizeList) {+ messages.push(NestedMessageRow {+ id: 40,+ name: "delta",+ chain: Some("doge"),+ ignored: Some(100),+ });+ }+ messages+}++fn error_messages(kind: NestedListKind) -> Vec<NestedMessageRow<'static>> {+ let mut messages = vec![NestedMessageRow {+ id: 10,+ name: "alpha",+ chain: Some("eth"),+ ignored: None,+ }];+ if matches!(kind, NestedListKind::FixedSizeList) {+ messages.push(NestedMessageRow {+ id: 20,+ name: "beta",+ chain: Some("doge"),+ ignored: None,+ });}+ messages}// Helper to set up a nested list test fixture.@@ -352,15 +416,11 @@ async fn assert_nested_list_struct_schema_evolution(kind: NestedListKind) -> Res);// new.parquet shape: messages item struct adds nullable `chain` and extra `ignored`.+ let new_messages = evolved_messages(kind);let new_batch = nested_messages_batch(kind,2,- &[NestedMessageRow {- id: 30,- name: "gamma",- chain: Some("eth"),- ignored: Some(99),- }],+ &new_messages,&message_fields(DataType::Utf8, true, true, true),);@@ -429,7 +489,12 @@ async fn assert_nested_list_struct_schema_evolution(kind: NestedListKind) -> Res.as_any().downcast_ref::<StringArray>().unwrap();- assert_eq!(new_chain.iter().collect::<Vec<_>>(), vec![Some("eth")]);+ let expected_new_chain = if matches!(kind, NestedListKind::FixedSizeList) {+ vec![Some("eth"), Some("doge")]+ } else {+ vec![Some("eth")]+ };+ assert_eq!(new_chain.iter().collect::<Vec<_>>(), expected_new_chain);let projected = ctx.sql(@@ -863,12 +928,12 @@ async fn test_struct_schema_evolution_projection_and_filter() -> Result<()> {Ok(())}-/// Macro to generate paired test functions for List and LargeList variants.-/// Expands to two `#[tokio::test]` functions with the specified names.-macro_rules! test_struct_schema_evolution_pair {+/// Macro to generate schema evolution tests for list-like variants.+macro_rules! test_struct_schema_evolution_variants {(list: $list_test:ident,large_list: $large_list_test:ident,+ fixed_size_list: $fixed_size_list_test:ident,fn: $assertion_fn:path $(, args: $($arg:expr),+)?) => {#[tokio::test]@@ -880,10 +945,16 @@ macro_rules! test_struct_schema_evolution_pair {async fn $large_list_test() {$assertion_fn(NestedListKind::LargeList $(, $($arg),+)?).await;}++ #[tokio::test]+ async fn $fixed_size_list_test() {+ $assertion_fn(NestedListKind::FixedSizeList $(, $($arg),+)?).await;+ }};(list: $list_test:ident,large_list: $large_list_test:ident,+ fixed_size_list: $fixed_size_list_test:ident,fn_result: $assertion_fn:path) => {#[tokio::test]@@ -895,31 +966,34 @@ macro_rules! test_struct_schema_evolution_pair {async fn $large_list_test() -> Result<()> {$assertion_fn(NestedListKind::LargeList).await}++ #[tokio::test]+ async fn $fixed_size_list_test() -> Result<()> {+ $assertion_fn(NestedListKind::FixedSizeList).await+ }};}-test_struct_schema_evolution_pair!(+test_struct_schema_evolution_variants!(list: test_list_struct_schema_evolution_end_to_end,large_list: test_large_list_struct_schema_evolution_end_to_end,+ fixed_size_list: test_fixed_size_list_struct_schema_evolution_end_to_end,fn_result: assert_nested_list_struct_schema_evolution);async fn assert_nested_list_struct_schema_evolution_errors(kind: NestedListKind,+ source_includes_chain: bool,chain_type: DataType,chain_nullable: bool,expected_error: &str,) {+ let messages = error_messages(kind);let batch = nested_messages_batch(kind,1,- &[NestedMessageRow {- id: 10,- name: "alpha",- chain: Some("eth"),- ignored: None,- }],- &message_fields(DataType::Utf8, true, true, false),+ &messages,+ &message_fields(DataType::Utf8, true, source_includes_chain, false),);let table_schema =@@ -949,6 +1023,7 @@ async fn assert_nested_list_struct_schema_evolution_errors(async fn assert_non_nullable_missing_chain_field_fails(kind: NestedListKind) {assert_nested_list_struct_schema_evolution_errors(kind,+ false,DataType::Utf8,false,"non-nullable",@@ -959,6 +1034,7 @@ async fn assert_non_nullable_missing_chain_field_fails(kind: NestedListKind) {async fn assert_incompatible_chain_field_fails(kind: NestedListKind) {assert_nested_list_struct_schema_evolution_errors(kind,+ true,incompatible_chain_type(),true,"Cannot cast struct field 'chain'",@@ -970,15 +1046,17 @@ fn incompatible_chain_type() -> DataType {DataType::Struct(vec![Arc::new(Field::new("value", DataType::Utf8, true))].into())}-test_struct_schema_evolution_pair!(+test_struct_schema_evolution_variants!(list: test_list_struct_schema_evolution_non_nullable_missing_field_fails,large_list: test_large_list_struct_schema_evolution_non_nullable_missing_field_fails,+ fixed_size_list: test_fixed_size_list_struct_schema_evolution_non_nullable_missing_field_fails,fn: assert_non_nullable_missing_chain_field_fails);-test_struct_schema_evolution_pair!(+test_struct_schema_evolution_variants!(list: test_list_struct_schema_evolution_incompatible_field_fails,large_list: test_large_list_struct_schema_evolution_incompatible_field_fails,+ fixed_size_list: test_fixed_size_list_struct_schema_evolution_incompatible_field_fails,fn: assert_incompatible_chain_field_fails);