Apache DataFusion SQL Query Engine
Add nested struct schema evolution support for Map types (#23914)
kosiewe2bdc2c21889
diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rsindex e915b91b91..3f5e4ab4b7 100644--- a/datafusion/common/src/nested_struct.rs+++ b/datafusion/common/src/nested_struct.rs@@ -19,13 +19,17 @@ use crate::error::{_plan_err, Result};use arrow::{array::{Array, ArrayRef, AsArray, DictionaryArray, FixedSizeListArray, GenericListArray,- GenericListViewArray, StructArray, downcast_integer, make_array, new_null_array,+ GenericListViewArray, MapArray, StructArray, UInt64Array, downcast_integer,+ make_array, new_null_array,},buffer::NullBuffer,- compute::{CastOptions, can_cast_types, cast_with_options},+ compute::{CastOptions, can_cast_types, cast_with_options, take},datatypes::{DataType, DataType::Struct, Field, FieldRef},};-use std::{collections::HashSet, sync::Arc};+use std::{+ collections::{HashMap, HashSet},+ sync::Arc,+};/// Cast a struct column to match target struct fields, handling nested structs recursively.///@@ -110,7 +114,7 @@ fn cast_struct_column(}let struct_array =- StructArray::new(fields.into(), arrays, source_struct.nulls().cloned());+ StructArray::try_new(fields.into(), arrays, source_struct.nulls().cloned())?;Ok(Arc::new(struct_array))} else {// Return error if source is not a struct type@@ -205,6 +209,18 @@ pub fn cast_column((DataType::LargeListView(_), DataType::LargeListView(target_inner)) => {cast_list_view_column::<i64>(source_col, target_inner, cast_options)}+ (DataType::Map(_, _), DataType::Map(target_entries, target_sorted)) => {+ if requires_nested_struct_cast(source_col.data_type(), target_type) {+ cast_map_column(+ source_col.as_map(),+ target_entries,+ *target_sorted,+ cast_options,+ )+ } else {+ Ok(cast_with_options(source_col, target_type, cast_options)?)+ }+ }(DataType::Dictionary(source_key_type, _),DataType::Dictionary(target_key_type, target_value_type),@@ -340,6 +356,79 @@ fn mask_array_values())}+/// Casts Map children by their semantic positions: key at index 0 and value at+/// index 1. Technical entry field names are taken from the target schema.+///+/// Nested Struct fields within keys and values are still matched by name. Key+/// evolution is restricted by [`validate_map_key_compatibility`] so it cannot+/// remove identity-bearing fields; sorted Maps require an unchanged key type.+///+/// Map-specific compaction is necessary because sliced Maps retain unreachable+/// entries and null parents hide entries in their backing array, which `keys()`+/// and `values()` expose to recursive casts. List-family casts currently+/// preserve offsets and cast their backing values directly; consistent handling+/// is tracked in #24506.+fn cast_map_column(+ source_map: &MapArray,+ target_entries: &FieldRef,+ target_sorted: bool,+ cast_options: &CastOptions,+) -> Result<ArrayRef> {+ let DataType::Map(source_entries, source_sorted) = source_map.data_type() else {+ unreachable!("MapArray data type must be Map")+ };+ let (target_key, target_value) = validate_map_compatibility(+ source_entries,+ *source_sorted,+ target_entries,+ target_sorted,+ None,+ )?;++ let offsets = source_map.value_offsets();+ let has_unreachable_entries = offsets[0] != 0+ || offsets[offsets.len() - 1] as usize != source_map.entries().len();+ let needs_compaction = has_unreachable_entries+ || source_map.offsets().has_non_empty_nulls(source_map.nulls());+ let compacted_map = if needs_compaction {+ Some(compact_map_entries(source_map)?)+ } else {+ None+ };+ let source_map = compacted_map.as_ref().unwrap_or(source_map);++ let cast_keys = cast_column(source_map.keys(), target_key.data_type(), cast_options)+ .map_err(|error| error.context("While casting Map keys"))?;+ let cast_values =+ cast_column(source_map.values(), target_value.data_type(), cast_options)+ .map_err(|error| error.context("While casting Map values"))?;+ let Struct(target_fields) = target_entries.data_type() else {+ unreachable!("validated Map entries must be Struct")+ };+ let cast_entries =+ StructArray::try_new(target_fields.clone(), vec![cast_keys, cast_values], None)?;++ Ok(Arc::new(MapArray::try_new(+ Arc::clone(target_entries),+ source_map.offsets().clone(),+ cast_entries,+ source_map.nulls().cloned(),+ target_sorted,+ )?))+}++/// Returns an equivalent MapArray whose entries contain only values reachable+/// from visible Map rows.+///+/// Arrow Map arrays can contain unreachable entries after slicing, or entries+/// hidden behind null parent rows. An identity `take` rebuilds the Map through+/// Arrow's selection kernel, normalizing offsets and dropping those unreachable+/// child entries before recursive key/value casts are applied.+fn compact_map_entries(map: &MapArray) -> Result<MapArray> {+ let indices = UInt64Array::from_iter_values(0..map.len() as u64);+ Ok(take(map, &indices, None)?.as_map().clone())+}+fn cast_dictionary_column(source_col: &ArrayRef,source_key_type: &DataType,@@ -490,8 +579,178 @@ fn validate_field_compatibility()}+fn validate_map_compatibility<'a>(+ source_entries: &Field,+ source_sorted: bool,+ target_entries: &'a Field,+ target_sorted: bool,+ field_name: Option<&str>,+) -> Result<(&'a FieldRef, &'a FieldRef)> {+ if source_sorted != target_sorted {+ if let Some(field_name) = field_name {+ return _plan_err!(+ "Cannot change Map sorted flag for field '{}' during schema adaptation",+ field_name+ );+ }+ return _plan_err!("Cannot change Map sorted flag during schema adaptation");+ }+ let (source_key, source_value) = validate_map_entries_field(source_entries)?;+ let (target_key, target_value) = validate_map_entries_field(target_entries)?;+ validate_map_key_compatibility(source_key, target_key, target_sorted)?;+ validate_field_compatibility(source_value, target_value)?;+ Ok((target_key, target_value))+}++fn validate_map_key_compatibility(+ source_key: &Field,+ target_key: &Field,+ sorted: bool,+) -> Result<()> {+ if sorted && source_key.data_type() != target_key.data_type() {+ return _plan_err!("Cannot evolve key type of a sorted Map");+ }+ validate_map_key_data_type(source_key.data_type(), target_key.data_type())+}++fn validate_map_key_data_type(+ source_type: &DataType,+ target_type: &DataType,+) -> Result<()> {+ if source_type == target_type {+ return Ok(());+ }++ match (source_type, target_type) {+ (Struct(source_fields), Struct(target_fields)) => {+ if !has_one_of_more_common_fields(source_fields, target_fields) {+ return _plan_err!(+ "Cannot cast Map key Struct because there is no field name overlap"+ );+ }++ let target_by_name: HashMap<&str, &FieldRef> = target_fields+ .iter()+ .map(|field| (field.name().as_str(), field))+ .collect();+ let source_names: HashSet<&str> = source_fields+ .iter()+ .map(|field| field.name().as_str())+ .collect();++ for source_field in source_fields {+ let Some(target_field) = target_by_name.get(source_field.name().as_str())+ else {+ return _plan_err!(+ "Cannot remove field '{}' from a Map key Struct",+ source_field.name()+ );+ };+ if source_field.is_nullable() && !target_field.is_nullable() {+ return _plan_err!(+ "Cannot cast nullable Map key field '{}' to non-nullable field",+ source_field.name()+ );+ }+ validate_map_key_data_type(+ source_field.data_type(),+ target_field.data_type(),+ )?;+ }+ for target_field in target_fields {+ if !source_names.contains(target_field.name().as_str())+ && !target_field.is_nullable()+ {+ return _plan_err!(+ "Cannot add non-nullable field '{}' to a Map key Struct",+ target_field.name()+ );+ }+ }+ Ok(())+ }+ _ if is_injective_map_key_cast(source_type, target_type) => Ok(()),+ _ => _plan_err!(+ "Cannot safely evolve Map key type from {} to {}",+ source_type,+ target_type+ ),+ }+}++/// Returns true when casting a Map key from `source_type` to `target_type` is+/// known to preserve key identity.+///+/// Keep this list conservative. Map keys define equality/lookup identity, so+/// only representation widenings that cannot merge distinct source keys belong+/// here. Other Arrow-supported casts, including decimal/timestamp widening and+/// dictionaries, should stay rejected until their identity semantics are+/// deliberately reviewed.+fn is_injective_map_key_cast(source_type: &DataType, target_type: &DataType) -> bool {+ matches!(+ (source_type, target_type),+ (+ DataType::Int8,+ DataType::Int16 | DataType::Int32 | DataType::Int64+ ) | (DataType::Int16, DataType::Int32 | DataType::Int64)+ | (DataType::Int32, DataType::Int64)+ | (+ DataType::UInt8,+ DataType::Int16+ | DataType::Int32+ | DataType::Int64+ | DataType::UInt16+ | DataType::UInt32+ | DataType::UInt64+ )+ | (+ DataType::UInt16,+ DataType::Int32 | DataType::Int64 | DataType::UInt32 | DataType::UInt64+ )+ | (DataType::UInt32, DataType::Int64 | DataType::UInt64)+ | (DataType::Utf8, DataType::LargeUtf8 | DataType::Utf8View)+ | (+ DataType::Binary,+ DataType::LargeBinary | DataType::BinaryView+ )+ )+}++fn validate_map_entries_field(entries: &Field) -> Result<(&FieldRef, &FieldRef)> {+ if entries.is_nullable() {+ return _plan_err!("Map entries field must be non-nullable");+ }++ let Struct(fields) = entries.data_type() else {+ return _plan_err!("Map entries field must be a struct");+ };+ if fields.len() != 2 {+ return _plan_err!(+ "Map entries struct must contain exactly two fields, found {}",+ fields.len()+ );+ }+ if fields[0].is_nullable() {+ return _plan_err!("Map key field must be non-nullable");+ }+ Ok((&fields[0], &fields[1]))+}+/// Validates that `source_type` can be cast to `target_type`, recursively/// handling container types that wrap structs.+///+/// # Map evolution+///+/// Map entry children are matched by position (key, then value), regardless of+/// their technical field names. Nested Struct fields within each child are+/// matched by name. Values use the standard Struct evolution rules: nullable+/// target fields may be added and extra source fields are omitted.+///+/// For Maps using specialized nested Struct adaptation, unsorted Struct keys may+/// add nullable fields and use only injective primitive widening casts, but may+/// not remove source fields. Sorted Maps require an unchanged key type. Map+/// entries and keys must remain non-nullable. Source and target sorted flags+/// must match. Maps without semantic Struct children use Arrow's normal cast.pub fn validate_data_type_compatibility(field_name: &str,source_type: &DataType,@@ -513,6 +772,24 @@ pub fn validate_data_type_compatibility(| (DataType::LargeListView(s), DataType::LargeListView(t)) => {validate_field_compatibility(s, t)?;}+ (DataType::Map(s, source_sorted), DataType::Map(t, target_sorted)) => {+ if requires_nested_struct_cast(source_type, target_type) {+ validate_map_compatibility(+ s,+ *source_sorted,+ t,+ *target_sorted,+ Some(field_name),+ )?;+ } else if !can_cast_types(source_type, target_type) {+ return _plan_err!(+ "Cannot cast struct field '{}' from type {} to type {}",+ field_name,+ source_type,+ target_type+ );+ }+ }(DataType::Dictionary(s_key, s_val), DataType::Dictionary(t_key, t_val)) => {if !can_cast_types(s_key, t_key) {return _plan_err!(@@ -539,13 +816,18 @@ pub fn validate_data_type_compatibility(}/// Returns true if casting from `source_type` to `target_type` requires-/// name-based nested struct casting logic, rather than Arrow's standard cast.+/// DataFusion's specialized nested/container casting logic rather than Arrow's+/// standard cast.////// This is the case when both types are struct types, or both are the same/// container type (List, LargeList, equal-width FixedSizeList, ListView,-/// LargeListView, Dictionary) wrapping types that recursively contain structs.+/// LargeListView, Map, Dictionary) wrapping types that recursively contain structs.+///+/// Map entries are technically Struct-backed, but that implementation detail does+/// not require name-based adaptation. A Map uses the specialized path only when+/// its semantic key or value child recursively contains a Struct.///-/// Use this predicate at both planning time (to decide whether to apply struct+/// Use this predicate at both planning time (to decide whether to apply nested/// compatibility validation) and execution time (to decide whether to route/// through [`cast_column`] instead of Arrow's generic cast).pub fn requires_nested_struct_cast(@@ -566,6 +848,9 @@ pub fn requires_nested_struct_cast(| (DataType::LargeListView(s), DataType::LargeListView(t)) => {requires_nested_struct_cast(s.data_type(), t.data_type())}+ (DataType::Map(source_entries, _), DataType::Map(target_entries, _)) => {+ map_entries_require_nested_struct_cast(source_entries, target_entries)+ }(DataType::Dictionary(_, s_val), DataType::Dictionary(_, t_val)) => {requires_nested_struct_cast(s_val, t_val)}@@ -573,6 +858,31 @@ pub fn requires_nested_struct_cast(}}+/// Returns whether corresponding semantic Map children require nested Struct casting.+///+/// Invalid Map entry layouts return `false`; normal Map validation/casting reports+/// those layouts through its usual error path.+fn map_entries_require_nested_struct_cast(+ source_entries: &FieldRef,+ target_entries: &FieldRef,+) -> bool {+ let (Struct(source_fields), Struct(target_fields)) =+ (source_entries.data_type(), target_entries.data_type())+ else {+ return false;+ };++ source_fields.len() == 2+ && target_fields.len() == 2+ && (requires_nested_struct_cast(+ source_fields[0].data_type(),+ target_fields[0].data_type(),+ ) || requires_nested_struct_cast(+ source_fields[1].data_type(),+ target_fields[1].data_type(),+ ))+}+/// Check if two field lists have at least one common field by name.////// This is useful for validating struct compatibility when casting between structs,@@ -597,10 +907,10 @@ mod tests {use arrow::{array::{BinaryArray, FixedSizeListArray, Int32Array, Int32Builder, Int64Array,- ListArray, ListViewArray, MapArray, MapBuilder, NullArray, StringArray,- StringBuilder,+ Int64Builder, ListArray, ListViewArray, MapArray, MapBuilder, NullArray,+ StringArray, StringBuilder, UInt32Array,},- buffer::{NullBuffer, ScalarBuffer},+ buffer::{NullBuffer, OffsetBuffer, ScalarBuffer},datatypes::{DataType, Field, FieldRef, Int32Type},};/// Macro to extract and downcast a column from a StructArray@@ -1292,6 +1602,782 @@ mod tests {assert!(b_col.is_null(1));}+ fn map_type(key_type: DataType, value_type: DataType) -> DataType {+ map_type_with_entry_names(key_type, value_type, "keys", "values", false)+ }++ fn map_type_with_entry_names(+ key_type: DataType,+ value_type: DataType,+ key_name: &str,+ value_name: &str,+ sorted: bool,+ ) -> DataType {+ DataType::Map(+ Arc::new(non_null_field(+ "entries",+ struct_type(vec![+ non_null_field(key_name, key_type),+ field(value_name, value_type),+ ]),+ )),+ sorted,+ )+ }++ fn struct_map_array() -> ArrayRef {+ struct_map_array_with_sorted(false)+ }++ fn struct_map_array_with_sorted(sorted: bool) -> ArrayRef {+ struct_map_array_with_key_fields(sorted, false)+ }++ fn struct_map_array_with_key_fields(sorted: bool, include_tenant: bool) -> ArrayRef {+ let mut key_fields = vec![(+ arc_field("id", DataType::Int32),+ Arc::new(Int32Array::from(vec![1])) as ArrayRef,+ )];+ if include_tenant {+ key_fields.push((+ arc_field("tenant", DataType::Utf8),+ Arc::new(StringArray::from(vec!["a"])) as ArrayRef,+ ));+ }+ let keys = StructArray::from(key_fields);+ let values = StructArray::from(vec![+ (+ arc_field("amount", DataType::Int32),+ Arc::new(Int32Array::from(vec![10])) as ArrayRef,+ ),+ (+ arc_field("ignored", DataType::Utf8),+ Arc::new(StringArray::from(vec!["x"])) as ArrayRef,+ ),+ ]);+ let entries = StructArray::new(+ vec![+ Arc::new(non_null_field("keys", keys.data_type().clone())),+ arc_field("values", values.data_type().clone()),+ ]+ .into(),+ vec![Arc::new(keys), Arc::new(values)],+ None,+ );+ Arc::new(MapArray::new(+ Arc::new(non_null_field("entries", entries.data_type().clone())),+ OffsetBuffer::new(vec![0, 1, 1].into()),+ entries,+ Some(NullBuffer::from(vec![true, false])),+ sorted,+ ))+ }++ fn nested_struct_map_array(+ key_name: &str,+ value_name: &str,+ sorted: bool,+ ) -> ArrayRef {+ let key_nested = StructArray::from(vec![(+ arc_field("id", DataType::Int32),+ Arc::new(Int32Array::from(vec![1])) as ArrayRef,+ )]);+ let keys = StructArray::from(vec![(+ arc_field("nested", key_nested.data_type().clone()),+ Arc::new(key_nested) as ArrayRef,+ )]);+ let value_nested = StructArray::from(vec![(+ arc_field("amount", DataType::Int32),+ Arc::new(Int32Array::from(vec![10])) as ArrayRef,+ )]);+ let values = StructArray::from(vec![(+ arc_field("nested", value_nested.data_type().clone()),+ Arc::new(value_nested) as ArrayRef,+ )]);+ let entries = StructArray::new(+ vec![+ Arc::new(non_null_field(key_name, keys.data_type().clone())),+ arc_field(value_name, values.data_type().clone()),+ ]+ .into(),+ vec![Arc::new(keys), Arc::new(values)],+ None,+ );+ Arc::new(MapArray::new(+ Arc::new(non_null_field("entries", entries.data_type().clone())),+ OffsetBuffer::new(vec![0, 1].into()),+ entries,+ None,+ sorted,+ ))+ }++ #[test]+ fn test_safe_cast_to_non_nullable_struct_field_returns_error() {+ let source_col: ArrayRef = Arc::new(StructArray::new(+ vec![Arc::new(non_null_field("value", DataType::Utf8))].into(),+ vec![Arc::new(StringArray::from(vec!["not-an-int"]))],+ None,+ ));+ let target_type = struct_type(vec![non_null_field("value", DataType::Int32)]);+ let cast_options = CastOptions {+ safe: true,+ ..DEFAULT_CAST_OPTIONS+ };++ let error = cast_column(&source_col, &target_type, &cast_options)+ .unwrap_err()+ .to_string();+ assert_contains!(+ error,+ "Found unmasked nulls for non-nullable StructArray field \"value\""+ );+ }++ #[test]+ fn test_safe_cast_to_non_nullable_map_value_returns_error() {+ let entries = StructArray::new(+ vec![+ Arc::new(non_null_field("keys", DataType::Utf8)),+ Arc::new(non_null_field("values", DataType::Utf8)),+ ]+ .into(),+ vec![+ Arc::new(StringArray::from(vec!["key"])) as ArrayRef,+ Arc::new(StringArray::from(vec!["not-an-int"])) as ArrayRef,+ ],+ None,+ );+ let source_col: ArrayRef = Arc::new(MapArray::new(+ Arc::new(non_null_field("entries", entries.data_type().clone())),+ OffsetBuffer::new(vec![0, 1].into()),+ entries,+ None,+ false,+ ));+ let target_type = DataType::Map(+ Arc::new(non_null_field(+ "entries",+ struct_type(vec![+ non_null_field("keys", DataType::Utf8),+ non_null_field("values", DataType::Int32),+ ]),+ )),+ false,+ );+ let cast_options = CastOptions {+ safe: true,+ ..DEFAULT_CAST_OPTIONS+ };++ let error = cast_column(&source_col, &target_type, &cast_options)+ .unwrap_err()+ .to_string();+ assert_contains!(+ error,+ "Found unmasked nulls for non-nullable StructArray field \"values\""+ );+ }++ #[test]+ fn test_cast_primitive_map_uses_arrow_cast() {+ let mut builder = MapBuilder::new(None, Int64Builder::new(), Int64Builder::new());+ builder.keys().append_value(1);+ builder.values().append_value(10);+ builder.append(true).unwrap();++ let source_col: ArrayRef = Arc::new(builder.finish());+ let target_type = map_type(DataType::Int32, DataType::Int32);++ assert!(!requires_nested_struct_cast(+ source_col.data_type(),+ &target_type+ ));+ assert!(+ validate_data_type_compatibility(+ "map_col",+ source_col.data_type(),+ &target_type+ )+ .is_ok()+ );++ let result =+ cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS).unwrap();+ assert_eq!(result.data_type(), &target_type);+ }++ #[test]+ fn test_cast_map_int32_keys_to_int64() {+ let mut builder =+ MapBuilder::new(None, Int32Builder::new(), StringBuilder::new());+ builder.keys().append_value(1);+ builder.keys().append_value(2);+ builder.values().append_value("a");+ builder.values().append_value("b");+ builder.append(true).unwrap();++ let source_col: ArrayRef = Arc::new(builder.finish());+ let target_type = map_type(DataType::Int64, DataType::Utf8);++ assert!(+ validate_data_type_compatibility(+ "map_col",+ source_col.data_type(),+ &target_type+ )+ .is_ok()+ );++ let result =+ cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS).unwrap();+ assert_eq!(result.data_type(), &target_type);+ let map = result.as_any().downcast_ref::<MapArray>().unwrap();+ assert_eq!(map.value_offsets(), &[0, 2]);++ let keys = map.keys().as_any().downcast_ref::<Int64Array>().unwrap();+ assert_eq!(keys.values(), &[1, 2]);+ let values = map.values().as_any().downcast_ref::<StringArray>().unwrap();+ assert_eq!(values.value(0), "a");+ assert_eq!(values.value(1), "b");+ }++ #[test]+ fn test_cast_map_uint32_keys_to_int64() {+ let entries = StructArray::new(+ vec![+ Arc::new(non_null_field("keys", DataType::UInt32)),+ arc_field("values", DataType::Utf8),+ ]+ .into(),+ vec![+ Arc::new(UInt32Array::from(vec![u32::MAX])) as ArrayRef,+ Arc::new(StringArray::from(vec!["value"])) as ArrayRef,+ ],+ None,+ );+ let source_col: ArrayRef = Arc::new(MapArray::new(+ Arc::new(non_null_field("entries", entries.data_type().clone())),+ OffsetBuffer::new(vec![0, 1].into()),+ entries,+ None,+ false,+ ));+ let target_type = map_type(DataType::Int64, DataType::Utf8);++ assert!(+ validate_data_type_compatibility(+ "map_col",+ source_col.data_type(),+ &target_type+ )+ .is_ok()+ );++ let result =+ cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS).unwrap();+ let keys = result+ .as_map()+ .keys()+ .as_any()+ .downcast_ref::<Int64Array>()+ .unwrap();+ assert_eq!(keys.value(0), i64::from(u32::MAX));+ }++ #[test]+ fn test_injective_unsigned_to_signed_map_key_casts() {+ for (source_type, target_type) in [+ (DataType::UInt8, DataType::Int16),+ (DataType::UInt8, DataType::Int32),+ (DataType::UInt8, DataType::Int64),+ (DataType::UInt16, DataType::Int32),+ (DataType::UInt16, DataType::Int64),+ (DataType::UInt32, DataType::Int64),+ ] {+ assert!(is_injective_map_key_cast(&source_type, &target_type));+ }+ for (source_type, target_type) in [+ (DataType::UInt8, DataType::Int8),+ (DataType::UInt16, DataType::Int16),+ (DataType::UInt32, DataType::Int32),+ ] {+ assert!(!is_injective_map_key_cast(&source_type, &target_type));+ }+ }++ #[test]+ fn test_map_entry_names_match_positionally_and_adapt_nested_structs() {+ let source_col = nested_struct_map_array("source_keys", "source_values", false);+ let target_type = map_type_with_entry_names(+ struct_type(vec![field(+ "nested",+ struct_type(vec![+ field("id", DataType::Int64),+ field("label", DataType::Utf8),+ ]),+ )]),+ struct_type(vec![field(+ "nested",+ struct_type(vec![+ field("amount", DataType::Int64),+ field("currency", DataType::Utf8),+ ]),+ )]),+ "key",+ "value",+ false,+ );++ assert!(+ validate_data_type_compatibility(+ "map_col",+ source_col.data_type(),+ &target_type+ )+ .is_ok()+ );+ let result =+ cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS).unwrap();+ let map = result.as_any().downcast_ref::<MapArray>().unwrap();+ let (key_field, value_field) = map.entries_fields();+ assert_eq!(key_field.name(), "key");+ assert_eq!(value_field.name(), "value");+ let keys = map.keys().as_struct();+ let key_nested = keys.column_by_name("nested").unwrap().as_struct();+ assert_eq!(get_column_as!(key_nested, "id", Int64Array).value(0), 1);+ assert!(get_column_as!(key_nested, "label", StringArray).is_null(0));+ let values = map.values().as_struct();+ let value_nested = values.column_by_name("nested").unwrap().as_struct();+ assert_eq!(+ get_column_as!(value_nested, "amount", Int64Array).value(0),+ 10+ );+ assert!(get_column_as!(value_nested, "currency", StringArray).is_null(0));+ }++ #[test]+ fn test_unsorted_map_key_struct_field_removal_rejected_by_planner_and_runtime() {+ let source_col = struct_map_array_with_key_fields(false, true);+ let target_type = map_type(+ struct_type(vec![field("id", DataType::Int32)]),+ struct_type(vec![+ field("amount", DataType::Int32),+ field("ignored", DataType::Utf8),+ ]),+ );+ assert_map_planning_runtime_error(+ &source_col,+ &target_type,+ "Cannot remove field 'tenant' from a Map key Struct",+ );+ }++ #[test]+ fn test_unsorted_map_non_injective_key_cast_rejected() {+ let source_col = struct_map_array();+ let target_type = map_type(+ struct_type(vec![field("id", DataType::Float64)]),+ struct_type(vec![+ field("amount", DataType::Int32),+ field("ignored", DataType::Utf8),+ ]),+ );+ assert_map_planning_runtime_error(+ &source_col,+ &target_type,+ "Cannot safely evolve Map key type",+ );+ }++ #[test]+ fn test_unsorted_map_key_struct_no_overlap_rejected_by_planner_and_runtime() {+ let keys = StructArray::new_empty_fields(1, Some(NullBuffer::from(vec![true])));+ let values = StructArray::from(vec![(+ arc_field("amount", DataType::Int32),+ Arc::new(Int32Array::from(vec![10])) as ArrayRef,+ )]);+ let entries = StructArray::new(+ vec![+ Arc::new(non_null_field("keys", keys.data_type().clone())),+ arc_field("values", values.data_type().clone()),+ ]+ .into(),+ vec![Arc::new(keys), Arc::new(values)],+ None,+ );+ let source_col: ArrayRef = Arc::new(MapArray::new(+ Arc::new(non_null_field("entries", entries.data_type().clone())),+ OffsetBuffer::new(vec![0, 1].into()),+ entries,+ None,+ false,+ ));+ let target_type = map_type(+ struct_type(vec![field("new_id", DataType::Int32)]),+ struct_type(vec![field("amount", DataType::Int32)]),+ );+ assert_map_planning_runtime_error(+ &source_col,+ &target_type,+ "Cannot cast Map key Struct because there is no field name overlap",+ );+ }++ #[test]+ fn test_sorted_map_key_struct_schema_evolution_rejected() {+ let source_col = struct_map_array_with_sorted(true);+ let target_type = map_type_with_entry_names(+ struct_type(vec![+ field("id", DataType::Int32),+ field("label", DataType::Utf8),+ ]),+ struct_type(vec![+ field("amount", DataType::Int32),+ field("ignored", DataType::Utf8),+ ]),+ "keys",+ "values",+ true,+ );+ assert_map_planning_runtime_error(+ &source_col,+ &target_type,+ "Cannot evolve key type of a sorted Map",+ );+ }++ #[test]+ fn test_sorted_map_value_struct_schema_evolution_preserves_sorted() {+ let source_col = struct_map_array_with_sorted(true);+ let target_type = map_type_with_entry_names(+ struct_type(vec![field("id", DataType::Int32)]),+ struct_type(vec![+ field("amount", DataType::Int64),+ field("currency", DataType::Utf8),+ ]),+ "keys",+ "values",+ true,+ );+ assert!(+ validate_data_type_compatibility(+ "map_col",+ source_col.data_type(),+ &target_type+ )+ .is_ok()+ );+ let result =+ cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS).unwrap();+ assert_eq!(result.data_type(), &target_type);+ let map = result.as_any().downcast_ref::<MapArray>().unwrap();+ let values = map.values().as_struct();+ assert_eq!(get_column_as!(values, "amount", Int64Array).value(0), 10);+ assert!(get_column_as!(values, "currency", StringArray).is_null(0));+ }++ #[test]+ fn test_sorted_map_value_struct_schema_evolution_to_unsorted_rejected() {+ let source_col = struct_map_array_with_sorted(true);+ let target_type = map_type_with_entry_names(+ struct_type(vec![+ field("id", DataType::Int32),+ field("label", DataType::Utf8),+ ]),+ struct_type(vec![+ field("amount", DataType::Int64),+ field("currency", DataType::Utf8),+ ]),+ "keys",+ "values",+ false,+ );++ assert_map_planning_runtime_error(+ &source_col,+ &target_type,+ "Cannot change Map sorted flag",+ );+ }++ #[test]+ fn test_null_map_parent_hides_invalid_nested_value_cast() {+ let values = StructArray::from(vec![(+ arc_field("amount", DataType::Utf8),+ Arc::new(StringArray::from(vec!["bad", "2"])) as ArrayRef,+ )]);+ let keys = StringArray::from(vec!["hidden", "visible"]);+ let entries = StructArray::new(+ vec![+ Arc::new(non_null_field("keys", DataType::Utf8)),+ arc_field("values", values.data_type().clone()),+ ]+ .into(),+ vec![Arc::new(keys), Arc::new(values)],+ None,+ );+ let source_col: ArrayRef = Arc::new(MapArray::new(+ Arc::new(non_null_field("entries", entries.data_type().clone())),+ OffsetBuffer::new(vec![0, 1, 2].into()),+ entries,+ Some(NullBuffer::from(vec![false, true])),+ false,+ ));+ let target_type = map_type(+ DataType::Utf8,+ struct_type(vec![field("amount", DataType::Int32)]),+ );++ let result =+ cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS).unwrap();+ let map = result.as_any().downcast_ref::<MapArray>().unwrap();+ assert!(map.is_null(0));+ assert_eq!(map.value_offsets(), &[0, 0, 1]);+ let visible_entries = map.value(1);+ let visible_values = visible_entries+ .column_by_name("values")+ .unwrap()+ .as_struct();+ assert_eq!(+ get_column_as!(visible_values, "amount", Int32Array).value(0),+ 2+ );+ }++ #[test]+ fn test_sliced_map_ignores_unreachable_invalid_nested_value() {+ let values = StructArray::from(vec![(+ arc_field("amount", DataType::Utf8),+ Arc::new(StringArray::from(vec!["bad", "2"])) as ArrayRef,+ )]);+ let entries = StructArray::new(+ vec![+ Arc::new(non_null_field("keys", DataType::Utf8)),+ arc_field("values", values.data_type().clone()),+ ]+ .into(),+ vec![+ Arc::new(StringArray::from(vec!["unreachable", "visible"])),+ Arc::new(values),+ ],+ None,+ );+ let source_col: ArrayRef = Arc::new(+ MapArray::new(+ Arc::new(non_null_field("entries", entries.data_type().clone())),+ OffsetBuffer::new(vec![0, 1, 2].into()),+ entries,+ None,+ false,+ )+ .slice(1, 1),+ );+ let target_type = map_type(+ DataType::Utf8,+ struct_type(vec![field("amount", DataType::Int32)]),+ );++ let result =+ cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS).unwrap();+ let map = result.as_any().downcast_ref::<MapArray>().unwrap();+ assert_eq!(map.value_offsets(), &[0, 1]);+ let values = map.values().as_struct();+ assert_eq!(get_column_as!(values, "amount", Int32Array).value(0), 2);+ }++ #[test]+ fn test_cast_map_struct_key_and_value() {+ let source_col = struct_map_array();+ let target_type = map_type(+ struct_type(vec![+ field("id", DataType::Int64),+ field("label", DataType::Utf8),+ ]),+ struct_type(vec![+ field("amount", DataType::Int64),+ field("currency", DataType::Utf8),+ ]),+ );++ assert!(requires_nested_struct_cast(+ source_col.data_type(),+ &target_type+ ));+ assert!(+ validate_data_type_compatibility(+ "map_col",+ source_col.data_type(),+ &target_type+ )+ .is_ok()+ );++ let result =+ cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS).unwrap();+ let map = result.as_any().downcast_ref::<MapArray>().unwrap();+ assert!(map.is_valid(0));+ assert!(map.is_null(1));++ let keys = map.keys().as_any().downcast_ref::<StructArray>().unwrap();+ let key_ids = get_column_as!(keys, "id", Int64Array);+ assert_eq!(key_ids.values(), &[1]);+ let key_labels = get_column_as!(keys, "label", StringArray);+ assert!(key_labels.is_null(0));++ let values = map.values().as_any().downcast_ref::<StructArray>().unwrap();+ let amounts = get_column_as!(values, "amount", Int64Array);+ assert_eq!(amounts.values(), &[10]);+ let currencies = get_column_as!(values, "currency", StringArray);+ assert!(currencies.is_null(0));+ assert!(values.column_by_name("ignored").is_none());+ }++ #[test]+ fn test_cast_all_null_map_with_struct_value() {+ let source_type = map_type(+ DataType::Utf8,+ struct_type(vec![field("amount", DataType::Int32)]),+ );+ let target_type = map_type(+ DataType::Utf8,+ struct_type(vec![+ field("amount", DataType::Int64),+ field("currency", DataType::Utf8),+ ]),+ );+ let source_col = new_null_array(&source_type, 2);++ let result =+ cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS).unwrap();+ let map = result.as_any().downcast_ref::<MapArray>().unwrap();+ assert_eq!(map.null_count(), 2);+ assert!(map.entries().is_empty());+ assert_eq!(map.data_type(), &target_type);+ }++ #[test]+ fn test_map_entry_invariants_rejected_during_validation() {+ let source_col = struct_map_array();+ let DataType::Map(target_entries, sorted) = map_type(+ struct_type(vec![field("id", DataType::Int32)]),+ struct_type(vec![field("amount", DataType::Int32)]),+ ) else {+ unreachable!()+ };+ let nullable_entries = DataType::Map(+ Arc::new(target_entries.as_ref().clone().with_nullable(true)),+ sorted,+ );+ let error = validate_data_type_compatibility(+ "map_col",+ source_col.data_type(),+ &nullable_entries,+ )+ .unwrap_err()+ .to_string();+ assert_contains!(error, "Map entries field must be non-nullable");++ let Struct(fields) = target_entries.data_type() else {+ unreachable!()+ };+ let nullable_key_entries = Arc::new(Field::new(+ "entries",+ Struct(+ vec![+ Arc::new(fields[0].as_ref().clone().with_nullable(true)),+ Arc::clone(&fields[1]),+ ]+ .into(),+ ),+ false,+ ));+ let error = validate_data_type_compatibility(+ "map_col",+ source_col.data_type(),+ &DataType::Map(nullable_key_entries, sorted),+ )+ .unwrap_err()+ .to_string();+ assert_contains!(error, "Map key field must be non-nullable");++ let sorted_target = DataType::Map(target_entries, !sorted);+ assert_map_planning_runtime_error(+ &source_col,+ &sorted_target,+ "Cannot change Map sorted flag",+ );+ }++ #[test]+ fn test_map_sorted_flag_validation_error_includes_field_name() {+ let source_col = struct_map_array();+ let DataType::Map(target_entries, sorted) = source_col.data_type() else {+ unreachable!()+ };+ let target_type = DataType::Map(Arc::clone(target_entries), !sorted);++ let error = validate_data_type_compatibility(+ "map_col",+ source_col.data_type(),+ &target_type,+ )+ .unwrap_err()+ .to_string();++ assert_contains!(+ error,+ "Cannot change Map sorted flag for field 'map_col' during schema adaptation"+ );+ }++ fn assert_map_planning_runtime_error(+ source: &ArrayRef,+ target: &DataType,+ expected: &str,+ ) {+ let planning_error =+ validate_data_type_compatibility("map_col", source.data_type(), target)+ .unwrap_err()+ .to_string();+ assert_contains!(planning_error, expected);++ let runtime_error = cast_column(source, target, &DEFAULT_CAST_OPTIONS)+ .unwrap_err()+ .to_string();+ assert_contains!(runtime_error, expected);+ }++ #[test]+ fn test_map_struct_planner_runtime_parity_on_invalid_evolution() {+ let source_col = struct_map_array();+ let target_type = map_type(+ struct_type(vec![field("id", DataType::Int32)]),+ struct_type(vec![+ field("amount", DataType::Int32),+ non_null_field("currency", DataType::Utf8),+ ]),+ );+ assert_map_planning_runtime_error(+ &source_col,+ &target_type,+ "target field 'currency' is non-nullable",+ );++ let incompatible_target = map_type(+ struct_type(vec![field("id", DataType::Int32)]),+ struct_type(vec![field(+ "amount",+ struct_type(vec![field("value", DataType::Utf8)]),+ )]),+ );+ assert_map_planning_runtime_error(+ &source_col,+ &incompatible_target,+ "Cannot cast struct field 'amount'",+ );+ }+#[test]fn test_validate_dictionary_value_evolution() {let source_inner = struct_type(vec![field("a", DataType::Int32)]);@@ -1679,6 +2765,14 @@ mod tests {&DataType::Dictionary(Box::new(DataType::Int32), Box::new(s1.clone())),&DataType::Dictionary(Box::new(DataType::Int32), Box::new(s2.clone())),));+ assert!(requires_nested_struct_cast(+ &map_type(s1.clone(), DataType::Utf8),+ &map_type(s2.clone(), DataType::Utf8),+ ));+ assert!(requires_nested_struct_cast(+ &map_type(DataType::Utf8, map_type(DataType::Utf8, s1.clone()),),+ &map_type(DataType::Utf8, map_type(DataType::Utf8, s2.clone())),+ ));assert!(requires_nested_struct_cast(&DataType::ListView(arc_field("item", s1.clone())),&DataType::ListView(arc_field("item", s2.clone())),@@ -1697,6 +2791,14 @@ mod tests {&DataType::List(arc_field("item", DataType::Int32)),&DataType::List(arc_field("item", DataType::Int64)),));+ assert!(!requires_nested_struct_cast(+ &map_type(DataType::Int64, DataType::Int64),+ &map_type(DataType::Int32, DataType::Int32),+ ));+ assert!(!requires_nested_struct_cast(+ &map_type(DataType::Utf8, map_type(DataType::Utf8, DataType::Int64),),+ &map_type(DataType::Utf8, map_type(DataType::Utf8, DataType::Int32),),+ ));assert!(!requires_nested_struct_cast(&DataType::FixedSizeList(arc_field("item", DataType::Int32), 2),&DataType::FixedSizeList(arc_field("item", DataType::Int64), 2),diff --git a/datafusion/core/tests/parquet/expr_adapter.rs b/datafusion/core/tests/parquet/expr_adapter.rsindex 535828fa29..3bee5e0070 100644--- a/datafusion/core/tests/parquet/expr_adapter.rs+++ b/datafusion/core/tests/parquet/expr_adapter.rs@@ -19,9 +19,10 @@ use std::sync::Arc;use arrow::array::{Array, ArrayRef, BooleanArray, FixedSizeListArray, Int32Array, Int64Array,- LargeListArray, ListArray, RecordBatch, StringArray, StructArray, record_batch,+ LargeListArray, ListArray, MapArray, RecordBatch, StringArray, StructArray,+ record_batch,};-use arrow::buffer::OffsetBuffer;+use arrow::buffer::{NullBuffer, OffsetBuffer};use arrow::compute::concat_batches;use arrow_schema::{DataType, Field, Fields, Schema, SchemaRef};use bytes::{BufMut, BytesMut};@@ -928,6 +929,176 @@ async fn test_struct_schema_evolution_projection_and_filter() -> Result<()> {Ok(())}+fn map_value_struct_evolution_batch() -> Result<RecordBatch> {+ let physical_value_fields: Fields = vec![+ Arc::new(Field::new("amount", DataType::Int32, false)),+ Arc::new(Field::new("ignored", DataType::Utf8, true)),+ ]+ .into();+ let values = StructArray::new(+ physical_value_fields.clone(),+ vec![+ Arc::new(Int32Array::from(vec![10])) as ArrayRef,+ Arc::new(StringArray::from(vec!["x"])) as ArrayRef,+ ],+ None,+ );+ let entry_fields: Fields = vec![+ Arc::new(Field::new("keys", DataType::Utf8, false)),+ Arc::new(Field::new(+ "values",+ DataType::Struct(physical_value_fields),+ true,+ )),+ ]+ .into();+ let entries = StructArray::new(+ entry_fields.clone(),+ vec![+ Arc::new(StringArray::from(vec!["a"])) as ArrayRef,+ Arc::new(values) as ArrayRef,+ ],+ None,+ );+ let map = MapArray::new(+ Arc::new(Field::new("entries", DataType::Struct(entry_fields), false)),+ OffsetBuffer::new(vec![0, 1, 1].into()),+ entries,+ Some(NullBuffer::from(vec![true, false])),+ false,+ );+ Ok(RecordBatch::try_from_iter(vec![+ ("row_id", Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef),+ ("attributes", Arc::new(map) as ArrayRef),+ ])?)+}++fn map_value_struct_table_schema(currency_nullable: bool) -> SchemaRef {+ let target_value_fields: Fields = vec![+ Arc::new(Field::new("amount", DataType::Int64, false)),+ Arc::new(Field::new("currency", DataType::Utf8, currency_nullable)),+ ]+ .into();+ let target_entries = Field::new(+ "entries",+ DataType::Struct(+ vec![+ Arc::new(Field::new("key", DataType::Utf8, false)),+ Arc::new(Field::new(+ "value",+ DataType::Struct(target_value_fields),+ true,+ )),+ ]+ .into(),+ ),+ false,+ );+ Arc::new(Schema::new(vec![+ Field::new("row_id", DataType::Int32, false),+ Field::new(+ "attributes",+ DataType::Map(Arc::new(target_entries), false),+ true,+ ),+ ]))+}++fn map_value_struct_nullable_schema() -> SchemaRef {+ map_value_struct_table_schema(true)+}++fn map_value_struct_non_nullable_schema() -> SchemaRef {+ map_value_struct_table_schema(false)+}++async fn setup_map_value_struct_table(+ table_path: &str,+ table_schema: SchemaRef,+) -> Result<SessionContext> {+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;+ let file_path = format!("{table_path}/data.parquet");+ write_parquet(+ map_value_struct_evolution_batch()?,+ Arc::clone(&store),+ &file_path,+ )+ .await;++ let ctx = test_context();+ let table_url = format!("memory:///{table_path}/");+ register_memory_listing_table(&ctx, store, &table_url, table_schema).await;+ Ok(ctx)+}++#[tokio::test]+async fn test_map_value_struct_schema_evolution_end_to_end() -> Result<()> {+ let ctx =+ setup_map_value_struct_table("map_evolution", map_value_struct_nullable_schema())+ .await?;++ let batches = ctx+ .sql("SELECT * FROM t ORDER BY row_id")+ .await?+ .collect()+ .await?;+ let map = batches[0]+ .column(1)+ .as_any()+ .downcast_ref::<MapArray>()+ .expect("attributes should be a MapArray");+ assert!(map.is_valid(0));+ assert!(map.is_null(1));+ let (key_field, value_field) = map.entries_fields();+ assert_eq!(key_field.name(), "key");+ assert_eq!(value_field.name(), "value");+ let keys = map+ .keys()+ .as_any()+ .downcast_ref::<StringArray>()+ .expect("map keys should be a StringArray");+ assert_eq!(keys.value(0), "a");+ let values = map+ .values()+ .as_any()+ .downcast_ref::<StructArray>()+ .expect("map values should be a StructArray");+ let amounts = values+ .column_by_name("amount")+ .unwrap()+ .as_any()+ .downcast_ref::<Int64Array>()+ .unwrap();+ assert_eq!(amounts.values(), &[10]);+ assert_eq!(values.column_by_name("currency").unwrap().null_count(), 1);+ assert!(values.column_by_name("ignored").is_none());++ Ok(())+}++#[tokio::test]+async fn test_map_value_struct_incompatible_schema_evolution_rejected() -> Result<()> {+ let ctx = setup_map_value_struct_table(+ "map_evolution_rejected",+ map_value_struct_non_nullable_schema(),+ )+ .await?;++ let error = ctx+ .sql("SELECT * FROM t ORDER BY row_id")+ .await?+ .collect()+ .await+ .unwrap_err()+ .to_string();+ assert!(+ error.contains("target field 'currency' is non-nullable"),+ "unexpected error: {error}"+ );++ Ok(())+}+/// Macro to generate schema evolution tests for list-like variants.macro_rules! test_struct_schema_evolution_variants {(diff --git a/datafusion/sqllogictest/test_files/map.slt b/datafusion/sqllogictest/test_files/map.sltindex 9ec2d0b894..59f340a083 100644--- a/datafusion/sqllogictest/test_files/map.slt+++ b/datafusion/sqllogictest/test_files/map.slt@@ -1199,3 +1199,27 @@ SELECT * FROM map_null_insert;statement okDROP TABLE map_null_insert;++# Primitive Map assignment casts keep Arrow's existing behavior. They must not+# use the nested Struct evolution path merely because Map entries are Struct-backed.+statement ok+CREATE TABLE map_primitive_cast_insert AS+SELECT map([arrow_cast(1, 'Int32')], [arrow_cast(10, 'Int32')]) AS m;++statement ok+INSERT INTO map_primitive_cast_insert VALUES (MAP {2: 20});++query ? rowsort+SELECT * FROM map_primitive_cast_insert;+----+{1: 10}+{2: 20}++statement error DataFusion error: Execution error: map key must be unique, duplicate key found: a+INSERT INTO map_primitive_cast_insert VALUES (MAP {'a': 1, 'a': 10});++statement error Execution error: map key cannot be null+INSERT INTO map_primitive_cast_insert VALUES (MAP {NULL: 1});++statement ok+DROP TABLE map_primitive_cast_insert;diff --git a/datafusion/sqllogictest/test_files/string_numeric_coercion.slt b/datafusion/sqllogictest/test_files/string_numeric_coercion.sltindex 1567a149bc..1a3ef93f7e 100644--- a/datafusion/sqllogictest/test_files/string_numeric_coercion.slt+++ b/datafusion/sqllogictest/test_files/string_numeric_coercion.slt@@ -498,6 +498,26 @@ SELECT arrow_typeof(m) FROM (SELECT m FROM t_map_int UNION ALL SELECT m FROM t_m----Map("entries": non-null Struct("key": non-null Utf8, "value": Utf8), unsorted)+# Map key coercion uses the common Int64 type for UInt32 and Int32 keys.+query ? rowsort+SELECT map([arrow_cast(1, 'UInt32')], ['a'])+UNION ALL+SELECT map([arrow_cast(2, 'Int32')], ['b']);+----+{1: a}+{2: b}++query T+SELECT arrow_typeof(m)+FROM (+ SELECT map([arrow_cast(1, 'UInt32')], ['a']) AS m+ UNION ALL+ SELECT map([arrow_cast(2, 'Int32')], ['b']) AS m+)+LIMIT 1;+----+Map("entries": non-null Struct("key": non-null Int64, "value": Utf8), unsorted)+statement okDROP TABLE t_map_int;diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.mdindex e88722bbf4..4483fe7605 100644--- a/docs/source/library-user-guide/upgrading/55.0.0.md+++ b/docs/source/library-user-guide/upgrading/55.0.0.md@@ -25,6 +25,25 @@in this section pertains to features and changes that have already been mergedto the main branch and are awaiting release in this version.+### Map casts with nested Structs use schema-evolution rules++Casts of a `Map` whose key or value recursively contains a `Struct` now adapt+nested Struct fields by case-sensitive name. Nullable fields added to the target+Struct are filled with `NULL`, and source-only fields are omitted. Missing+non-nullable target fields, nullable-to-non-nullable changes, and incompatible+nested types are rejected.++Map keys define map identity. During this nested-Struct adaptation, key Struct+fields cannot be removed and primitive key type changes must preserve every+source value (for example, `Int32` to `Int64` is allowed, while `Int64` to+`Int32` is rejected). Sorted Maps also require an unchanged key type and the+same sorted flag.++**Migration guide:** Schemas that relied on positional Struct-field matching,+key-field removal, or narrowing Map-key casts must be updated to retain+case-sensitive field names and key identity. Ordinary Maps without semantic+Struct children continue to use Arrow's normal Map cast behavior.+### `TableProvider::scan` takes `Option<&[usize]>` for the projection`TableProvider::scan` and the related APIs listed below previously took the