Apache DataFusion SQL Query Engine
fix: preserve explicit Struct casts and evolved field semantics (#24680)
Chao Sunf1a736ec3792
diff --git a/datafusion/core/tests/parquet/expr_adapter.rs b/datafusion/core/tests/parquet/expr_adapter.rsindex 3bee5e0070..dfed2eb5bd 100644--- a/datafusion/core/tests/parquet/expr_adapter.rs+++ b/datafusion/core/tests/parquet/expr_adapter.rs@@ -790,6 +790,212 @@ async fn test_physical_expr_adapter_with_non_null_defaults() {assert_batches_eq!(expected, &batches);}+#[tokio::test]+async fn test_explicit_struct_cast_projection_preserves_sibling_errors() -> Result<()> {+ let physical_fields: Fields = vec![+ Field::new("x", DataType::Int32, true),+ Field::new("y", DataType::Utf8, true),+ ]+ .into();+ let batch = RecordBatch::try_from_iter(vec![(+ "s",+ Arc::new(StructArray::new(+ physical_fields,+ vec![+ Arc::new(Int32Array::from(vec![1])) as ArrayRef,+ Arc::new(StringArray::from(vec!["bad"])) as ArrayRef,+ ],+ None,+ )) as ArrayRef,+ )])?;+ let table_schema = Arc::new(Schema::new(vec![Field::new(+ "s",+ DataType::Struct(+ vec![+ Field::new("x", DataType::Int64, true),+ Field::new("y", DataType::Utf8, true),+ ]+ .into(),+ ),+ true,+ )]));+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;+ write_parquet(batch, Arc::clone(&store), "explicit_cast/data.parquet").await;+ let ctx = test_context();+ register_memory_listing_table(&ctx, store, "memory:///explicit_cast/", table_schema)+ .await;++ // The file requires schema adaptation for x. The explicit SQL cast also+ // converts y, and selecting x must not hide that invalid conversion.+ let error = ctx+ .sql("SELECT get_field(CAST(s AS STRUCT<x BIGINT, y INT>), 'x') FROM t")+ .await?+ .collect()+ .await+ .unwrap_err()+ .to_string();+ datafusion_common::assert_contains!(error, "While casting struct field 'y'");+ Ok(())+}++#[tokio::test]+async fn test_all_null_struct_decimal_cast_filter_pushdown() -> Result<()> {+ use arrow::array::new_null_array;+ use datafusion_physical_plan::{collect, displayable};++ for (physical_type, logical_type) in [+ (DataType::Utf8, DataType::Decimal128(10, -1)),+ (+ DataType::new_list(DataType::Utf8, true),+ DataType::new_list(DataType::Decimal128(10, -1), true),+ ),+ ] {+ let physical_fields: Fields =+ vec![Field::new("x", physical_type.clone(), true)].into();+ let batch = RecordBatch::try_from_iter(vec![+ ("row_id", Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef),+ (+ "s",+ Arc::new(StructArray::new(+ physical_fields,+ vec![new_null_array(&physical_type, 2)],+ Some(NullBuffer::new_null(2)),+ )) as ArrayRef,+ ),+ ])?;+ let table_schema = Arc::new(Schema::new(vec![+ Field::new("row_id", DataType::Int32, false),+ Field::new(+ "s",+ DataType::Struct(vec![Field::new("x", logical_type, true)].into()),+ true,+ ),+ ]));+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;+ write_parquet(batch, Arc::clone(&store), "null_decimal/data.parquet").await;++ for pushdown_filters in [false, true] {+ let mut config = SessionConfig::new()+ .with_collect_statistics(false)+ .with_parquet_pruning(false)+ .with_parquet_page_index_pruning(false);+ config.options_mut().execution.parquet.pushdown_filters = pushdown_filters;+ let ctx = SessionContext::new_with_config(config);+ register_memory_listing_table(+ &ctx,+ Arc::clone(&store),+ "memory:///null_decimal/",+ Arc::clone(&table_schema),+ )+ .await;++ for (predicate, expected_rows) in [("IS NULL", 2), ("IS NOT NULL", 0)] {+ let plan = ctx+ .sql(&format!(+ "SELECT row_id FROM t WHERE get_field(s, 'x') {predicate}"+ ))+ .await?+ .create_physical_plan()+ .await?;+ if pushdown_filters {+ let plan_text = displayable(plan.as_ref()).indent(false).to_string();+ assert!(+ !plan_text.contains("FilterExec"),+ "the scan must fully handle the filter: {plan_text}"+ );+ }+ let batches = collect(plan, ctx.task_ctx()).await?;+ assert_eq!(+ batches.iter().map(RecordBatch::num_rows).sum::<usize>(),+ expected_rows,+ "physical_type={physical_type:?}, pushdown_filters={pushdown_filters}, predicate={predicate}"+ );+ }+ }+ }+ Ok(())+}++#[tokio::test]+async fn test_evolved_decimal_ignores_unselected_sibling() -> Result<()> {+ let physical_fields: Fields = vec![+ Field::new("x", DataType::Int32, true),+ Field::new("y", DataType::Utf8, true),+ ]+ .into();+ let batch = RecordBatch::try_from_iter(vec![+ ("row_id", Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef),+ (+ "s",+ Arc::new(StructArray::new(+ physical_fields,+ vec![+ Arc::new(Int32Array::from(vec![1, 0])),+ Arc::new(StringArray::from(vec!["bad", "bad"])),+ ],+ None,+ )) as ArrayRef,+ ),+ ])?;+ let table_schema = Arc::new(Schema::new(vec![+ Field::new("row_id", DataType::Int32, false),+ Field::new(+ "s",+ DataType::Struct(+ vec![+ Field::new("x", DataType::Decimal128(10, 2), true),+ Field::new("y", DataType::Int32, true),+ ]+ .into(),+ ),+ true,+ ),+ ]));+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;+ write_parquet(batch, Arc::clone(&store), "decimal_sibling/data.parquet").await;++ for pushdown_filters in [false, true] {+ let mut config = SessionConfig::new()+ .with_collect_statistics(false)+ .with_parquet_pruning(false)+ .with_parquet_page_index_pruning(false);+ config.options_mut().execution.parquet.pushdown_filters = pushdown_filters;+ let ctx = SessionContext::new_with_config(config);+ register_memory_listing_table(+ &ctx,+ Arc::clone(&store),+ "memory:///decimal_sibling/",+ Arc::clone(&table_schema),+ )+ .await;++ // Adapting x must not evaluate the invalid conversion of y.+ for (sql, expected) in [+ (+ "SELECT get_field(s, 'x') AS x FROM t",+ vec![+ "+------+", "| x |", "+------+", "| 1.00 |", "| 0.00 |",+ "+------+",+ ],+ ),+ (+ "SELECT row_id FROM t WHERE get_field(s, 'x') > 0",+ vec![+ "+--------+",+ "| row_id |",+ "+--------+",+ "| 1 |",+ "+--------+",+ ],+ ),+ ] {+ let batches = ctx.sql(sql).await?.collect().await?;+ assert_batches_eq!(expected, &batches);+ }+ }+ Ok(())+}+#[tokio::test]async fn test_struct_schema_evolution_projection_and_filter() -> Result<()> {use std::collections::HashMap;diff --git a/datafusion/datasource-parquet/src/projection_read_plan.rs b/datafusion/datasource-parquet/src/projection_read_plan.rsindex 7793852acf..9c7028cb77 100644--- a/datafusion/datasource-parquet/src/projection_read_plan.rs+++ b/datafusion/datasource-parquet/src/projection_read_plan.rs@@ -178,12 +178,18 @@ pub(crate) struct PushdownChecker<'schema> {/// Struct field accesses via `get_field`.struct_field_accesses: Vec<StructFieldAccess>,/// Whole-column casts to a narrower nested type- /// (`CAST(col AS narrower_struct)`). Only collected when- /// [`Self::with_cast_collection`] enables it (projection analysis);- /// filter pushdown leaves this off.+ /// (`CAST(col AS narrower_struct)`), collected either when+ /// [`Self::with_cast_collection`] enables it (projection analysis) or when+ /// [`Self::allow_struct_casts`] accepts a retained cast under a `get_field`+ /// (filter pushdown).cast_accesses: Vec<CastColumnAccess>,/// Whether to collect [`Self::cast_accesses`].collect_cast_accesses: bool,+ /// Allow `get_field(CAST(struct_column AS Struct(...)), 'field', ...)`+ /// after schema adaptation, preserving the cast and reading the leaves its+ /// target names. Both source and target must be Struct types.+ /// Planning keeps this disabled so explicit casts retain a residual filter.+ allow_struct_casts: bool,/// Whether nested list columns are supported by the predicate semantics.allow_list_columns: bool,/// The Arrow schema of the parquet file.@@ -191,7 +197,11 @@ pub(crate) struct PushdownChecker<'schema> {}impl<'schema> PushdownChecker<'schema> {- pub(crate) fn new(file_schema: &'schema Schema, allow_list_columns: bool) -> Self {+ pub(crate) fn new(+ file_schema: &'schema Schema,+ allow_list_columns: bool,+ allow_struct_casts: bool,+ ) -> Self {Self {non_primitive_columns: false,projected_columns: false,@@ -200,6 +210,7 @@ impl<'schema> PushdownChecker<'schema> {struct_field_accesses: Vec::new(),cast_accesses: Vec::new(),collect_cast_accesses: false,+ allow_struct_casts,allow_list_columns,file_schema,}@@ -248,6 +259,68 @@ impl<'schema> PushdownChecker<'schema> {None}+ /// Preserve a Struct cast retained by schema adaptation and record the+ /// leaves its target consumes.+ ///+ /// The cast is kept intact — moving it could change errors or nulls — but+ /// the target itself names every field the conversion touches, so the read+ /// can be clipped to those leaves. `cast_struct_column` resolves source+ /// children by name and ignores the rest, so a leaf the target does not+ /// name cannot affect the result. Note this clips by the *cast target*, not+ /// by the `get_field` key: an explicit query cast names every field the+ /// user asked to convert, so its siblings stay in the read and their+ /// conversions still run.+ fn check_cast_struct_field_access(+ &mut self,+ func: &ScalarFunctionExpr,+ ) -> Option<TreeNodeRecursion> {+ if !self.allow_struct_casts {+ return None;+ }+ let (source, field_names) = func.args().split_first()?;+ if field_names.is_empty() {+ return None;+ }+ let cast = source.downcast_ref::<CastExpr>()?;+ let column = cast.expr().downcast_ref::<Column>()?;+ let index = self.file_schema.index_of(column.name()).ok()?;+ if !matches!(+ self.file_schema.field(index).data_type(),+ DataType::Struct(_)+ ) {+ return None;+ }+ let return_type = func.return_type();+ if DataType::is_nested(return_type) && !self.is_nested_type_supported(return_type)+ {+ return None;+ }++ // Every key must resolve through Struct fields in the cast target.+ // In particular, a key following a Map field is a runtime lookup.+ let mut data_type = cast.cast_type();+ for field_name in field_names {+ let name = field_name+ .downcast_ref::<Literal>()?+ .value()+ .try_as_str()+ .flatten()?;+ let DataType::Struct(fields) = data_type else {+ return None;+ };+ data_type = fields+ .iter()+ .find(|field| field.name() == name)?+ .data_type();+ }++ self.cast_accesses.push(CastColumnAccess {+ root_index: index,+ target_type: cast.cast_type().clone(),+ });+ Some(TreeNodeRecursion::Jump)+ }+fn check_single_column(&mut self, column_name: &str) -> Option<TreeNodeRecursion> {let Ok(idx) = self.file_schema.index_of(column_name) else {// Column does not exist in the file schema, so we can't push this down.@@ -344,6 +417,9 @@ impl TreeNodeVisitor<'_> for PushdownChecker<'_> {if let Some(func) =ScalarFunctionExpr::try_downcast_func::<GetFieldFunc>(node.as_ref()){+ if let Some(recursion) = self.check_cast_struct_field_access(func) {+ return Ok(recursion);+ }let args = func.args();if let Some(column) = args.first().and_then(|a| a.downcast_ref::<Column>()) {@@ -455,8 +531,8 @@ pub(crate) struct PushdownColumns {/// Struct field accesses via `get_field`. Each entry records the root struct/// column index and the field path being accessed.pub(crate) struct_field_accesses: Vec<StructFieldAccess>,- /// Whole-column casts to a narrower nested type. Empty unless cast- /// collection was enabled on the checker.+ /// Whole-column casts to a narrower nested type, collected for projections+ /// or retained Struct casts accepted by the runtime filter checker.pub(crate) cast_accesses: Vec<CastColumnAccess>,}@@ -524,7 +600,8 @@ pub(crate) fn build_projection_read_plan(let mut all_cast_accesses = Vec::new();for expr in exprs {- let mut checker = PushdownChecker::new(file_schema, true).with_cast_collection();+ let mut checker =+ PushdownChecker::new(file_schema, true, false).with_cast_collection();let _ = expr.visit(&mut checker);let columns = checker.into_sorted_columns();@@ -543,13 +620,14 @@ pub(crate) fn build_projection_read_plan(all_cast_accesses.retain(|c| all_root_indices.binary_search(&c.root_index).is_err());if !all_cast_accesses.is_empty() {- return build_read_plan_with_cast_clipping(+ let (read_plan, _leaf_indices) = build_read_plan_with_cast_clipping(file_schema,schema_descr,&all_root_indices,&all_struct_accesses,&all_cast_accesses,);+ return read_plan;}// when no struct field accesses were found, fall back to root-level projection@@ -581,8 +659,7 @@ enum RootRead {////// Per root, in ascending root-index order:/// - roots referenced as whole columns keep every leaf and their full-/// physical field (whole-column reads take precedence; cast accesses on-/// such roots were already dropped by the caller);+/// physical field (whole-column reads take precedence over cast accesses);/// - roots consumed through one or more casts keep the union of the leaves/// their targets name (see `crate::nested_schema_pruning`);/// - a root consumed through both casts and `get_field` accesses keeps the@@ -593,13 +670,16 @@ enum RootRead {/// `nested_schema_pruning::clip_for_cast`), an access that resolves to no/// leaf at all, or a merged leaf set whose emitted Arrow type can't be/// derived safely, falls back to a full read of that root.-fn build_read_plan_with_cast_clipping(+///+/// Also returns the resolved Parquet leaf indices, sorted and deduplicated, so+/// callers can size the columns the decoder will read.+pub(crate) fn build_read_plan_with_cast_clipping(file_schema: &Schema,schema_descr: &SchemaDescriptor,whole_root_indices: &[usize],struct_accesses: &[StructFieldAccess],cast_accesses: &[CastColumnAccess],-) -> ParquetReadPlan {+) -> (ParquetReadPlan, Vec<usize>) {// Every referenced root's Parquet leaves, grouped in one pass over the// schema descriptor rather than one `leaf_indices_for_roots` scan per// root (this function may look up several roots).@@ -703,17 +783,24 @@ fn build_read_plan_with_cast_clipping(fields.push(Arc::new(field.clone()));}// `ProjectionMask::leaves` only flips flags in a `vec![false; num_columns]`,- // so `leaf_indices` needs no sorting or deduplication here.- ParquetReadPlan {- projection_mask: ProjectionMask::leaves(- schema_descr,- leaf_indices.iter().copied(),- ),- projected_schema: Arc::new(Schema::new_with_metadata(- fields,- file_schema.metadata().clone(),- )),- }+ // so the mask itself needs neither ordering nor deduplication. Callers that+ // size the read do care, so normalize before handing the indices back:+ // `size_of_columns` sums per index and would double-count a repeat.+ leaf_indices.sort_unstable();+ leaf_indices.dedup();+ (+ ParquetReadPlan {+ projection_mask: ProjectionMask::leaves(+ schema_descr,+ leaf_indices.iter().copied(),+ ),+ projected_schema: Arc::new(Schema::new_with_metadata(+ fields,+ file_schema.metadata().clone(),+ )),+ },+ leaf_indices,+ )}/// Groups every Parquet leaf index by its root (Arrow) column index, in one@@ -1693,7 +1780,7 @@ mod test {vec![Arc::new(Field::new("p", DataType::Int32, true))].into(),),};- let read_plan = build_read_plan_with_cast_clipping(+ let (read_plan, _leaf_indices) = build_read_plan_with_cast_clipping(&file_schema,schema_descr,&[],@@ -1863,7 +1950,7 @@ mod test {Schema::new(vec![divergent("a", "p", "q"), divergent("b", "m", "n")]);// `a` is reached by a narrowing cast, `b` only by `get_field`.- let read_plan = build_read_plan_with_cast_clipping(+ let (read_plan, _leaf_indices) = build_read_plan_with_cast_clipping(&file_schema,schema_descr,&[],diff --git a/datafusion/datasource-parquet/src/row_filter.rs b/datafusion/datasource-parquet/src/row_filter.rsindex 267f0f5917..833764ba08 100644--- a/datafusion/datasource-parquet/src/row_filter.rs+++ b/datafusion/datasource-parquet/src/row_filter.rs@@ -87,6 +87,7 @@ use super::ParquetFileMetrics;use super::supported_predicates::supports_list_predicates;use crate::projection_read_plan::{ParquetReadPlan, PushdownChecker, PushdownColumns, assemble_read_plan,+ build_read_plan_with_cast_clipping,};/// A "compiled" predicate passed to `ParquetRecordBatchStream` to perform@@ -243,12 +244,17 @@ impl FilterCandidateBuilder {////// Returns `None` if the expression cannot be pushed down (e.g., references/// unsupported nested types or columns not in the file).+/// Struct casts are accepted only after schema adaptation, not while planning+/// against the table schema: adaptation may insert another cast underneath an+/// explicit cast, leaving an expression the runtime checker cannot handle.fn pushdown_columns(expr: &Arc<dyn PhysicalExpr>,file_schema: &Schema,+ allow_struct_casts: bool,) -> Result<Option<PushdownColumns>> {let allow_list_columns = supports_list_predicates(expr);- let mut checker = PushdownChecker::new(file_schema, allow_list_columns);+ let mut checker =+ PushdownChecker::new(file_schema, allow_list_columns, allow_struct_casts);expr.visit(&mut checker)?;Ok((!checker.prevents_pushdown()).then(|| checker.into_sorted_columns()))}@@ -272,16 +278,30 @@ pub(crate) fn build_parquet_read_plan() -> Result<Option<(ParquetReadPlan, usize)>> {let schema_descr = metadata.file_metadata().schema_descr();- let Some(required_columns) = pushdown_columns(expr, file_schema)? else {+ let Some(required_columns) = pushdown_columns(expr, file_schema, true)? else {return Ok(None);};- let (read_plan, leaf_indices) = assemble_read_plan(- &required_columns.required_columns,- &required_columns.struct_field_accesses,- file_schema,- schema_descr,- );+ // A retained Struct cast names the fields its conversion touches, so the+ // read is clipped to those leaves rather than decoding the whole root.+ // A cast whose target covers every leaf, or that cannot be clipped safely,+ // falls back to a full read of that root inside the helper.+ let (read_plan, leaf_indices) = if required_columns.cast_accesses.is_empty() {+ assemble_read_plan(+ &required_columns.required_columns,+ &required_columns.struct_field_accesses,+ file_schema,+ schema_descr,+ )+ } else {+ build_read_plan_with_cast_clipping(+ file_schema,+ schema_descr,+ &required_columns.required_columns,+ &required_columns.struct_field_accesses,+ &required_columns.cast_accesses,+ )+ };let required_bytes = size_of_columns(&leaf_indices, metadata)?;@@ -358,7 +378,7 @@ pub fn can_expr_be_pushed_down_with_schemas(expr: &Arc<dyn PhysicalExpr>,file_schema: &Schema,) -> bool {- match pushdown_columns(expr, file_schema) {+ match pushdown_columns(expr, file_schema, false) {Ok(Some(_)) => true,Ok(None) | Err(_) => false,}@@ -1232,11 +1252,12 @@ mod test {fn get_field_filter_candidate_has_correct_leaf_indices() {use arrow::array::{Int32Array, StringArray, StructArray};- // Schema: id (Int32), s (Struct{value: Int32, label: Utf8})- // Parquet leaves: id=0, s.value=1, s.label=2+ // Schema: id (Int32), s (Struct{value: Int32, label: Utf8, unused: Utf8})+ // Parquet leaves: id=0, s.value=1, s.label=2, s.unused=3let struct_fields: Fields = vec![Arc::new(Field::new("value", DataType::Int32, false)),Arc::new(Field::new("label", DataType::Utf8, false)),+ Arc::new(Field::new("unused", DataType::Utf8, false)),].into();let schema = Arc::new(Schema::new(vec![@@ -1253,6 +1274,9 @@ mod test {vec![Arc::new(Int32Array::from(vec![10, 20, 30])) as _,Arc::new(StringArray::from(vec!["a", "b", "c"])) as _,+ Arc::new(StringArray::from(vec![+ "unused-a", "unused-b", "unused-c",+ ])) as _,],None,)),@@ -1281,19 +1305,151 @@ mod test {let expr = get_field_expr.gt(Expr::Literal(ScalarValue::Int32(Some(5)), None));let expr = logical2physical(&expr, &file_schema);- let candidate = FilterCandidateBuilder::new(expr, file_schema)- .build(&metadata)- .expect("building candidate")- .expect("get_field filter on struct should be pushable");+ let candidate =+ FilterCandidateBuilder::new(Arc::clone(&expr), Arc::clone(&file_schema))+ .build(&metadata)+ .expect("building candidate")+ .expect("get_field filter on struct should be pushable");// The filter accesses only s.value, so only Parquet leaf 1 is needed.- // Leaf 2 (s.label) is not read, reducing unnecessary I/O.+ // Neither sibling is read, reducing unnecessary I/O.let expected_mask =ProjectionMask::leaves(metadata.file_metadata().schema_descr(), [1]);assert_eq!(candidate.read_plan.projection_mask, expected_mask,"projection_mask should select only the accessed struct field leaf");++ // Schema adaptation retains Struct ancestors for some decimal conversions+ // so an all-null parent can skip child conversion. Runtime filters must+ // preserve every conversion named by the retained cast target.+ // This target includes `label`, so its conversion must still run even+ // though get_field selects only `value`. Planning keeps a residual filter+ // for explicit Struct casts.+ let cast_type = DataType::Struct(+ vec![+ Field::new("value", DataType::Int32, false),+ Field::new("label", DataType::Int32, true),+ ]+ .into(),+ );+ let cast_field = get_field().call(vec![+ datafusion_expr::cast(col("s"), cast_type),+ lit("value"),+ ]);+ let projection = logical2physical(&cast_field, &file_schema);+ let cast_predicate = logical2physical(&cast_field.gt(lit(5)), &file_schema);+ assert!(!can_expr_be_pushed_down_with_schemas(+ &cast_predicate,+ &file_schema+ ));+ let candidate =+ FilterCandidateBuilder::new(cast_predicate, Arc::clone(&file_schema))+ .build(&metadata)+ .expect("building cast candidate")+ .expect("an adapted struct cast must remain evaluable");+ // Clip the unused sibling, but preserve the failing `label` conversion.+ let expected_mask =+ ProjectionMask::leaves(metadata.file_metadata().schema_descr(), [1, 2]);+ assert_eq!(candidate.read_plan.projection_mask, expected_mask);+ let DataType::Struct(physical_fields) = file_schema.field(1).data_type() else {+ unreachable!("s is a struct")+ };+ let clipped_field =+ file_schema+ .field(1)+ .clone()+ .with_data_type(DataType::Struct(+ physical_fields.iter().take(2).cloned().collect(),+ ));+ assert_eq!(+ candidate.read_plan.projected_schema.as_ref(),+ &Schema::new(vec![clipped_field])+ );+ assert_eq!(+ candidate.required_bytes,+ (metadata.row_group(0).column(1).compressed_size()+ + metadata.row_group(0).column(2).compressed_size()) as usize,+ "filter cost must count only the leaves the cast target reads"+ );++ // A simultaneous direct access must not prune siblings that the cast+ // needs in the output projection either.+ let projection_plan = crate::projection_read_plan::build_projection_read_plan(+ [expr, projection],+ &file_schema,+ metadata.file_metadata().schema_descr(),+ );+ assert_eq!(projection_plan.projection_mask, expected_mask);+ assert_eq!(+ projection_plan.projected_schema,+ candidate.read_plan.projected_schema+ );++ let mut row_filter = DatafusionArrowPredicate::try_new(+ candidate,+ Count::new(),+ Count::new(),+ Time::new(),+ )+ .unwrap();+ let batch = builder+ .with_projection(row_filter.projection().clone())+ .build()+ .unwrap()+ .next()+ .unwrap()+ .unwrap();+ let error = row_filter.evaluate(batch).unwrap_err().to_string();+ datafusion_common::assert_contains!(error, "While casting struct field 'label'");++ // A retained cast whose target names only the selected field — the+ // shape `retain_field_path` produces for an evolved decimal — clips the+ // read to that field's leaf instead of decoding the whole root.+ let narrow_cast_type =+ DataType::Struct(vec![Field::new("value", DataType::Int32, false)].into());+ let narrow_field = get_field().call(vec![+ datafusion_expr::cast(col("s"), narrow_cast_type.clone()),+ lit("value"),+ ]);+ let narrow_predicate = logical2physical(&narrow_field.gt(lit(5)), &file_schema);+ let candidate =+ FilterCandidateBuilder::new(narrow_predicate, Arc::clone(&file_schema))+ .build(&metadata)+ .expect("building narrow cast candidate")+ .expect("a clipped struct cast must remain evaluable");+ assert_eq!(+ candidate.read_plan.projection_mask,+ ProjectionMask::leaves(metadata.file_metadata().schema_descr(), [1]),+ "the read must be clipped to the leaf the cast target names"+ );+ assert_eq!(candidate.read_plan.projected_schema.fields().len(), 1);+ assert_eq!(+ candidate.read_plan.projected_schema.field(0).data_type(),+ &narrow_cast_type,+ "sibling leaves must be pruned from the filter schema"+ );++ // The clipped schema must still evaluate: every row has value > 5.+ let mut row_filter = DatafusionArrowPredicate::try_new(+ candidate,+ Count::new(),+ Count::new(),+ Time::new(),+ )+ .unwrap();+ let batch = ParquetRecordBatchReaderBuilder::try_new(file.reopen().unwrap())+ .unwrap()+ .with_projection(row_filter.projection().clone())+ .build()+ .unwrap()+ .next()+ .unwrap()+ .unwrap();+ assert_eq!(+ row_filter.evaluate(batch).unwrap(),+ BooleanArray::from(vec![true, true, true])+ );}/// Deeply nested get_field: get_field(struct_col, 'outer', 'inner') where thediff --git a/datafusion/physical-expr-adapter/src/schema_rewriter.rs b/datafusion/physical-expr-adapter/src/schema_rewriter.rsindex d8cf664707..2548ffc6fb 100644--- a/datafusion/physical-expr-adapter/src/schema_rewriter.rs+++ b/datafusion/physical-expr-adapter/src/schema_rewriter.rs@@ -29,7 +29,7 @@ use arrow::datatypes::{DataType, FieldRef, Fields, SchemaRef};use datafusion_common::{DataFusionError, Result, ScalarValue, exec_err,metadata::FieldMetadata,- nested_struct::validate_data_type_compatibility,+ nested_struct::{requires_nested_struct_cast, validate_data_type_compatibility},tree_node::{Transformed, TransformedResult, TreeNode},};use datafusion_functions::core::getfield::GetFieldFunc;@@ -259,9 +259,10 @@ impl DefaultPhysicalExprAdapter {impl PhysicalExprAdapter for DefaultPhysicalExprAdapter {fn rewrite(&self, expr: Arc<dyn PhysicalExpr>) -> Result<Arc<dyn PhysicalExpr>> {- let rewriter = DefaultPhysicalExprAdapterRewriter {+ let mut rewriter = DefaultPhysicalExprAdapterRewriter {logical_file_schema: Arc::clone(&self.logical_file_schema),physical_file_schema: Arc::clone(&self.physical_file_schema),+ generated_struct_casts: HashMap::new(),};expr.transform(|expr| rewriter.rewrite_expr(Arc::clone(&expr))).data()@@ -271,6 +272,13 @@ impl PhysicalExprAdapter for DefaultPhysicalExprAdapter {struct DefaultPhysicalExprAdapterRewriter {logical_file_schema: SchemaRef,physical_file_schema: SchemaRef,+ // A fresh map is created for each `rewrite()` call. Tracking relies on+ // bottom-up `transform` traversal: a generated child cast is recorded before+ // its parent `get_field` sees the same Arc allocation. Owned Arc clones keep+ // recorded allocations alive even after removal from the tree, preventing+ // pointer-address reuse. Keeping provenance here avoids adding markers to+ // expression types or threading it through rewrite results.+ generated_struct_casts: HashMap<*const (), Arc<dyn PhysicalExpr>>,}/// Outcome of walking a `get_field` key path through nested struct fields.@@ -309,9 +317,63 @@ fn resolve_field_path<'a>(}}+/// Whether a type or any nested value type matches the predicate.+fn contains_type(data_type: &DataType, predicate: &impl Fn(&DataType) -> bool) -> bool {+ if predicate(data_type) {+ return true;+ }+ match data_type {+ DataType::List(field)+ | DataType::LargeList(field)+ | DataType::FixedSizeList(field, _)+ | DataType::ListView(field)+ | DataType::LargeListView(field)+ | DataType::RunEndEncoded(_, field) => {+ contains_type(field.data_type(), predicate)+ }+ DataType::Map(entries, _) => {+ // The entries Struct is a layout wrapper, not a Struct-valued child.+ let DataType::Struct(fields) = entries.data_type() else {+ return false;+ };+ fields+ .iter()+ .any(|field| contains_type(field.data_type(), predicate))+ }+ DataType::Struct(fields) => fields+ .iter()+ .any(|field| contains_type(field.data_type(), predicate)),+ DataType::Union(fields, _) => fields+ .iter()+ .any(|(_, field)| contains_type(field.data_type(), predicate)),+ DataType::Dictionary(_, values) => contains_type(values, predicate),+ _ => false,+ }+}++/// Retain only the selected field path in a cast target, preserving its Struct+/// ancestors' metadata and nullability. This excludes unselected sibling+/// conversions while keeping the all-null Struct shortcut for decimal casts.+fn retain_field_path(field: &FieldRef, path: &[&str]) -> Option<FieldRef> {+ let Some((name, rest)) = path.split_first() else {+ return Some(Arc::clone(field));+ };+ let DataType::Struct(fields) = field.data_type() else {+ return None;+ };+ let child = fields.iter().find(|child| child.name() == *name)?;+ let child = retain_field_path(child, rest)?;+ Some(Arc::new(+ field+ .as_ref()+ .clone()+ .with_data_type(DataType::Struct(vec![child].into())),+ ))+}+impl DefaultPhysicalExprAdapterRewriter {fn rewrite_expr(- &self,+ &mut self,expr: Arc<dyn PhysicalExpr>,) -> Result<Transformed<Arc<dyn PhysicalExpr>>> {if let Some(transformed) = self.try_rewrite_struct_field_access(&expr)? {@@ -319,16 +381,30 @@ impl DefaultPhysicalExprAdapterRewriter {}if let Some(transformed) = self.try_narrow_struct_cast(&expr)? {+ // A narrowed Struct cast may be accessed by another get_field.+ self.record_generated_struct_cast(&transformed);return Ok(Transformed::yes(transformed));}if let Some(column) = expr.downcast_ref::<Column>() {- return self.rewrite_column(Arc::clone(&expr), column);+ let transformed = self.rewrite_column(Arc::clone(&expr), column)?;+ self.record_generated_struct_cast(&transformed.data);+ return Ok(transformed);}Ok(Transformed::no(expr))}+ fn record_generated_struct_cast(&mut self, expr: &Arc<dyn PhysicalExpr>) {+ if expr+ .downcast_ref::<CastExpr>()+ .is_some_and(|cast| matches!(cast.cast_type(), DataType::Struct(_)))+ {+ self.generated_struct_casts+ .insert(Arc::as_ptr(expr).cast::<()>(), Arc::clone(expr));+ }+ }+/// Rewrite `get_field(cast(s AS Struct<..>), 'f')` into/// `cast(get_field(s, 'f') AS <type of f>)`.///@@ -365,7 +441,9 @@ impl DefaultPhysicalExprAdapterRewriter {/// simplified to `get_field(s, 'a', 'b')`, so the whole field path is/// resolved here rather than just the first key.///- /// Only struct casts are narrowed. `get_field` on a Map column performs a+ /// Only struct casts introduced by this adapter are narrowed. Explicit+ /// casts must still evaluate sibling conversions, which may fail.+ /// `get_field` on a Map column performs a/// runtime key lookup rather than a schema-level field access, so the map/// value must keep its cast.fn try_narrow_struct_cast(@@ -384,6 +462,12 @@ impl DefaultPhysicalExprAdapterRewriter {let Some(cast) = source_expr.downcast_ref::<CastExpr>() else {return Ok(None);};+ if !self+ .generated_struct_casts+ .contains_key(&Arc::as_ptr(source_expr).cast::<()>())+ {+ return Ok(None);+ }// Every key has to be a string literal, otherwise the leaf field// cannot be resolved statically.@@ -436,6 +520,53 @@ impl DefaultPhysicalExprAdapterRewriter {FieldPathResolution::NotAStruct => return Ok(None),};+ // Decimal conversions, including those inside containers, can fail+ // during setup even for all-null inputs, while a Struct cast skips its+ // children when the parent is all null.+ // Keep the Struct ancestors for that shortcut, but exclude unselected+ // siblings whose conversions may fail. Same-type metadata casts remain+ // safe to narrow to a scalar cast.+ // A Struct-to-Struct leaf cast keeps its own shortcut and must remain+ // narrowable by a parent get_field.+ // Container casts involving Struct values must keep their existing+ // dispatch: Arrow can unwrap a container into a Struct where cast_column+ // cannot. That leaves one shape uncovered: unwrapping hands the Struct+ // to Arrow's own cast, which has no all-null shortcut, so a decimal+ // below it can still fail on an all-null input (for example+ // `Dictionary(Int8, Struct<b: Utf8>)` to `Struct<b: Decimal128(10, -1)>`).+ // The Parquet reader does not produce dictionary-encoded Struct columns,+ // so this is not reachable through a Parquet scan; closing it would mean+ // telling "Arrow must unwrap this" apart from "Arrow will convert a+ // decimal while unwrapping" rather than dropping the carve-out.+ let source_type = physical_struct_field.data_type();+ let target_type = logical_struct_field.data_type();+ let is_struct = |data_type: &DataType| matches!(data_type, DataType::Struct(_));+ if source_type != target_type+ && !matches!(+ (source_type, target_type),+ (DataType::Struct(_), DataType::Struct(_))+ )+ && (contains_type(source_type, &DataType::is_decimal)+ || contains_type(target_type, &DataType::is_decimal))+ && (source_type.is_decimal()+ || target_type.is_decimal()+ || requires_nested_struct_cast(source_type, target_type)+ || (!contains_type(source_type, &is_struct)+ && !contains_type(target_type, &is_struct)))+ {+ let Some(target_field) = retain_field_path(cast.target_field(), &field_path)+ else {+ return Ok(None);+ };+ let mut args = get_field_expr.args().to_vec();+ args[0] = Arc::new(CastExpr::new_with_target_field(+ Arc::clone(inner),+ target_field,+ Some(cast.cast_options().clone()),+ ));+ return Arc::clone(expr).with_new_children(args).map(Some);+ }+// Rebuild `get_field` over the uncast struct so its return field is// recomputed from the physical field type.let mut args = Vec::with_capacity(get_field_expr.args().len());@@ -448,12 +579,17 @@ impl DefaultPhysicalExprAdapterRewriter {Arc::new(get_field_expr.config_options().clone()),)?) as Arc<dyn PhysicalExpr>;- if physical_struct_field == logical_struct_field {+ // get_field inherits nullability from every parent along the path.+ // Its complete return field can differ even when the leaf fields match.+ let logical_return_field = expr.return_field(&self.logical_file_schema)?;+ if physical_struct_field == logical_struct_field+ && extracted.return_field(&self.physical_file_schema)? == logical_return_field+ {return Ok(Some(extracted));}Ok(Some(Arc::new(CastExpr::new_with_target_field(extracted,- Arc::clone(logical_struct_field),+ logical_return_field,Some(cast.cast_options().clone()),))))}@@ -1573,6 +1709,7 @@ mod tests {let rewriter = DefaultPhysicalExprAdapterRewriter {logical_file_schema: Arc::new(logical_schema),physical_file_schema: Arc::new(physical_schema),+ generated_struct_casts: HashMap::new(),};// Test that when a field exists in physical schema, it returns None@@ -1623,6 +1760,119 @@ mod tests {(logical, physical)}+ fn decimal_cast_leaf_types(data_type: DataType) -> Vec<DataType> {+ let item = Arc::new(Field::new("item", data_type.clone(), true));+ vec![+ data_type.clone(),+ DataType::List(Arc::clone(&item)),+ DataType::LargeList(Arc::clone(&item)),+ DataType::FixedSizeList(Arc::clone(&item), 2),+ DataType::ListView(Arc::clone(&item)),+ DataType::LargeListView(item),+ DataType::Map(+ Arc::new(Field::new(+ "entries",+ DataType::Struct(+ vec![+ Field::new("key", DataType::Utf8, false),+ Field::new("value", data_type.clone(), true),+ ]+ .into(),+ ),+ false,+ )),+ false,+ ),+ DataType::Dictionary(Box::new(DataType::Int8), Box::new(data_type.clone())),+ DataType::new_list(+ DataType::Struct(+ vec![Field::new("value", data_type.clone(), true)].into(),+ ),+ true,+ ),+ DataType::new_list(DataType::new_list(data_type, true), true),+ ]+ }++ #[test]+ fn test_narrow_struct_cast_preserves_struct_unwrapping() -> Result<()> {+ use arrow::array::{+ ArrayRef, Decimal128Array, DictionaryArray, Int8Array, ListArray,+ };+ use arrow::buffer::OffsetBuffer;+ use arrow::datatypes::Int8Type;++ let values = Arc::new(StructArray::new(+ vec![Field::new("value", DataType::Int32, true)].into(),+ vec![Arc::new(Int32Array::from(vec![1]))],+ None,+ )) as ArrayRef;+ let dictionary = Arc::new(DictionaryArray::<Int8Type>::try_new(+ Int8Array::from(vec![0]),+ values,+ )?) as ArrayRef;+ let expected_struct = Arc::new(StructArray::new(+ vec![Field::new("value", DataType::Decimal128(10, 2), true)].into(),+ vec![Arc::new(+ Decimal128Array::from(vec![100]).with_precision_and_scale(10, 2)?,+ )],+ None,+ )) as ArrayRef;+ let wrap_list = |values: ArrayRef| -> ArrayRef {+ Arc::new(ListArray::new(+ Arc::new(Field::new("item", values.data_type().clone(), true)),+ OffsetBuffer::from_lengths([1]),+ values,+ None,+ ))+ };+ let wrap_dictionary = |values: ArrayRef| -> ArrayRef {+ Arc::new(+ DictionaryArray::<Int8Type>::try_new(Int8Array::from(vec![0]), values)+ .unwrap(),+ )+ };+ for (label, physical, expected) in [+ (+ "direct Dictionary",+ Arc::clone(&dictionary),+ Arc::clone(&expected_struct),+ ),+ (+ "List of Dictionary",+ wrap_list(Arc::clone(&dictionary)),+ wrap_list(Arc::clone(&expected_struct)),+ ),+ (+ "Dictionary of Dictionary",+ wrap_dictionary(dictionary),+ wrap_dictionary(expected_struct),+ ),+ ] {+ let (logical_schema, physical_schema) = struct_schemas(+ vec![Field::new("x", physical.data_type().clone(), true)],+ vec![Field::new("x", expected.data_type().clone(), true)],+ );+ let DataType::Struct(fields) = physical_schema.field(0).data_type() else {+ unreachable!()+ };+ let batch = RecordBatch::try_new(+ Arc::clone(&physical_schema),+ vec![Arc::new(StructArray::new(+ fields.clone(),+ vec![physical],+ None,+ ))],+ )?;+ let adapter = DefaultPhysicalExprAdapterFactory+ .create(Arc::clone(&logical_schema), physical_schema)?;+ let rewritten = adapter.rewrite(get_field_expr(&logical_schema, "s", "x"))?;+ let actual = rewritten.evaluate(&batch)?.into_array(1)?;+ assert_eq!(actual.to_data(), expected.to_data(), "{label}");+ }+ Ok(())+ }+/// `s['x']` where the file stores `x` as `Int32` and the table declares/// `Int64` must cast the extracted field, not the whole struct, so that/// the column stays visible under the `get_field`.@@ -1630,29 +1880,372 @@ mod tests {/// See <https://github.com/apache/datafusion/issues/24109>.#[test]fn test_narrow_struct_cast_to_field_access() {- let (logical_schema, physical_schema) = struct_schemas(- vec![Field::new("x", DataType::Int32, true)],- vec![Field::new("x", DataType::Int64, true)],- );+ for (physical_type, logical_type) in [+ (DataType::Int32, DataType::Int64),+ (+ DataType::new_list(DataType::Int32, true),+ DataType::new_list(DataType::Int64, true),+ ),+ ] {+ let (logical_schema, physical_schema) = struct_schemas(+ vec![Field::new("x", physical_type.clone(), true)],+ vec![Field::new("x", logical_type.clone(), true)],+ );++ let adapter = DefaultPhysicalExprAdapterFactory+ .create(Arc::clone(&logical_schema), physical_schema)+ .unwrap();+ let rewritten = adapter+ .rewrite(get_field_expr(&logical_schema, "s", "x"))+ .unwrap();++ let cast = assert_cast_expr(&rewritten);+ assert_eq!(cast.cast_type(), &logical_type);+ let get_field = cast+ .expr()+ .downcast_ref::<ScalarFunctionExpr>()+ .expect("Expected get_field under the cast");+ assert_eq!(get_field.return_type(), &physical_type);+ assert!(+ get_field.args()[0].downcast_ref::<Column>().is_some(),+ "the struct column must not be hidden behind a cast, got: {rewritten}"+ );+ }+ }- let adapter = DefaultPhysicalExprAdapterFactory- .create(Arc::clone(&logical_schema), physical_schema)- .unwrap();- let rewritten = adapter- .rewrite(get_field_expr(&logical_schema, "s", "x"))- .unwrap();+ /// Selecting one field of an explicit cast must still evaluate sibling+ /// conversions, even when schema adaptation inserts another cast below it.+ #[test]+ fn test_narrow_struct_cast_preserves_explicit_cast_errors() -> Result<()> {+ use arrow::array::{ArrayRef, Int16Array};++ for adapt_input in [false, true] {+ let physical_x_type = if adapt_input {+ DataType::Int16+ } else {+ DataType::Int32+ };+ let (logical_schema, physical_schema) = struct_schemas(+ vec![+ Field::new("x", physical_x_type, true),+ Field::new("y", DataType::Utf8, true),+ ],+ vec![+ Field::new("x", DataType::Int32, true),+ Field::new("y", DataType::Utf8, true),+ ],+ );+ let DataType::Struct(physical_fields) = physical_schema.field(0).data_type()+ else {+ unreachable!()+ };+ let x: ArrayRef = if adapt_input {+ Arc::new(Int16Array::from(vec![1]))+ } else {+ Arc::new(Int32Array::from(vec![1]))+ };+ let batch = RecordBatch::try_new(+ Arc::clone(&physical_schema),+ vec![Arc::new(StructArray::new(+ physical_fields.clone(),+ vec![x, Arc::new(StringArray::from(vec!["bad"]))],+ None,+ ))],+ )?;+ let user_cast = Arc::new(CastExpr::new(+ Arc::new(Column::new("s", 0)),+ DataType::Struct(+ vec![+ Field::new("x", DataType::Int32, true),+ Field::new("y", DataType::Int32, true),+ ]+ .into(),+ ),+ None,+ ));+ let expr = Arc::new(ScalarFunctionExpr::try_new(+ Arc::new(datafusion_expr::ScalarUDF::from(GetFieldFunc::new())),+ vec![user_cast, Arc::new(Literal::new(ScalarValue::from("x")))],+ &logical_schema,+ Arc::new(datafusion_common::config::ConfigOptions::default()),+ )?) as Arc<dyn PhysicalExpr>;+ let original_error = expr.evaluate(&batch).unwrap_err().to_string();+ assert_contains!(original_error, "While casting struct field 'y'");++ let adapter = DefaultPhysicalExprAdapterFactory+ .create(logical_schema, physical_schema)?;+ let rewritten = adapter.rewrite(expr)?;+ let error = rewritten.evaluate(&batch).unwrap_err().to_string();+ assert_contains!(error, "While casting struct field 'y'");+ }+ Ok(())+ }- let cast = assert_cast_expr(&rewritten);- assert_eq!(cast.cast_type(), &DataType::Int64);- let get_field = cast- .expr()- .downcast_ref::<ScalarFunctionExpr>()- .expect("Expected get_field under the cast");- assert_eq!(get_field.return_type(), &DataType::Int32);- assert!(- get_field.args()[0].downcast_ref::<Column>().is_some(),- "the struct column must not be hidden behind a cast, got: {rewritten}"- );+ /// The result inherits nullability from every parent, not just the leaf.+ /// Equal leaf fields do not justify dropping a cast if that loses the+ /// logical return field's nullability.+ #[test]+ fn test_narrow_struct_cast_preserves_logical_return_field() -> Result<()> {+ let metadata = HashMap::from([("logical_meta".to_string(), "1".to_string())]);+ for nested in [false, true] {+ let schema = |leaf: Field, parent_nullable| {+ let (field, root_nullable) = if nested {+ (+ Field::new(+ "inner",+ DataType::Struct(vec![leaf].into()),+ parent_nullable,+ ),+ false,+ )+ } else {+ (leaf, parent_nullable)+ };+ Arc::new(Schema::new(vec![Field::new(+ "s",+ DataType::Struct(vec![field].into()),+ root_nullable,+ )]))+ };+ for same_leaf_type in [false, true] {+ let physical_leaf = Field::new("x", DataType::Int32, false)+ .with_metadata(metadata.clone());+ let logical_leaf = if same_leaf_type {+ physical_leaf.clone()+ } else {+ Field::new("x", DataType::Int64, false).with_metadata(HashMap::from(+ [("logical_meta".to_string(), "2".to_string())],+ ))+ };+ let physical_schema = schema(physical_leaf, false);+ let logical_schema = schema(logical_leaf, true);+ let mut args: Vec<Arc<dyn PhysicalExpr>> =+ vec![Arc::new(Column::new("s", 0))];+ if nested {+ args.push(Arc::new(Literal::new(ScalarValue::from("inner"))));+ }+ args.push(Arc::new(Literal::new(ScalarValue::from("x"))));+ let expr = Arc::new(ScalarFunctionExpr::try_new(+ Arc::new(datafusion_expr::ScalarUDF::from(GetFieldFunc::new())),+ args,+ &logical_schema,+ Arc::new(datafusion_common::config::ConfigOptions::default()),+ )?) as Arc<dyn PhysicalExpr>;+ let expected_field = expr.return_field(&logical_schema)?;+ assert!(expected_field.is_nullable());++ let adapter = DefaultPhysicalExprAdapterFactory+ .create(logical_schema, Arc::clone(&physical_schema))?;+ let rewritten = adapter.rewrite(expr)?;+ assert_eq!(+ rewritten.return_field(&physical_schema)?,+ expected_field,+ "nested={nested}, same_leaf_type={same_leaf_type}"+ );+ assert!(rewritten.nullable(&physical_schema)?);+ }+ }+ Ok(())+ }++ /// Some decimal casts can fail while preparing the conversion, even for+ /// an entirely null input. An all-null Struct skips its child conversions.+ #[test]+ fn test_narrow_struct_cast_preserves_all_null_decimal_casts() -> Result<()> {+ use arrow::array::new_null_array;+ use arrow::buffer::NullBuffer;+ use arrow::datatypes::{UnionFields, UnionMode};++ for (physical_type, logical_type) in [+ (DataType::Decimal128(38, -38), DataType::Decimal128(38, 38)),+ (DataType::Utf8, DataType::Decimal128(10, -1)),+ (DataType::Decimal128(38, -39), DataType::Int64),+ ]+ .into_iter()+ .flat_map(|(physical, logical)| {+ decimal_cast_leaf_types(physical)+ .into_iter()+ .zip(decimal_cast_leaf_types(logical))+ })+ // A directly decimal target must still retain the ancestor even when+ // an unrelated Union arm contains a Struct.+ .chain([(+ DataType::Union(+ UnionFields::try_new(+ [0, 1],+ [+ Field::new("string", DataType::Utf8, true),+ Field::new(+ "struct",+ DataType::Struct(+ vec![Field::new("z", DataType::Int32, true)].into(),+ ),+ true,+ ),+ ],+ )?,+ UnionMode::Dense,+ ),+ DataType::Decimal128(10, -1),+ )]) {+ let (logical_schema, physical_schema) = struct_schemas(+ vec![Field::new("x", physical_type.clone(), true)],+ vec![Field::new("x", logical_type, true)],+ );+ let DataType::Struct(physical_fields) = physical_schema.field(0).data_type()+ else {+ unreachable!()+ };+ let batch = RecordBatch::try_new(+ Arc::clone(&physical_schema),+ vec![Arc::new(StructArray::new(+ physical_fields.clone(),+ vec![new_null_array(&physical_type, 2)],+ Some(NullBuffer::new_null(2)),+ ))],+ )?;+ let adapter = DefaultPhysicalExprAdapterFactory+ .create(Arc::clone(&logical_schema), physical_schema)?;+ let expr = get_field_expr(&logical_schema, "s", "x");++ // Establish the result of the original whole-struct conversion.+ let whole_struct_cast = adapter.rewrite(Arc::new(Column::new("s", 0)))?;+ let original = Arc::clone(&expr).with_new_children(vec![+ whole_struct_cast,+ Arc::new(Literal::new(ScalarValue::from("x"))),+ ])?;+ let expected = original.evaluate(&batch)?.into_array(batch.num_rows())?;+ assert_eq!(expected.null_count(), batch.num_rows());++ let rewritten = adapter.rewrite(expr)?;+ let result = rewritten.evaluate(&batch)?.into_array(batch.num_rows())?;+ assert_eq!(result.to_data(), expected.to_data());+ }+ Ok(())+ }++ #[test]+ fn test_narrow_struct_cast_keeps_matching_decimal_fields_optimized() -> Result<()> {+ for (decimal_type, change_metadata) in+ decimal_cast_leaf_types(DataType::Decimal128(10, -1))+ .into_iter()+ .flat_map(|data_type| [(data_type.clone(), false), (data_type, true)])+ {+ let physical_field = Field::new("x", decimal_type.clone(), true);+ let logical_field = if change_metadata {+ physical_field.clone().with_metadata(HashMap::from([(+ "logical_meta".to_string(),+ "1".to_string(),+ )]))+ } else {+ physical_field.clone()+ };+ let (logical_schema, physical_schema) = struct_schemas(+ vec![physical_field, Field::new("y", DataType::Int32, true)],+ vec![logical_field, Field::new("y", DataType::Int64, true)],+ );+ let expr = get_field_expr(&logical_schema, "s", "x");+ let expected_field = expr.return_field(&logical_schema)?;+ let adapter = DefaultPhysicalExprAdapterFactory+ .create(logical_schema, Arc::clone(&physical_schema))?;+ let rewritten = adapter.rewrite(expr)?;+ assert_eq!(rewritten.return_field(&physical_schema)?, expected_field);+ let extracted = if change_metadata {+ assert_cast_expr(&rewritten).expr()+ } else {+ &rewritten+ };+ let get_field = extracted.downcast_ref::<ScalarFunctionExpr>().unwrap();+ assert!(get_field.args()[0].downcast_ref::<Column>().is_some());+ }+ Ok(())+ }++ #[test]+ fn test_narrow_decimal_struct_cast_ignores_siblings() -> Result<()> {+ use arrow::array::ArrayRef;+ use datafusion_physical_expr::planner::logical2physical;++ for (nested, list_leaf) in+ [(false, false), (true, false), (false, true), (true, true)]+ {+ let (physical_type, logical_type, x, expected) = if list_leaf {+ (+ DataType::new_list(DataType::Int32, true),+ DataType::new_list(DataType::Decimal128(10, 2), true),+ ScalarValue::new_list(+ &[ScalarValue::Int32(Some(1))],+ &DataType::Int32,+ true,+ ) as ArrayRef,+ ScalarValue::List(ScalarValue::new_list(+ &[ScalarValue::Decimal128(Some(100), 10, 2)],+ &DataType::Decimal128(10, 2),+ true,+ )),+ )+ } else {+ (+ DataType::Int32,+ DataType::Decimal128(10, 2),+ Arc::new(Int32Array::from(vec![1])) as ArrayRef,+ ScalarValue::Decimal128(Some(100), 10, 2),+ )+ };+ let mut physical_fields = vec![+ Field::new("x", physical_type, true),+ Field::new("y", DataType::Utf8, true),+ ];+ let mut logical_fields = vec![+ Field::new("x", logical_type, true).with_metadata(HashMap::from([(+ "logical_meta".to_string(),+ "1".to_string(),+ )])),+ Field::new("y", DataType::Int32, true),+ ];+ let mut column = Arc::new(StructArray::new(+ physical_fields.clone().into(),+ vec![x, Arc::new(StringArray::from(vec!["bad"]))],+ None,+ )) as ArrayRef;+ let mut args = vec![datafusion_expr::col("s")];+ if nested {+ physical_fields = vec![+ Field::new("inner", DataType::Struct(physical_fields.into()), true),+ Field::new("y", DataType::Utf8, true),+ ];+ logical_fields = vec![+ Field::new("inner", DataType::Struct(logical_fields.into()), true),+ Field::new("y", DataType::Int32, true),+ ];+ column = Arc::new(StructArray::new(+ physical_fields.clone().into(),+ vec![column, Arc::new(StringArray::from(vec!["bad"]))],+ None,+ ));+ args.push(datafusion_expr::lit("inner"));+ }+ args.push(datafusion_expr::lit("x"));+ let (logical_schema, physical_schema) =+ struct_schemas(physical_fields, logical_fields);+ let expr = logical2physical(+ &datafusion_functions::core::get_field().call(args),+ &logical_schema,+ );+ let expected_field = expr.return_field(&logical_schema)?;+ let batch = RecordBatch::try_new(Arc::clone(&physical_schema), vec![column])?;+ let adapter = DefaultPhysicalExprAdapterFactory+ .create(logical_schema, Arc::clone(&physical_schema))?;+ let rewritten = adapter.rewrite(expr)?;+ assert_eq!(rewritten.return_field(&physical_schema)?, expected_field);+ let values = rewritten.evaluate(&batch)?.into_array(1)?;+ assert_eq!(+ ScalarValue::try_from_array(&values, 0)?,+ expected,+ "nested={nested}, list_leaf={list_leaf}"+ );+ }+ Ok(())}/// A struct field that only differs in a nested leaf type still ends up@@ -1662,12 +2255,24 @@ mod tests {let (logical_schema, physical_schema) = struct_schemas(vec![Field::new("inner",- DataType::Struct(vec![Field::new("x", DataType::Utf8, true)].into()),+ DataType::Struct(+ vec![+ Field::new("x", DataType::Utf8, true),+ Field::new("y", DataType::Utf8, true),+ ]+ .into(),+ ),true,)],vec![Field::new("inner",- DataType::Struct(vec![Field::new("x", DataType::Utf8View, true)].into()),+ DataType::Struct(+ vec![+ Field::new("x", DataType::Utf8View, true),+ Field::new("y", DataType::Decimal128(10, -1), true),+ ]+ .into(),+ ),true,)],);diff --git a/datafusion/sqllogictest/test_files/schema_evolution_nested.slt b/datafusion/sqllogictest/test_files/schema_evolution_nested.sltindex 53bc16fe51..ce6b50d5d1 100644--- a/datafusion/sqllogictest/test_files/schema_evolution_nested.slt+++ b/datafusion/sqllogictest/test_files/schema_evolution_nested.slt@@ -16,7 +16,7 @@# under the License.##########-# End-user-facing happy-path coverage for nested list/struct Parquet schema evolution.+# End-user-facing coverage for nested list/struct Parquet schema evolution.## These tests generate mixed-schema parquet files through SQL COPY statements and# query them through CREATE EXTERNAL TABLE, rather than constructing batches@@ -122,3 +122,128 @@ FROM large_list_messages;----1 10 NULL2 30 eth++##########+# Schema adaptation widens x from Int32 to BIGINT. An explicit SQL Struct cast+# also converts y, so selecting only x must not hide that invalid conversion.+##########++statement ok+COPY (+ SELECT named_struct('x', arrow_cast(1, 'Int32'), 'y', arrow_cast('bad', 'Utf8')) AS s+) TO 'test_files/scratch/schema_evolution_nested/explicit_struct_cast/data.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE explicit_struct_cast (+ s STRUCT<x BIGINT, y VARCHAR>+)+STORED AS PARQUET+LOCATION 'test_files/scratch/schema_evolution_nested/explicit_struct_cast/';++statement error While casting struct field 'y'+SELECT get_field(CAST(s AS STRUCT<x BIGINT, y INT>), 'x') FROM explicit_struct_cast;++##########+# An all-null Struct must stay null when a child evolves to a decimal whose+# conversion is rejected while it is being set up, before any value is read.+# A whole-Struct conversion never reaches the child; extracting the field first+# would. This holds whether the decimal is the selected field itself or sits+# inside a container below it.+##########++# Disable statistics-based shortcuts so the filters below evaluate the values.+statement ok+SET datafusion.execution.collect_statistics = false;++statement ok+SET datafusion.execution.parquet.pruning = false;++statement ok+SET datafusion.execution.parquet.enable_page_index = false;++statement ok+SET datafusion.execution.parquet.pushdown_filters = false;++statement ok+COPY (+ SELECT 1 AS row_id,+ CASE WHEN false THEN named_struct('a', arrow_cast('x', 'Utf8')) END AS s+) TO 'test_files/scratch/schema_evolution_nested/null_struct_decimal/data.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE null_struct_decimal (+ row_id INT,+ s STRUCT<a DECIMAL(10, -1)>+)+STORED AS PARQUET+LOCATION 'test_files/scratch/schema_evolution_nested/null_struct_decimal/';++query IR+SELECT row_id, get_field(s, 'a') FROM null_struct_decimal;+----+1 NULL++statement ok+COPY (+ SELECT 1 AS row_id,+ CASE WHEN false+ THEN named_struct('a', arrow_cast(['x'], 'List(Utf8)'))+ END AS s+) TO 'test_files/scratch/schema_evolution_nested/null_struct_list_decimal/data.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE null_struct_list_decimal (+ row_id INT,+ s STRUCT<a ARRAY<DECIMAL(10, -1)>>+)+STORED AS PARQUET+LOCATION 'test_files/scratch/schema_evolution_nested/null_struct_list_decimal/';++query I?+SELECT row_id, s FROM null_struct_list_decimal;+----+1 NULL++query I?+SELECT row_id, get_field(s, 'a') FROM null_struct_list_decimal;+----+1 NULL++# The same null-parent semantics apply with and without Parquet filter pushdown.+query I+SELECT row_id FROM null_struct_decimal WHERE get_field(s, 'a') IS NULL;+----+1++query I+SELECT row_id FROM null_struct_list_decimal WHERE get_field(s, 'a') IS NULL;+----+1++statement ok+SET datafusion.execution.parquet.pushdown_filters = true;++query I+SELECT row_id FROM null_struct_decimal WHERE get_field(s, 'a') IS NULL;+----+1++query I+SELECT row_id FROM null_struct_list_decimal WHERE get_field(s, 'a') IS NULL;+----+1++statement ok+RESET datafusion.execution.collect_statistics;++statement ok+RESET datafusion.execution.parquet.pruning;++statement ok+RESET datafusion.execution.parquet.enable_page_index;++statement ok+RESET datafusion.execution.parquet.pushdown_filters;