Apache DataFusion SQL Query Engine
feat: prune unread Parquet leaves when a nested column is cast to a narrower type (#24090)
Matt Butrovich5eba27f5bd4d
diff --git a/datafusion/core/benches/parquet_nested_schema_pruning.rs b/datafusion/core/benches/parquet_nested_schema_pruning.rsindex 8db67de9ff..de4f0a57a5 100644--- a/datafusion/core/benches/parquet_nested_schema_pruning.rs+++ b/datafusion/core/benches/parquet_nested_schema_pruning.rs@@ -36,10 +36,9 @@//!//! At setup the benchmark reads the parquet scan's `bytes_scanned` metric for//! (1), (2) and (3) so the IO pattern is visible in addition to wall time, and-//! asserts the current baseline: today a narrow declared schema scans the same-//! bytes as the full schema. When nested projection pruning lands, that-//! assertion is expected to fail, which is the signal to flip it to-//! `narrow < full` (see [`assert_scan_baseline`]).+//! asserts that nested projection pruning keeps the narrow declared schema's+//! scan well below the full schema's, close to the physically-narrow floor+//! (see [`assert_scan_prunes`]).use arrow::array::{ArrayRef, Int32Array, Int64Array, ListArray, StringArray, StructArray,@@ -267,15 +266,14 @@ fn scan_bytes(ctx: &SessionContext, rt: &Runtime, sql: &str) -> usize {.expect("parquet scan should report a bytes_scanned metric")}-/// Report and assert the `bytes_scanned` baseline for one dataset shape.+/// Report and assert the `bytes_scanned` improvement for one dataset shape.////// `narrow` selects from a wide file through a narrow declared schema, `full`-/// through the full schema, and `floor` from a physically-narrow file. Today-/// the extra leaves are fetched and discarded, so `narrow == full`; that-/// equality is the checked-in baseline. When nested projection pruning lands,-/// `narrow` should drop toward `floor` and this assertion is expected to fail —-/// the signal to flip it to `assert!(narrow < full)`.-fn assert_scan_baseline(+/// through the full schema, and `floor` from a physically-narrow file.+/// Nested projection pruning clips the narrow read to the declared leaves, so+/// `narrow` should read substantially less than `full`, close to `floor`,+/// the cost of a file that never had the extra leaves to begin with.+fn assert_scan_prunes(ctx: &SessionContext,rt: &Runtime,label: &str,@@ -290,13 +288,11 @@ fn assert_scan_baseline("{label}: bytes_scanned narrow_schema={narrow} full_schema={full} \physically_narrow={floor}");- assert_eq!(- narrow, full,- "{label}: narrow declared schema scanned {narrow} bytes vs {full} for \- the full schema. The baseline is that a narrow schema still reads \- every leaf, so these should be equal; if narrow is now smaller, \- nested projection pruning has likely landed — flip this to \- `assert!(narrow < full)`."+ assert!(+ narrow * 2 < full,+ "{label}: expected the narrow declared schema to read less than half \+ of the full schema's {full} bytes (physically-narrow floor is \+ {floor} bytes), but it read {narrow}");}@@ -363,7 +359,7 @@ fn list_struct_benchmarks(c: &mut Criterion) {let f = setup("list_struct", list_schema, list_batch);let (ctx, rt) = (&f.ctx, &f.rt);- assert_scan_baseline(+ assert_scan_prunes(ctx,rt,"list_struct",@@ -410,7 +406,7 @@ fn top_level_struct_benchmarks(c: &mut Criterion) {let f = setup("struct", struct_schema, struct_batch);let (ctx, rt) = (&f.ctx, &f.rt);- assert_scan_baseline(+ assert_scan_prunes(ctx,rt,"top_level_struct",diff --git a/datafusion/datasource-parquet/src/mod.rs b/datafusion/datasource-parquet/src/mod.rsindex 25b79a6188..35f831230b 100644--- a/datafusion/datasource-parquet/src/mod.rs+++ b/datafusion/datasource-parquet/src/mod.rs@@ -30,6 +30,7 @@ mod decoder_projection;pub mod file_format;pub mod metadata;mod metrics;+mod nested_schema_pruning;mod opener;mod page_filter;mod projection_read_plan;diff --git a/datafusion/datasource-parquet/src/nested_schema_pruning.rs b/datafusion/datasource-parquet/src/nested_schema_pruning.rsnew file mode 100644index 0000000000..9768282c3b--- /dev/null+++ b/datafusion/datasource-parquet/src/nested_schema_pruning.rs@@ -0,0 +1,775 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Schema-driven nested projection pruning.+//!+//! When a scan's projection consumes a nested column only through a cast to a+//! *narrower* nested type, for example the file contains+//! `events: List<Struct<x, y, z, ...>>` but the expression is+//! `CAST(events AS List<Struct<x, y>>)`, the Parquet reader does not need to+//! fetch or decode the leaves the cast target never names. This module+//! computes which Parquet leaves survive such a cast, and the Arrow type the+//! reader will emit for them, by walking the physical and target type trees+//! in parallel and matching struct fields by name (the equivalent of Spark's+//! `ParquetReadSupport.clipParquetSchema`).+//!+//! This situation arises whenever a table's logical schema declares a nested+//! column narrower than the physical Parquet file: the physical expression+//! adapter rewrites the projected column into exactly such a whole-column+//! cast (see `datafusion_physical_expr_adapter`). Engines like Spark+//! communicate nested projection pruning to the scan this way, as a clipped+//! read *schema* rather than as `get_field` expressions.+//!+//! # Safety of clipping+//!+//! The runtime cast for nested types+//! ([`datafusion_common::nested_struct::cast_column`]) consumes source struct+//! children exclusively by looking up the *target* field names, recursively+//! through list wrappers. Physical subtrees not named by the target are+//! provably dead: removing them from the read cannot change the cast's+//! output. That holds for *any*+//! [`CastExpr`](datafusion_physical_expr::expressions::CastExpr) over a+//! nested type, not just the ones the schema adapter inserts:+//! `ColumnarValue::cast_to` routes every+//! cast for which+//! [`requires_nested_struct_cast`](datafusion_common::nested_struct::requires_nested_struct_cast)+//! holds, the same predicate the projection analysis gates on, through+//! `cast_column`.+//!+//! Struct-level nullability is preserved because the Parquet reader+//! reconstructs ancestor validity from the definition levels of any surviving+//! leaf, so every struct level that is clipped must keep at least one leaf.+//! A struct cast with zero field-name overlap at *any* nesting depth would+//! break that: the reader drops a field whose leaves are all masked out, so+//! the emitted type would not match the one predicted here. Such a cast is+//! rejected during physical planning+//! (`datafusion_common::nested_struct::validate_struct_compatibility`, called+//! recursively from `DefaultPhysicalExprAdapter::rewrite`) and by the logical+//! planner's own castability check, so it should never reach this module; if+//! one does anyway (a custom `PhysicalExprAdapter` could build one),+//! [`clip_for_cast`] detects the empty level and declines to clip.+//!+//! The clip is *total*: any type shape it does not understand (maps,+//! dictionaries, wrapper-kind mismatches, ...) keeps all of its leaves, so+//! the worst case is today's behavior of reading the full column. Map values+//! are deliberately not clipped: the runtime cast routes maps through Arrow's+//! positional struct cast, which requires all children to be present. Nor are+//! `ListView`/`LargeListView`/`Dictionary` wrappers clipped here, even though+//! `cast_column` does recurse through them by name. That is a conservative+//! choice (safe, since the worst case is still just a full read) left as a+//! candidate follow-up rather than something this module currently handles.++use std::collections::HashMap;+use std::sync::Arc;++use arrow::datatypes::{DataType, Field, FieldRef, Fields};++/// The single child type one level of container nesting wraps, or `None` for+/// a type this module does not descend through (leaves, `Struct`, `Map`, and+/// wrapper kinds this module intentionally does not clip, see the module+/// doc). Shared by [`count_leaves`] and [`contains_struct`], which otherwise+/// need to agree on the exact same set of container variants.+fn nested_child(dt: &DataType) -> Option<&DataType> {+ match dt {+ DataType::List(f)+ | DataType::LargeList(f)+ | DataType::ListView(f)+ | DataType::LargeListView(f)+ | DataType::FixedSizeList(f, _)+ | DataType::Map(f, _) => Some(f.data_type()),+ DataType::Dictionary(_, value) => Some(value),+ DataType::RunEndEncoded(_, value) => Some(value.data_type()),+ _ => None,+ }+}++/// Clip `physical` against `cast_target`, returning the Parquet leaves the+/// cast actually consumes (as offsets relative to the root column's first+/// leaf, sorted ascending and non-empty) together with the Arrow type the+/// reader will emit for exactly those leaves.+///+/// Returns `None` when nothing can be pruned (every leaf is consumed, or the+/// shapes do not allow safe clipping), in which case the caller should read+/// the whole column as before. This function never fails: unknown shapes+/// degrade to keeping all leaves.+pub(crate) fn clip_for_cast(+ physical: &DataType,+ cast_target: &DataType,+) -> Option<(Vec<usize>, DataType)> {+ let total = count_leaves(physical);+ let mut kept = Vec::new();+ let mut next_leaf = 0;+ let mut unclippable = false;+ let pruned_type = clip_type(+ physical,+ cast_target,+ &mut next_leaf,+ &mut kept,+ &mut unclippable,+ );+ debug_assert_eq!(next_leaf, total, "leaf accounting must cover the type");+ if unclippable || kept.is_empty() || kept.len() >= total {+ return None;+ }+ Some((kept, pruned_type))+}++/// Number of Parquet leaf columns a (Parquet-derived) Arrow type occupies.+pub(crate) fn count_leaves(dt: &DataType) -> usize {+ match dt {+ DataType::Struct(fields) => {+ fields.iter().map(|f| count_leaves(f.data_type())).sum()+ }+ _ => nested_child(dt).map_or(1, count_leaves),+ }+}++/// Does this type contain a struct at any nesting depth? Used as a fast-path+/// gate: a root with no struct anywhere in its type has no leaves this+/// module could ever clip.+pub(crate) fn contains_struct(dt: &DataType) -> bool {+ matches!(dt, DataType::Struct(_)) || nested_child(dt).is_some_and(contains_struct)+}++/// Above this many target fields, matching physical children against them one+/// by one turns into a quadratic string comparison; build a name lookup+/// instead. Below it the map's allocation costs more than the linear scan it+/// saves (Spark's `ParquetReadSupport.clipParquetGroupFields` builds the map+/// unconditionally; struct widths in practice are small enough that the+/// threshold is worth the branch).+const LINEAR_FIELD_SCAN_MAX: usize = 8;++/// Find `name` among `fields`, using `by_name` when it was worth building.+/// Duplicate names resolve to the first occurrence either way.+fn lookup_field<'a>(+ fields: &'a Fields,+ by_name: &Option<HashMap<&'a str, &'a FieldRef>>,+ name: &str,+) -> Option<&'a FieldRef> {+ match by_name {+ Some(map) => map.get(name).copied(),+ None => fields.iter().find(|f| f.name() == name),+ }+}++/// Recursive walker: advances `next_leaf` across every leaf of `physical`,+/// pushing the offsets the cast target consumes into `kept`, and returns the+/// Arrow type the reader emits for those kept leaves.+///+/// `unclippable` is set when a shape is encountered whose emitted type this+/// module cannot predict; the caller must then read the whole column. The walk+/// still runs to completion so `next_leaf` stays a valid leaf count.+fn clip_type(+ physical: &DataType,+ target: &DataType,+ next_leaf: &mut usize,+ kept: &mut Vec<usize>,+ unclippable: &mut bool,+) -> DataType {+ match (physical, target) {+ (DataType::Struct(p_children), DataType::Struct(t_children)) => {+ let t_by_name = (t_children.len() > LINEAR_FIELD_SCAN_MAX).then(|| {+ let mut map = HashMap::with_capacity(t_children.len());+ for tc in t_children.iter() {+ map.entry(tc.name().as_str()).or_insert(tc);+ }+ map+ });+ let kept_children: Fields = p_children+ .iter()+ .filter_map(|pc| {+ let Some(tc) = lookup_field(t_children, &t_by_name, pc.name()) else {+ skip_leaves(pc.data_type(), next_leaf);+ return None;+ };+ let before = kept.len();+ let pruned = clip_type(+ pc.data_type(),+ tc.data_type(),+ next_leaf,+ kept,+ unclippable,+ );+ if kept.len() == before {+ // This child matched by name but kept no leaves at+ // all, which only happens when a nested struct level+ // below it shares no field name with its target. The+ // reader drops a field whose leaves are all masked+ // out, so the emitted type could not be predicted;+ // give up on clipping this column entirely rather+ // than promise a type the decoder will not produce.+ // (`DefaultPhysicalExprAdapter` never builds such a+ // cast — `validate_struct_compatibility` rejects a+ // zero-overlap struct level at planning time — but a+ // custom `PhysicalExprAdapter` could.)+ *unclippable = true;+ }+ Some(field_with_type(pc, pruned))+ })+ .collect();+ DataType::Struct(kept_children)+ }+ (DataType::List(p_item), DataType::List(t_item)) => {+ let pruned = clip_type(+ p_item.data_type(),+ t_item.data_type(),+ next_leaf,+ kept,+ unclippable,+ );+ DataType::List(field_with_type(p_item, pruned))+ }+ (DataType::LargeList(p_item), DataType::LargeList(t_item)) => {+ let pruned = clip_type(+ p_item.data_type(),+ t_item.data_type(),+ next_leaf,+ kept,+ unclippable,+ );+ DataType::LargeList(field_with_type(p_item, pruned))+ }+ // Anything else, leaf pairs, wrapper-kind mismatches, maps,+ // dictionaries, fixed-size lists, views, is kept wholesale.+ _ => keep_all_leaves(physical, next_leaf, kept),+ }+}++/// Keep every leaf of `dt` (no pruning below this point); returns `dt`+/// unchanged since nothing was clipped.+fn keep_all_leaves(+ dt: &DataType,+ next_leaf: &mut usize,+ kept: &mut Vec<usize>,+) -> DataType {+ let n = count_leaves(dt);+ kept.extend(*next_leaf..*next_leaf + n);+ *next_leaf += n;+ dt.clone()+}++fn skip_leaves(dt: &DataType, next_leaf: &mut usize) {+ *next_leaf += count_leaves(dt);+}++/// A projected root column that is consumed through a cast to a narrower+/// nested type (`CAST(col AS target_type)`), recorded during projection+/// analysis.+#[derive(Debug, Clone)]+pub(crate) struct CastColumnAccess {+ /// Arrow root column index of the column in the file schema.+ pub(crate) root_index: usize,+ /// The cast's target type.+ pub(crate) target_type: DataType,+}++/// Rebuild `field` with a new data type, preserving name, nullability and+/// metadata.+pub(crate) fn field_with_type(field: &Field, data_type: DataType) -> FieldRef {+ Arc::new(field.clone().with_data_type(data_type))+}++#[cfg(test)]+mod tests {+ use super::*;++ fn utf8(name: &str) -> Field {+ Field::new(name, DataType::Utf8, true)+ }++ fn int64(name: &str) -> Field {+ Field::new(name, DataType::Int64, true)+ }++ fn struct_of(fields: Vec<Field>) -> DataType {+ DataType::Struct(Fields::from(fields))+ }++ fn list_of(item: DataType) -> DataType {+ DataType::List(Arc::new(Field::new("item", item, true)))+ }++ #[test]+ fn count_leaves_shapes() {+ assert_eq!(count_leaves(&DataType::Int32), 1);+ assert_eq!(count_leaves(&struct_of(vec![utf8("a"), int64("b")])), 2);+ assert_eq!(+ count_leaves(&list_of(struct_of(vec![+ utf8("a"),+ struct_of(vec![int64("x"), int64("y")]).into_field("s")+ ]))),+ 3+ );+ let map = DataType::Map(+ Arc::new(Field::new(+ "entries",+ struct_of(vec![utf8("key"), int64("value")]),+ false,+ )),+ false,+ );+ assert_eq!(count_leaves(&map), 2);+ let dict =+ DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8));+ assert_eq!(count_leaves(&dict), 1);+ // Wrapper kinds must be descended through, not counted as one leaf.+ // A dictionary or run-end-encoded *value* that is itself a struct has+ // as many leaves as the struct: counting it as 1 would misalign every+ // later leaf index in the mask.+ assert_eq!(+ count_leaves(&DataType::Dictionary(+ Box::new(DataType::Int32),+ Box::new(struct_of(vec![utf8("a"), int64("b")]))+ )),+ 2+ );+ assert_eq!(+ count_leaves(&DataType::RunEndEncoded(+ Arc::new(Field::new("run_ends", DataType::Int32, false)),+ Arc::new(Field::new(+ "values",+ struct_of(vec![utf8("a"), int64("b")]),+ true+ ))+ )),+ 2+ );+ }++ /// [`contains_struct`] gates the projection fast path, so it has to agree+ /// with [`count_leaves`] about which wrappers are descended through.+ #[test]+ fn contains_struct_shapes() {+ assert!(!contains_struct(&DataType::Int32));+ assert!(!contains_struct(&list_of(DataType::Int32)));+ assert!(contains_struct(&struct_of(vec![int64("a")])));+ assert!(contains_struct(&list_of(struct_of(vec![int64("a")]))));+ assert!(contains_struct(&DataType::LargeList(Arc::new(Field::new(+ "item",+ struct_of(vec![int64("a")]),+ true+ )))));+ assert!(contains_struct(&DataType::Dictionary(+ Box::new(DataType::Int32),+ Box::new(struct_of(vec![int64("a")]))+ )));+ assert!(!contains_struct(&DataType::Dictionary(+ Box::new(DataType::Int32),+ Box::new(DataType::Utf8)+ )));+ // A map's entries are a struct, so a map always contains one.+ assert!(contains_struct(&DataType::Map(+ Arc::new(Field::new(+ "entries",+ struct_of(vec![utf8("key"), int64("value")]),+ false+ )),+ false+ )));+ }++ /// `{a, b, c} CAST TO {b}` keeps only b's leaf.+ #[test]+ fn clip_struct_subset() {+ let physical = struct_of(vec![utf8("a"), int64("b"), utf8("c")]);+ let target = struct_of(vec![int64("b")]);+ let (kept, emitted) = clip_for_cast(&physical, &target).unwrap();+ assert_eq!(kept, vec![1]);+ assert_eq!(emitted, struct_of(vec![int64("b")]));+ }++ /// Target field order does not matter: emitted type is in physical order.+ #[test]+ fn clip_struct_reordered_target() {+ let physical = struct_of(vec![utf8("a"), int64("b"), utf8("c")]);+ let target = struct_of(vec![utf8("c"), utf8("a")]);+ let (kept, emitted) = clip_for_cast(&physical, &target).unwrap();+ assert_eq!(kept, vec![0, 2]);+ assert_eq!(emitted, struct_of(vec![utf8("a"), utf8("c")]));+ }++ /// Target fields missing from the physical type are ignored (the runtime+ /// cast null-fills them).+ #[test]+ fn clip_struct_target_field_missing_from_physical() {+ let physical = struct_of(vec![utf8("a"), int64("b")]);+ let target = struct_of(vec![utf8("a"), int64("z")]);+ let (kept, emitted) = clip_for_cast(&physical, &target).unwrap();+ assert_eq!(kept, vec![0]);+ assert_eq!(emitted, struct_of(vec![utf8("a")]));+ }++ /// Leaf-level type mismatch (promotion) still clips: the emitted type+ /// keeps the physical leaf type; the cast performs the promotion.+ #[test]+ fn clip_keeps_physical_leaf_types() {+ let physical =+ struct_of(vec![Field::new("x", DataType::Int32, true), utf8("pad")]);+ let target = struct_of(vec![int64("x")]);+ let (kept, emitted) = clip_for_cast(&physical, &target).unwrap();+ assert_eq!(kept, vec![0]);+ assert_eq!(+ emitted,+ struct_of(vec![Field::new("x", DataType::Int32, true)])+ );+ }++ /// Nested struct-in-struct clips at both levels.+ #[test]+ fn clip_nested_struct() {+ let inner_physical = struct_of(vec![int64("x"), utf8("pad_inner")]);+ let physical = struct_of(vec![+ inner_physical.clone().into_field("inner"),+ utf8("pad_outer"),+ ]);+ let target = struct_of(vec![struct_of(vec![int64("x")]).into_field("inner")]);+ let (kept, emitted) = clip_for_cast(&physical, &target).unwrap();+ assert_eq!(kept, vec![0]);+ assert_eq!(+ emitted,+ struct_of(vec![struct_of(vec![int64("x")]).into_field("inner")])+ );+ }++ /// List<Struct>, the headline case.+ #[test]+ fn clip_list_of_struct() {+ let physical = list_of(struct_of(vec![int64("x"), utf8("y"), utf8("pad")]));+ let target = list_of(struct_of(vec![int64("x"), utf8("y")]));+ let (kept, emitted) = clip_for_cast(&physical, &target).unwrap();+ assert_eq!(kept, vec![0, 1]);+ assert_eq!(emitted, list_of(struct_of(vec![int64("x"), utf8("y")])));+ }++ /// Two levels of `list<struct>` nesting, the inner one also narrowed,+ /// the `events: array<struct<..., items: array<struct<...>>>>` shape+ /// reported in `datafusion-comet#4859`, where a sibling struct field at+ /// the outer level (`aux`, standing in for that report's+ /// `latency_parts`) is dropped entirely rather than clipped.+ #[test]+ fn clip_two_level_nested_list_of_struct() {+ let physical = list_of(struct_of(vec![+ int64("a"),+ utf8("pad"),+ struct_of(vec![int64("x"), utf8("y")]).into_field("aux"),+ list_of(struct_of(vec![int64("g"), utf8("pad2")])).into_field("items"),+ ]));+ let target = list_of(struct_of(vec![+ int64("a"),+ list_of(struct_of(vec![int64("g")])).into_field("items"),+ ]));++ let (kept, emitted) = clip_for_cast(&physical, &target).unwrap();+ // a=0, pad=1, aux.x=2, aux.y=3, items.g=4, items.pad2=5: only a and+ // items.g survive; pad, all of aux, and items.pad2 are dropped.+ assert_eq!(kept, vec![0, 4]);+ assert_eq!(+ emitted,+ list_of(struct_of(vec![+ int64("a"),+ list_of(struct_of(vec![int64("g")])).into_field("items"),+ ]))+ );+ }++ #[test]+ fn clip_large_list_of_struct() {+ let item = |fields| Arc::new(Field::new("item", struct_of(fields), true));+ let physical = DataType::LargeList(item(vec![int64("x"), utf8("pad")]));+ let target = DataType::LargeList(item(vec![int64("x")]));+ let (kept, emitted) = clip_for_cast(&physical, &target).unwrap();+ assert_eq!(kept, vec![0]);+ assert_eq!(emitted, DataType::LargeList(item(vec![int64("x")])));+ }++ /// Wrapper-kind mismatch cannot be clipped.+ #[test]+ fn no_clip_on_wrapper_mismatch() {+ let physical = list_of(struct_of(vec![int64("x"), utf8("pad")]));+ let target = DataType::LargeList(Arc::new(Field::new(+ "item",+ struct_of(vec![int64("x")]),+ true,+ )));+ assert!(clip_for_cast(&physical, &target).is_none());+ }++ /// Maps are opaque: never clipped.+ #[test]+ fn no_clip_on_map() {+ let entries = |fields| Arc::new(Field::new("entries", struct_of(fields), false));+ let physical =+ DataType::Map(entries(vec![utf8("key"), int64("a"), int64("b")]), false);+ let target = DataType::Map(entries(vec![utf8("key"), int64("a")]), false);+ assert!(clip_for_cast(&physical, &target).is_none());+ }++ /// Identical types: nothing to prune.+ #[test]+ fn no_clip_when_identical() {+ let t = struct_of(vec![utf8("a"), int64("b")]);+ assert!(clip_for_cast(&t, &t).is_none());+ }++ /// Non-nested types: nothing to prune.+ #[test]+ fn no_clip_on_primitives() {+ assert!(clip_for_cast(&DataType::Int32, &DataType::Int64).is_none());+ }++ /// A struct level with zero field-name overlap can't actually reach this+ /// code: `validate_struct_compatibility` rejects it during physical+ /// planning (see the module doc), so `clip_for_cast` is only ever called+ /// with targets that overlap at every nesting level. If it were reached+ /// anyway, the generic catch-all keeps every leaf, still safe, just+ /// unpruned.+ #[test]+ fn no_clip_on_zero_overlap() {+ let physical = struct_of(vec![utf8("a"), int64("b")]);+ let target = struct_of(vec![utf8("z")]);+ assert!(clip_for_cast(&physical, &target).is_none());+ }++ /// A *nested* struct level with zero field-name overlap must not be+ /// clipped, even when a sibling keeps leaves. The reader drops a field+ /// whose leaves are all masked out (pinned by+ /// [`reader_drops_struct_child_with_no_selected_leaves`]), so predicting+ /// `{inner: Struct[], c}` here would be a schema the decoder never+ /// produces. Read the whole column instead.+ #[test]+ fn no_clip_when_nested_struct_level_has_no_overlap() {+ let physical = struct_of(vec![+ struct_of(vec![int64("a"), int64("b")]).into_field("inner"),+ int64("c"),+ ]);+ let target = struct_of(vec![+ struct_of(vec![int64("z")]).into_field("inner"),+ int64("c"),+ ]);+ assert!(clip_for_cast(&physical, &target).is_none());+ }++ /// Same, one level deeper and behind a list wrapper.+ #[test]+ fn no_clip_when_nested_list_struct_level_has_no_overlap() {+ let physical = struct_of(vec![+ list_of(struct_of(vec![int64("a"), int64("b")])).into_field("items"),+ int64("c"),+ ]);+ let target = struct_of(vec![+ list_of(struct_of(vec![int64("z")])).into_field("items"),+ int64("c"),+ ]);+ assert!(clip_for_cast(&physical, &target).is_none());+ }++ /// Wide structs take the name-map matching path rather than the linear+ /// scan; both must produce the same clip.+ #[test]+ fn clip_wide_struct_matches_by_name() {+ let width = LINEAR_FIELD_SCAN_MAX * 4;+ let physical = struct_of((0..width).map(|i| int64(&format!("f{i}"))).collect());+ // Even fields only, declared in reverse order: the emitted type is+ // still in physical order.+ let target = struct_of(+ (0..width)+ .rev()+ .filter(|i| i % 2 == 0)+ .map(|i| int64(&format!("f{i}")))+ .collect(),+ );+ let (kept, emitted) = clip_for_cast(&physical, &target).unwrap();+ assert_eq!(kept, (0..width).filter(|i| i % 2 == 0).collect::<Vec<_>>());+ assert_eq!(+ emitted,+ struct_of(+ (0..width)+ .filter(|i| i % 2 == 0)+ .map(|i| int64(&format!("f{i}")))+ .collect()+ )+ );+ }++ /// Duplicate physical field names both match the single target field and+ /// are both kept, which is what the reader emits for that mask.+ #[test]+ fn clip_keeps_duplicate_physical_field_names() {+ let physical = struct_of(vec![int64("a"), utf8("pad"), int64("a")]);+ let target = struct_of(vec![int64("a")]);+ let (kept, emitted) = clip_for_cast(&physical, &target).unwrap();+ assert_eq!(kept, vec![0, 2]);+ assert_eq!(emitted, struct_of(vec![int64("a"), int64("a")]));+ }++ /// Pins the arrow-rs behavior the empty-level guard above depends on: a+ /// struct child none of whose leaves are selected disappears from the+ /// type the reader emits, rather than surviving as an empty struct.+ #[test]+ fn reader_drops_struct_child_with_no_selected_leaves() {+ use arrow::array::{ArrayRef, Int64Array, StructArray};+ use arrow::record_batch::RecordBatch;+ use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;+ use parquet::arrow::{ArrowWriter, ProjectionMask};++ let inner_fields = Fields::from(vec![int64("a"), int64("b")]);+ let outer_fields = Fields::from(vec![+ Field::new("inner", DataType::Struct(inner_fields.clone()), true),+ int64("c"),+ ]);+ let inner: ArrayRef = Arc::new(StructArray::new(+ inner_fields,+ vec![+ Arc::new(Int64Array::from(vec![1, 2])) as ArrayRef,+ Arc::new(Int64Array::from(vec![3, 4])) as ArrayRef,+ ],+ None,+ ));+ let outer = StructArray::new(+ outer_fields.clone(),+ vec![inner, Arc::new(Int64Array::from(vec![5, 6])) as ArrayRef],+ None,+ );+ let schema = Arc::new(arrow::datatypes::Schema::new(vec![Field::new(+ "s",+ DataType::Struct(outer_fields),+ true,+ )]));+ let batch =+ RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(outer)]).unwrap();++ let file = tempfile::NamedTempFile::new().unwrap();+ let mut writer =+ ArrowWriter::try_new(file.reopen().unwrap(), schema, None).unwrap();+ writer.write(&batch).unwrap();+ writer.close().unwrap();++ let builder =+ ParquetRecordBatchReaderBuilder::try_new(file.reopen().unwrap()).unwrap();+ assert_eq!(builder.parquet_schema().num_columns(), 3);+ // Keep only s.c (leaf 2): every leaf of s.inner is masked out.+ let mask = ProjectionMask::leaves(builder.parquet_schema(), [2usize]);+ let reader = builder.with_projection(mask).build().unwrap();+ let out: Vec<RecordBatch> = reader.map(|b| b.unwrap()).collect();+ assert_eq!(+ out[0].schema().field(0).data_type(),+ &struct_of(vec![int64("c")]),+ "the fully masked `inner` child is dropped, not emitted as an empty struct"+ );+ }++ /// Pins the arrow-rs behavior this module relies on: selecting a subset+ /// of leaves under a `List<Struct>` column with `ProjectionMask::leaves`+ /// makes the reader emit exactly the type predicted by [`clip_for_cast`],+ /// and null list rows / null struct elements survive (their validity is+ /// reconstructed from the surviving leaves' definition levels).+ #[test]+ fn arrow_reader_emits_clipped_type_for_masked_list_struct() {+ use arrow::array::{+ Array, ArrayRef, Int64Array, ListArray, StringArray, StructArray,+ };+ use arrow::buffer::{NullBuffer, OffsetBuffer};+ use arrow::record_batch::RecordBatch;+ use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;+ use parquet::arrow::{ArrowWriter, ProjectionMask};++ let item_fields = Fields::from(vec![int64("x"), utf8("y"), utf8("pad")]);+ let item_field = Arc::new(Field::new(+ "item",+ DataType::Struct(item_fields.clone()),+ true,+ ));+ let schema = Arc::new(arrow::datatypes::Schema::new(vec![Field::new(+ "events",+ DataType::List(Arc::clone(&item_field)),+ true,+ )]));++ // 3 elements; element 1 is a NULL struct. Rows: [e0, e1], NULL, [e2].+ let columns: Vec<ArrayRef> = vec![+ Arc::new(Int64Array::from(vec![Some(1), None, Some(3)])),+ Arc::new(StringArray::from(vec![Some("a"), None, Some("c")])),+ Arc::new(StringArray::from(vec![Some("p0"), None, Some("p2")])),+ ];+ let struct_validity = NullBuffer::from(vec![true, false, true]);+ let values = StructArray::new(item_fields, columns, Some(struct_validity));+ let list_validity = NullBuffer::from(vec![true, false, true]);+ let events = ListArray::new(+ item_field,+ OffsetBuffer::from_lengths([2, 0, 1]),+ Arc::new(values),+ Some(list_validity),+ );+ let batch =+ RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(events)]).unwrap();++ let file = tempfile::NamedTempFile::new().unwrap();+ let mut writer =+ ArrowWriter::try_new(file.reopen().unwrap(), schema, None).unwrap();+ writer.write(&batch).unwrap();+ writer.close().unwrap();++ // Clip to the narrow target {x, y}.+ let physical = batch.schema().field(0).data_type().clone();+ let target = list_of(struct_of(vec![int64("x"), utf8("y")]));+ let (kept, predicted_type) = clip_for_cast(&physical, &target).unwrap();+ assert_eq!(kept, vec![0, 1]);++ let builder =+ ParquetRecordBatchReaderBuilder::try_new(file.reopen().unwrap()).unwrap();+ let mask = ProjectionMask::leaves(builder.parquet_schema(), kept.iter().copied());+ let reader = builder.with_projection(mask).build().unwrap();+ let out: Vec<RecordBatch> = reader.map(|b| b.unwrap()).collect();+ assert_eq!(out.len(), 1);+ let out = &out[0];++ // Emitted type matches the prediction.+ assert_eq!(out.schema().field(0).data_type(), &predicted_type);++ // Null semantics survive the clip.+ let events = out.column(0).as_any().downcast_ref::<ListArray>().unwrap();+ assert!(events.is_valid(0));+ assert!(events.is_null(1));+ assert!(events.is_valid(2));+ let structs = events+ .values()+ .as_any()+ .downcast_ref::<StructArray>()+ .unwrap();+ assert_eq!(structs.len(), 3);+ assert!(structs.is_valid(0));+ assert!(structs.is_null(1));+ assert!(structs.is_valid(2));+ let x = structs+ .column(0)+ .as_any()+ .downcast_ref::<Int64Array>()+ .unwrap();+ assert_eq!(x.value(0), 1);+ assert_eq!(x.value(2), 3);+ }++ trait IntoField {+ fn into_field(self, name: &str) -> Field;+ }++ impl IntoField for DataType {+ fn into_field(self, name: &str) -> Field {+ Field::new(name, self, true)+ }+ }+}diff --git a/datafusion/datasource-parquet/src/projection_read_plan.rs b/datafusion/datasource-parquet/src/projection_read_plan.rsindex 96c99ab207..350ed9596b 100644--- a/datafusion/datasource-parquet/src/projection_read_plan.rs+++ b/datafusion/datasource-parquet/src/projection_read_plan.rs@@ -35,13 +35,18 @@ use parquet::arrow::ProjectionMask;use parquet::schema::types::SchemaDescriptor;use datafusion_common::Result;+use datafusion_common::nested_struct::requires_nested_struct_cast;use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion, TreeNodeVisitor};use datafusion_functions::core::file_row_index::FileRowIndexFunc;use datafusion_functions::core::getfield::GetFieldFunc;-use datafusion_physical_expr::expressions::{Column, Literal};+use datafusion_physical_expr::expressions::{CastExpr, Column, Literal};use datafusion_physical_expr::utils::collect_columns;use datafusion_physical_expr::{PhysicalExpr, ScalarFunctionExpr};+use crate::nested_schema_pruning::{+ CastColumnAccess, clip_for_cast, contains_struct, count_leaves, field_with_type,+};+/// The result of resolving which Parquet leaf columns and Arrow schema fields/// are needed to evaluate an expression against a Parquet file///@@ -94,6 +99,13 @@ pub(crate) struct PushdownChecker<'schema> {required_columns: Vec<usize>,/// Struct field accesses via `get_field`.struct_field_accesses: Vec<StructFieldAccess>,+ /// Whole-column casts to a narrower nested type+ /// (`CAST(col AS narrower_struct)`). Only collected when+ /// [`Self::with_cast_collection`] enables it (projection analysis);+ /// filter pushdown leaves this off.+ cast_accesses: Vec<CastColumnAccess>,+ /// Whether to collect [`Self::cast_accesses`].+ collect_cast_accesses: bool,/// Whether nested list columns are supported by the predicate semantics.allow_list_columns: bool,/// The Arrow schema of the parquet file.@@ -108,11 +120,19 @@ impl<'schema> PushdownChecker<'schema> {has_unpushable_udfs: false,required_columns: Vec::new(),struct_field_accesses: Vec::new(),+ cast_accesses: Vec::new(),+ collect_cast_accesses: false,allow_list_columns,file_schema,}}+ /// Enable collection of whole-column casts to narrower nested types.+ pub(crate) fn with_cast_collection(mut self) -> Self {+ self.collect_cast_accesses = true;+ self+ }+/// Checks whether a struct's root column exists in the file schema and, if so,/// records its index so the entire struct is decoded for filter evaluation.///@@ -217,6 +237,7 @@ impl<'schema> PushdownChecker<'schema> {PushdownColumns {required_columns: self.required_columns,struct_field_accesses: self.struct_field_accesses,+ cast_accesses: self.cast_accesses,}}}@@ -308,6 +329,28 @@ impl TreeNodeVisitor<'_> for PushdownChecker<'_> {}}+ // Handle whole-column casts to a narrower nested type, e.g.+ // `CAST(events AS List<Struct<subset of fields>>)` as inserted by the+ // physical expression adapter when the logical file schema declares a+ // nested column narrower than the physical file. Recording the cast+ // target lets the projection read only the leaves the cast consumes+ // (see `crate::nested_schema_pruning`).+ if self.collect_cast_accesses+ && let Some(cast) = node.downcast_ref::<CastExpr>()+ && let Some(column) = cast.expr().downcast_ref::<Column>()+ && let Ok(idx) = self.file_schema.index_of(column.name())+ && requires_nested_struct_cast(+ self.file_schema.field(idx).data_type(),+ cast.cast_type(),+ )+ {+ self.cast_accesses.push(CastColumnAccess {+ root_index: idx,+ target_type: cast.cast_type().clone(),+ });+ return Ok(TreeNodeRecursion::Jump);+ }+if let Some(column) = node.downcast_ref::<Column>()&& let Some(recursion) = self.check_single_column(column.name()){@@ -337,6 +380,9 @@ pub(crate) struct PushdownColumns {/// Struct field accesses via `get_field`. Each entry records the root struct/// column index and the field path being accessed.pub(crate) struct_field_accesses: Vec<StructFieldAccess>,+ /// Whole-column casts to a narrower nested type. Empty unless cast+ /// collection was enabled on the checker.+ pub(crate) cast_accesses: Vec<CastColumnAccess>,}/// Builds a unified [`ParquetReadPlan`] for a set of projection expressions@@ -369,19 +415,29 @@ pub(crate) fn build_projection_read_plan(return root_level_plan(&root_indices, file_schema, schema_descr);}- // secondary fast path: if the schema has no struct columns, we can skip- // PushdownChecker traversal and use root-level projection- let has_struct_columns = file_schema- .fields()- .iter()- .any(|f| matches!(f.data_type(), DataType::Struct(_)));+ // secondary fast path: if none of the *projected* columns contains a+ // struct at any nesting level, there are no leaves to prune and we can+ // skip the PushdownChecker traversal and use root-level projection.+ //+ // Gating on the projected roots rather than on every field of the file+ // schema keeps this step O(projected columns): a wide file with a nested+ // column the projection never touches should not push the whole+ // projection through the slower, name-resolving path. Any column whose+ // `index` does not line up with the file schema (a stale `Column` from an+ // earlier rewrite) falls through to that path, which resolves by name.+ let projected_columns = exprs.iter().flat_map(collect_columns).collect::<Vec<_>>();+ let all_resolvable_and_struct_free = projected_columns.iter().all(|col| {+ file_schema+ .fields()+ .get(col.index())+ .is_some_and(|f| f.name() == col.name() && !contains_struct(f.data_type()))+ });- if !has_struct_columns {- let mut root_indices = exprs- .into_iter()- .flat_map(|e| collect_columns(&e).into_iter().map(|col| col.index()))+ if all_resolvable_and_struct_free {+ let mut root_indices = projected_columns+ .iter()+ .map(|c| c.index()).collect::<Vec<_>>();-root_indices.sort_unstable();root_indices.dedup();@@ -390,19 +446,37 @@ pub(crate) fn build_projection_read_plan(let mut all_root_indices = Vec::new();let mut all_struct_accesses = Vec::new();+ let mut all_cast_accesses = Vec::new();for expr in exprs {- let mut checker = PushdownChecker::new(file_schema, true);+ let mut checker = PushdownChecker::new(file_schema, true).with_cast_collection();let _ = expr.visit(&mut checker);let columns = checker.into_sorted_columns();all_root_indices.extend_from_slice(&columns.required_columns);all_struct_accesses.extend(columns.struct_field_accesses);+ all_cast_accesses.extend(columns.cast_accesses);}all_root_indices.sort_unstable();all_root_indices.dedup();+ // A whole-column reference reads every leaf of the root, so a cast+ // access on the same root would be overridden anyway: drop those up+ // front. `all_root_indices` is already sorted, so a binary search+ // avoids building a second set just for this filter.+ all_cast_accesses.retain(|c| all_root_indices.binary_search(&c.root_index).is_err());++ if !all_cast_accesses.is_empty() {+ return build_read_plan_with_cast_clipping(+ file_schema,+ schema_descr,+ &all_root_indices,+ &all_struct_accesses,+ &all_cast_accesses,+ );+ }+// when no struct field accesses were found, fall back to root-level projection// to match the performance of the simple pathif all_struct_accesses.is_empty() {@@ -419,6 +493,192 @@ pub(crate) fn build_projection_read_plan(read_plan}+/// Builds a [`ParquetReadPlan`] when at least one projected root column is+/// consumed through a cast to a narrower nested type.+///+/// Per root, in ascending root-index order:+/// - roots referenced as whole columns keep every leaf and their full+/// physical field (whole-column reads take precedence; cast accesses on+/// such roots were already dropped by the caller);+/// - roots consumed through a cast, and not also through a `get_field`+/// access on the same root, keep only the leaves the cast target names+/// (see `crate::nested_schema_pruning`);+/// - roots consumed only through `get_field` accesses keep the union of the+/// leaves those accesses reach, as before;+/// - any other referenced root, a cast that can't be safely clipped (see+/// `nested_schema_pruning::clip_for_cast`), a root reached by two casts+/// with *different* targets (a projection can consume the same column+/// through more than one narrowing cast, e.g.+/// `SELECT CAST(s AS STRUCT(a)), CAST(s AS STRUCT(b)) FROM t`; clipping to+/// either target alone would starve the other), or a root reached by both a+/// cast and a `get_field` access (not produced by+/// `DefaultPhysicalExprAdapter`, which always routes a `get_field` over a+/// narrowed column through the same cast rather than a separate access,+/// but a custom `PhysicalExprAdapter` could in principle inject both),+/// falls back to a full read of that root.+fn build_read_plan_with_cast_clipping(+ file_schema: &Schema,+ schema_descr: &SchemaDescriptor,+ whole_root_indices: &[usize],+ struct_accesses: &[StructFieldAccess],+ cast_accesses: &[CastColumnAccess],+) -> ParquetReadPlan {+ let whole_roots: BTreeSet<usize> = whole_root_indices.iter().copied().collect();+ let struct_access_roots: BTreeSet<usize> =+ struct_accesses.iter().map(|a| a.root_index).collect();+ // Every referenced root's Parquet leaves, grouped in one pass over the+ // schema descriptor rather than one `leaf_indices_for_roots` scan per+ // root (this function may look up several roots).+ let leaves_by_root = leaves_grouped_by_root(schema_descr);++ // Root -> (absolute kept leaf indices, cast-clipped Arrow type) for+ // roots successfully clipped via a cast.+ let mut clipped_by_root: BTreeMap<usize, (Vec<usize>, DataType)> = BTreeMap::new();+ // Roots with a cast access that must fall back to a full read.+ let mut fallback_roots: BTreeSet<usize> = BTreeSet::new();+ // The cast target already clipped for a root, so a second cast on the+ // same root can be recognised as either a repeat (same target: nothing to+ // do) or a conflict (different target: neither clip is valid on its own).+ let mut clipped_target_by_root: BTreeMap<usize, &DataType> = BTreeMap::new();++ for access in cast_accesses {+ let root = access.root_index;+ if whole_roots.contains(&root) || fallback_roots.contains(&root) {+ continue;+ }+ if let Some(previous) = clipped_target_by_root.get(&root) {+ if **previous != access.target_type {+ // The projection consumes this root through two different+ // narrowing casts. Each cast only needs its own leaves, but+ // the mask is per column: clipping to the first target would+ // silently null-fill whatever the second one needs. Read the+ // whole root instead.+ clipped_by_root.remove(&root);+ clipped_target_by_root.remove(&root);+ fallback_roots.insert(root);+ }+ continue;+ }+ if struct_access_roots.contains(&root) {+ fallback_roots.insert(root);+ continue;+ }++ let physical_type = file_schema.field(root).data_type();+ let root_leaves = leaves_by_root.get(&root).map_or(&[][..], Vec::as_slice);++ // Defensive: the arrow type's leaf count must agree with the+ // Parquet schema (it can diverge if the file embeds a different+ // arrow schema). If not, never risk a wrong mask: read the whole+ // root.+ if root_leaves.len() != count_leaves(physical_type) {+ fallback_roots.insert(root);+ continue;+ }++ match clip_for_cast(physical_type, &access.target_type) {+ Some((kept_offsets, pruned_type)) => {+ let start = root_leaves[0];+ let absolute = kept_offsets.into_iter().map(|o| start + o).collect();+ clipped_by_root.insert(root, (absolute, pruned_type));+ clipped_target_by_root.insert(root, &access.target_type);+ }+ // Nothing prunable for this cast: every leaf is consumed.+ None => {+ fallback_roots.insert(root);+ }+ }+ }++ // `get_field` accesses on roots not already read in full (as a whole+ // column, or as a cast that fell back) keep the existing (non-cast) leaf+ // resolution.+ let get_field_accesses: Vec<StructFieldAccess> = struct_accesses+ .iter()+ .filter(|a| {+ // A root carrying a `get_field` access is put into+ // `fallback_roots` before any clip is attempted (see the loop+ // above), so it can never also be clipped. Assert that rather+ // than re-testing it here, so a future reordering trips the+ // assert instead of silently changing which leaves are read.+ debug_assert!(!clipped_by_root.contains_key(&a.root_index));+ !whole_roots.contains(&a.root_index)+ && !fallback_roots.contains(&a.root_index)+ })+ .cloned()+ .collect();++ let mut leaf_indices: Vec<usize> = Vec::new();+ let mut fields: BTreeMap<usize, Arc<Field>> = BTreeMap::new();++ for root in whole_roots.iter().chain(fallback_roots.iter()) {+ // A root with no parquet leaves contributes nothing to the mask;+ // `ProjectionMask::roots` handles that case the same way, so match it+ // rather than indexing and panicking.+ if let Some(leaves) = leaves_by_root.get(root) {+ leaf_indices.extend(leaves.iter().copied());+ }+ fields.insert(*root, Arc::new(file_schema.field(*root).clone()));+ }++ for (&root, (kept, pruned_type)) in &clipped_by_root {+ leaf_indices.extend(kept.iter().copied());+ fields.insert(+ root,+ field_with_type(file_schema.field(root), pruned_type.clone()),+ );+ }++ if !get_field_accesses.is_empty() {+ leaf_indices.extend(resolve_struct_field_leaves(+ &get_field_accesses,+ file_schema,+ schema_descr,+ ));+ let get_field_schema = build_filter_schema(file_schema, &[], &get_field_accesses);+ let get_field_roots: BTreeSet<usize> =+ get_field_accesses.iter().map(|a| a.root_index).collect();+ // `build_filter_schema` emits one field per accessed root in+ // ascending root order, which is the order `get_field_roots` iterates+ // in, so the two line up positionally. Pairing them beats looking each+ // one up by name: no repeated linear scans, and no ambiguity if two+ // roots happen to share a name.+ debug_assert_eq!(get_field_roots.len(), get_field_schema.fields().len());+ for (root, field) in get_field_roots.iter().zip(get_field_schema.fields()) {+ fields.insert(*root, Arc::clone(field));+ }+ }++ leaf_indices.sort_unstable();+ leaf_indices.dedup();++ ParquetReadPlan {+ projection_mask: ProjectionMask::leaves(+ schema_descr,+ leaf_indices.iter().copied(),+ ),+ projected_schema: Arc::new(Schema::new_with_metadata(+ fields.into_values().collect::<Vec<_>>(),+ file_schema.metadata().clone(),+ )),+ }+}++/// Groups every Parquet leaf index by its root (Arrow) column index, in one+/// pass over the schema descriptor.+fn leaves_grouped_by_root(+ schema_descr: &SchemaDescriptor,+) -> BTreeMap<usize, Vec<usize>> {+ let mut by_root: BTreeMap<usize, Vec<usize>> = BTreeMap::new();+ for leaf_idx in 0..schema_descr.num_columns() {+ by_root+ .entry(schema_descr.get_column_root_idx(leaf_idx))+ .or_default()+ .push(leaf_idx);+ }+ by_root+}+/// Builds a leaf-level [`ParquetReadPlan`] covering `root_indices` in full plus/// the individual leaves reached by `struct_field_accesses`.///@@ -677,6 +937,7 @@ mod test {use datafusion_physical_expr::planner::logical2physical;use parquet::arrow::ArrowWriter;use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;+ use parquet::file::metadata::ParquetMetaData;use tempfile::NamedTempFile;#[test]@@ -758,4 +1019,442 @@ mod test {let expected_mask = ProjectionMask::leaves(schema_descr, [0, 1, 2]);assert_eq!(read_plan.projection_mask, expected_mask,);}++ /// Writes the id/struct fixture and returns the schema and metadata a+ /// reader sees for it, so callers don't each repeat the reopen ++ /// `ParquetRecordBatchReaderBuilder` boilerplate.+ ///+ /// Schema: id (Int32), s (Struct{value: Int32, label: Utf8, pad: Utf8}).+ /// Parquet leaves: id=0, s.value=1, s.label=2, s.pad=3.+ fn write_id_struct_file() -> (SchemaRef, Arc<ParquetMetaData>) {+ let struct_fields: Fields = vec![+ Arc::new(Field::new("value", DataType::Int32, false)),+ Arc::new(Field::new("label", DataType::Utf8, false)),+ Arc::new(Field::new("pad", DataType::Utf8, false)),+ ]+ .into();++ let schema = Arc::new(Schema::new(vec![+ Field::new("id", DataType::Int32, false),+ Field::new("s", DataType::Struct(struct_fields.clone()), false),+ ]));++ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(Int32Array::from(vec![1, 2, 3])),+ Arc::new(StructArray::new(+ struct_fields,+ vec![+ Arc::new(Int32Array::from(vec![10, 20, 30])) as _,+ Arc::new(StringArray::from(vec!["a", "b", "c"])) as _,+ Arc::new(StringArray::from(vec!["p0", "p1", "p2"])) as _,+ ],+ None,+ )),+ ],+ )+ .unwrap();++ let file = NamedTempFile::new().expect("temp file");+ let mut writer =+ ArrowWriter::try_new(file.reopen().unwrap(), Arc::clone(&schema), None)+ .expect("writer");+ writer.write(&batch).expect("write batch");+ writer.close().expect("close writer");++ let builder = ParquetRecordBatchReaderBuilder::try_new(file.reopen().unwrap())+ .expect("reader builder");+ (builder.schema().clone(), builder.metadata().clone())+ }++ /// Writes a two-struct-root fixture so tests can combine a cast on one+ /// root with an access on another.+ ///+ /// Schema: a (Struct{p: Int32, q: Utf8}), b (Struct{m: Int32, n: Utf8}).+ /// Parquet leaves: a.p=0, a.q=1, b.m=2, b.n=3.+ fn write_two_struct_file() -> (SchemaRef, Arc<ParquetMetaData>) {+ let group = |first: &str, second: &str| -> Fields {+ vec![+ Arc::new(Field::new(first, DataType::Int32, false)),+ Arc::new(Field::new(second, DataType::Utf8, false)),+ ]+ .into()+ };+ let (a_fields, b_fields) = (group("p", "q"), group("m", "n"));++ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Struct(a_fields.clone()), false),+ Field::new("b", DataType::Struct(b_fields.clone()), false),+ ]));++ let values = |fields: Fields, ints: [i32; 2], strs: [&str; 2]| {+ Arc::new(StructArray::new(+ fields,+ vec![+ Arc::new(Int32Array::from(ints.to_vec())) as _,+ Arc::new(StringArray::from(strs.to_vec())) as _,+ ],+ None,+ )) as _+ };+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ values(a_fields, [1, 2], ["a0", "a1"]),+ values(b_fields, [3, 4], ["b0", "b1"]),+ ],+ )+ .unwrap();++ let file = NamedTempFile::new().expect("temp file");+ let mut writer =+ ArrowWriter::try_new(file.reopen().unwrap(), Arc::clone(&schema), None)+ .expect("writer");+ writer.write(&batch).expect("write batch");+ writer.close().expect("close writer");++ let builder = ParquetRecordBatchReaderBuilder::try_new(file.reopen().unwrap())+ .expect("reader builder");+ (builder.schema().clone(), builder.metadata().clone())+ }++ /// Builds `CAST(Column(name, index) AS Struct{fields})`.+ fn cast_to_struct(+ name: &str,+ index: usize,+ fields: Vec<(&str, DataType)>,+ ) -> Arc<dyn PhysicalExpr> {+ let target = DataType::Struct(+ fields+ .into_iter()+ .map(|(n, dt)| Arc::new(Field::new(n, dt, true)))+ .collect::<Vec<_>>()+ .into(),+ );+ Arc::new(CastExpr::new(+ Arc::new(PhysicalColumn::new(name, index)),+ target,+ None,+ ))+ }++ /// Builds `get_field(Column(name, index), field)`.+ fn get_field_of(+ file_schema: &Schema,+ name: &str,+ field: &str,+ ) -> Arc<dyn PhysicalExpr> {+ logical2physical(+ &get_field().call(vec![+ col(name),+ Expr::Literal(ScalarValue::Utf8(Some(field.to_string())), None),+ ]),+ file_schema,+ )+ }++ /// Clipping a cast whose only surviving field is *not* the struct's first+ /// one: the kept offsets are relative to the root's first leaf and must be+ /// rebased onto it. With `s` starting at leaf 1 and `label` at offset 1,+ /// getting the arithmetic wrong reads `id` (leaf 0) instead of `s.label`.+ #[test]+ fn build_projection_read_plan_clips_cast_to_a_non_leading_field() {+ let (file_schema, metadata) = write_id_struct_file();+ let schema_descr = metadata.file_metadata().schema_descr();++ let exprs = vec![cast_to_struct("s", 1, vec![("label", DataType::Utf8)])];+ let read_plan = build_projection_read_plan(exprs, &file_schema, schema_descr);++ assert_eq!(+ read_plan.projection_mask,+ ProjectionMask::leaves(schema_descr, [2])+ );+ let s_field = read_plan.projected_schema.field_with_name("s").unwrap();+ assert_eq!(+ s_field.data_type(),+ &DataType::Struct(+ vec![Arc::new(Field::new("label", DataType::Utf8, false))].into()+ ),+ );+ }++ /// A cast on one root and a `get_field` on a *different* root: each root+ /// keeps only what it needs, and both appear in the projected schema in+ /// root order.+ #[test]+ fn build_projection_read_plan_clips_cast_beside_get_field_on_another_root() {+ let (file_schema, metadata) = write_two_struct_file();+ let schema_descr = metadata.file_metadata().schema_descr();++ let exprs = vec![+ cast_to_struct("a", 0, vec![("p", DataType::Int32)]),+ get_field_of(&file_schema, "b", "n"),+ ];+ let read_plan = build_projection_read_plan(exprs, &file_schema, schema_descr);++ // a.p (leaf 0) from the clip, b.n (leaf 3) from the field access.+ assert_eq!(+ read_plan.projection_mask,+ ProjectionMask::leaves(schema_descr, [0, 3])+ );+ let field_types = read_plan+ .projected_schema+ .fields()+ .iter()+ .map(|f| (f.name().clone(), f.data_type().clone()))+ .collect::<Vec<_>>();+ assert_eq!(+ field_types,+ vec![+ (+ "a".to_string(),+ DataType::Struct(+ vec![Arc::new(Field::new("p", DataType::Int32, false))].into()+ )+ ),+ (+ "b".to_string(),+ DataType::Struct(+ vec![Arc::new(Field::new("n", DataType::Utf8, false))].into()+ )+ ),+ ]+ );+ }++ /// Once conflicting cast targets have demoted a root to a full read, a+ /// *third* cast on it must not resurrect the clip.+ #[test]+ fn build_projection_read_plan_keeps_full_read_after_a_third_cast() {+ let (file_schema, metadata) = write_id_struct_file();+ let schema_descr = metadata.file_metadata().schema_descr();++ let exprs = vec![+ cast_to_struct("s", 1, vec![("value", DataType::Int32)]),+ cast_to_struct("s", 1, vec![("label", DataType::Utf8)]),+ cast_to_struct("s", 1, vec![("value", DataType::Int32)]),+ ];+ let read_plan = build_projection_read_plan(exprs, &file_schema, schema_descr);++ assert_eq!(+ read_plan.projection_mask,+ ProjectionMask::leaves(schema_descr, [1, 2, 3])+ );+ let s_field = read_plan.projected_schema.field_with_name("s").unwrap();+ assert_eq!(s_field.data_type(), file_schema.field(1).data_type());+ }++ /// A whole-column reference wins over a `get_field` access on the same+ /// root even when another root is being clipped: `a` keeps every leaf and+ /// its full type, `b` keeps only the cast target's.+ #[test]+ fn build_projection_read_plan_whole_column_beats_get_field_beside_a_clip() {+ let (file_schema, metadata) = write_two_struct_file();+ let schema_descr = metadata.file_metadata().schema_descr();++ let exprs: Vec<Arc<dyn PhysicalExpr>> = vec![+ Arc::new(PhysicalColumn::new("a", 0)),+ get_field_of(&file_schema, "a", "p"),+ cast_to_struct("b", 1, vec![("m", DataType::Int32)]),+ ];+ let read_plan = build_projection_read_plan(exprs, &file_schema, schema_descr);++ // Every leaf of `a` (0, 1) plus b.m (leaf 2).+ assert_eq!(+ read_plan.projection_mask,+ ProjectionMask::leaves(schema_descr, [0, 1, 2])+ );+ let a_field = read_plan.projected_schema.field_with_name("a").unwrap();+ assert_eq!(+ a_field.data_type(),+ file_schema.field(0).data_type(),+ "the whole-column reference must keep `a`'s full type"+ );+ }++ /// Columns are resolved by *name*: a `Column` whose index points at a+ /// different field (a stale index left by an earlier rewrite) must not be+ /// taken at face value by the struct fast-path gate.+ #[test]+ fn build_projection_read_plan_resolves_stale_column_indices_by_name() {+ let (file_schema, metadata) = write_id_struct_file();+ let schema_descr = metadata.file_metadata().schema_descr();++ // `s` is at index 1; this claims index 0, which is `id`.+ let exprs = vec![cast_to_struct("s", 0, vec![("value", DataType::Int32)])];+ let read_plan = build_projection_read_plan(exprs, &file_schema, schema_descr);++ assert_eq!(+ read_plan.projection_mask,+ ProjectionMask::leaves(schema_descr, [1]),+ "the cast must resolve to `s`, not to whatever sits at index 0"+ );+ }++ /// A projection consisting solely of a narrowing cast over a struct root+ /// clips the read to the cast target's leaves.+ #[test]+ fn build_projection_read_plan_clips_cast_over_struct() {+ let (file_schema, metadata) = write_id_struct_file();+ let schema_descr = metadata.file_metadata().schema_descr();++ let narrow = DataType::Struct(+ vec![Arc::new(Field::new("value", DataType::Int32, true))].into(),+ );+ let exprs: Vec<Arc<dyn PhysicalExpr>> = vec![+ Arc::new(PhysicalColumn::new("id", 0)),+ Arc::new(CastExpr::new(+ Arc::new(PhysicalColumn::new("s", 1)),+ narrow.clone(),+ None,+ )),+ ];++ let read_plan = build_projection_read_plan(exprs, &file_schema, schema_descr);++ // Only id's leaf (0) and s.value's leaf (1) should be read: s.label+ // and s.pad are clipped away.+ let expected_mask = ProjectionMask::leaves(schema_descr, [0, 1]);+ assert_eq!(read_plan.projection_mask, expected_mask);++ let s_field = read_plan.projected_schema.field_with_name("s").unwrap();+ assert_eq!(+ s_field.data_type(),+ &DataType::Struct(+ vec![Arc::new(Field::new("value", DataType::Int32, false))].into()+ ),+ );+ }++ /// Two casts on the same root with the *same* target still clip: this is+ /// the shape the expression adapter produces when one column is+ /// referenced several times (`SELECT s, s FROM narrowed`).+ #[test]+ fn build_projection_read_plan_clips_repeated_identical_casts() {+ let (file_schema, metadata) = write_id_struct_file();+ let schema_descr = metadata.file_metadata().schema_descr();++ let narrow = DataType::Struct(+ vec![Arc::new(Field::new("value", DataType::Int32, true))].into(),+ );+ let cast = || -> Arc<dyn PhysicalExpr> {+ Arc::new(CastExpr::new(+ Arc::new(PhysicalColumn::new("s", 1)),+ narrow.clone(),+ None,+ ))+ };++ let read_plan =+ build_projection_read_plan(vec![cast(), cast()], &file_schema, schema_descr);++ assert_eq!(+ read_plan.projection_mask,+ ProjectionMask::leaves(schema_descr, [1])+ );+ }++ /// Two casts on the same root with *different* targets cannot both be+ /// served by one mask: clipping to either target alone would null-fill+ /// whatever the other one needs (or fail its runtime struct-compatibility+ /// check outright). Read the whole root instead.+ #[test]+ fn build_projection_read_plan_falls_back_on_conflicting_cast_targets() {+ let (file_schema, metadata) = write_id_struct_file();+ let schema_descr = metadata.file_metadata().schema_descr();++ let narrow = |name: &str, dt: DataType| -> Arc<dyn PhysicalExpr> {+ Arc::new(CastExpr::new(+ Arc::new(PhysicalColumn::new("s", 1)),+ DataType::Struct(vec![Arc::new(Field::new(name, dt, true))].into()),+ None,+ ))+ };+ let exprs = vec![+ narrow("value", DataType::Int32),+ narrow("label", DataType::Utf8),+ ];++ let read_plan = build_projection_read_plan(exprs, &file_schema, schema_descr);++ assert_eq!(+ read_plan.projection_mask,+ ProjectionMask::leaves(schema_descr, [1, 2, 3]),+ "every leaf of `s` must be read so both casts see their fields"+ );+ let s_field = read_plan.projected_schema.field_with_name("s").unwrap();+ assert_eq!(s_field.data_type(), file_schema.field(1).data_type());+ }++ /// The struct fast-path gate looks at the *projected* columns, not at+ /// every field of the file schema: projecting only `id` produces the same+ /// root-level plan it would for a schema with no struct in it at all.+ #[test]+ fn build_projection_read_plan_ignores_unprojected_struct_columns() {+ let (file_schema, metadata) = write_id_struct_file();+ let schema_descr = metadata.file_metadata().schema_descr();++ // Not a bare column, so the all-plain-columns fast path does not apply.+ let exprs: Vec<Arc<dyn PhysicalExpr>> = vec![Arc::new(CastExpr::new(+ Arc::new(PhysicalColumn::new("id", 0)),+ DataType::Int64,+ None,+ ))];++ let read_plan = build_projection_read_plan(exprs, &file_schema, schema_descr);++ assert_eq!(+ read_plan.projection_mask,+ ProjectionMask::roots(schema_descr, [0])+ );+ assert_eq!(read_plan.projected_schema.fields().len(), 1);+ }++ /// A root reached by both a narrowing cast and a `get_field` access (not+ /// producible by `DefaultPhysicalExprAdapter`, but a custom+ /// `PhysicalExprAdapter` could inject both) falls back to a full read of+ /// that root rather than attempting to union the two leaf sets.+ #[test]+ fn build_projection_read_plan_falls_back_when_cast_and_get_field_share_a_root() {+ let (file_schema, metadata) = write_id_struct_file();+ let schema_descr = metadata.file_metadata().schema_descr();++ let narrow = DataType::Struct(+ vec![Arc::new(Field::new("value", DataType::Int32, true))].into(),+ );+ let exprs: Vec<Arc<dyn PhysicalExpr>> = vec![+ Arc::new(CastExpr::new(+ Arc::new(PhysicalColumn::new("s", 1)),+ narrow,+ None,+ )),+ logical2physical(+ &get_field().call(vec![+ col("s"),+ Expr::Literal(ScalarValue::Utf8(Some("label".to_string())), None),+ ]),+ &file_schema,+ ),+ ];++ let read_plan = build_projection_read_plan(exprs, &file_schema, schema_descr);++ // Every leaf of `s` is read (full fallback), not just value/label.+ let expected_mask = ProjectionMask::leaves(schema_descr, [1, 2, 3]);+ assert_eq!(read_plan.projection_mask, expected_mask);++ let s_field = read_plan.projected_schema.field_with_name("s").unwrap();+ assert_eq!(+ s_field.data_type(),+ &DataType::Struct(+ vec![+ Arc::new(Field::new("value", DataType::Int32, false)),+ Arc::new(Field::new("label", DataType::Utf8, false)),+ Arc::new(Field::new("pad", DataType::Utf8, false)),+ ]+ .into()+ ),+ );+ }}diff --git a/datafusion/sqllogictest/test_files/parquet_nested_schema_pruning.slt b/datafusion/sqllogictest/test_files/parquet_nested_schema_pruning.sltnew file mode 100644index 0000000000..d936a89beb--- /dev/null+++ b/datafusion/sqllogictest/test_files/parquet_nested_schema_pruning.slt@@ -0,0 +1,565 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++##########+# Nested projection pruning: a table whose declared nested type is narrower+# than the Parquet file's physical type reads only the declared leaves.+#+# This file covers both halves of that claim: the results are correct, and+# the scan really did read less. Each `explain analyze` below pins a literal+# bytes_scanned against a same-context baseline table declaring the file's+# own physical schema, so no cast is inserted and every leaf is read. A+# change that silently widens a clipped read shows up as a mismatch here.+##########++# The file contains events: ARRAY<STRUCT<x, y, pad_a, pad_b>> and+# s: STRUCT<x, y, pad>; the table below declares narrower nested types.+statement ok+COPY (+ SELECT id, events, s+ FROM (VALUES+ (1, [named_struct('x', 10, 'y', 'a1', 'pad_a', 'p', 'pad_b', 'q')],+ named_struct('x', 100, 'y', 's1', 'pad', 'sp1')),+ (2, [named_struct('x', 20, 'y', 'b1', 'pad_a', 'p', 'pad_b', 'q'),+ named_struct('x', 21, 'y', 'b2', 'pad_a', 'p', 'pad_b', 'q')],+ named_struct('x', 200, 'y', 's2', 'pad', 'sp2')),+ (3, NULL,+ NULL)+ ) AS t(id, events, s)+) TO 'test_files/scratch/parquet_nested_schema_pruning/wide.parquet'+STORED AS PARQUET;++# Declared schema drops pad_a/pad_b from the list elements and pad from the+# struct, declares x as BIGINT (the file has INT), and adds a z column that+# does not exist in the file.+statement ok+CREATE EXTERNAL TABLE narrow (+ id INT,+ events ARRAY<STRUCT<x BIGINT, y VARCHAR, z BIGINT>>,+ s STRUCT<x BIGINT, y VARCHAR>+)+STORED AS PARQUET+LOCATION 'test_files/scratch/parquet_nested_schema_pruning/wide.parquet';++query I??+SELECT id, events, s FROM narrow ORDER BY id;+----+1 [{x: 10, y: a1, z: NULL}] {x: 100, y: s1}+2 [{x: 20, y: b1, z: NULL}, {x: 21, y: b2, z: NULL}] {x: 200, y: s2}+3 NULL NULL++# Struct-level nullability is preserved: row 3's struct is NULL, not a+# struct of NULLs.+query IBB+SELECT id, events IS NULL, s IS NULL FROM narrow ORDER BY id;+----+1 false false+2 false false+3 true true++query II+SELECT id, s['x'] FROM narrow ORDER BY id;+----+1 100+2 200+3 NULL++query II+SELECT id, e['x'] FROM (SELECT id, unnest(events) AS e FROM narrow) ORDER BY id, e['x'];+----+1 10+2 20+2 21++# `full_schema` names every field the file has, so nothing can be clipped away+# and the scan always reads every leaf: a same-context baseline for the+# bytes_scanned comparison below. (A cast is still inserted — the declared+# leaf types differ from the file's, e.g. VARCHAR maps to Utf8View here while+# the file holds Utf8 — but it is not a *narrowing* one, so `clip_for_cast`+# keeps all the leaves.)+statement ok+CREATE EXTERNAL TABLE full_schema (+ id INT,+ events ARRAY<STRUCT<x INT, y VARCHAR, pad_a VARCHAR, pad_b VARCHAR>>,+ s STRUCT<x INT, y VARCHAR, pad VARCHAR>+)+STORED AS PARQUET+LOCATION 'test_files/scratch/parquet_nested_schema_pruning/wide.parquet';++# bytes_scanned is a literal (not <slt:ignore>) checked-in value: narrow+# reads fewer bytes than full_schema because the cast-clipped leaves drop+# pad_a, pad_b, and pad. A future change that widens the narrow read shows+# up here as a bytes_scanned mismatch.+query TT+explain analyze select events from narrow;+----+Plan with Metrics DataSourceExec: <slt:ignore>metrics=[output_rows=3, <slt:ignore>bytes_scanned=172<slt:ignore>]++query TT+explain analyze select events from full_schema;+----+Plan with Metrics DataSourceExec: <slt:ignore>metrics=[output_rows=3, <slt:ignore>bytes_scanned=312<slt:ignore>]++# Same for the top-level struct column: the clipped read drops `pad`.+query TT+explain analyze select s from narrow;+----+Plan with Metrics DataSourceExec: <slt:ignore>metrics=[output_rows=3, <slt:ignore>bytes_scanned=146<slt:ignore>]++query TT+explain analyze select s from full_schema;+----+Plan with Metrics DataSourceExec: <slt:ignore>metrics=[output_rows=3, <slt:ignore>bytes_scanned=219<slt:ignore>]++# `get_field` on a schema-narrowed struct becomes `get_field(CAST(s), 'x')`;+# the read clips to the cast target (every field the *narrow* schema+# declares), not further down to just `x`. The fair "nothing was clipped"+# baseline is therefore reading every physical leaf of `s`+# (`select s from full_schema` above), not the same `get_field` query against+# `full_schema` -- that one needs no cast at all and takes `get_field`'s own,+# more precise, single-leaf pushdown path.+query TT+explain analyze select s['x'] from narrow;+----+Plan with Metrics DataSourceExec: <slt:ignore>metrics=[output_rows=3, <slt:ignore>bytes_scanned=146<slt:ignore>]++# Mixed access -- the whole (narrowed) column and a subfield of it -- still+# reads only the narrow schema's leaves.+query TT+explain analyze select s, s['y'] from narrow;+----+Plan with Metrics DataSourceExec: <slt:ignore>metrics=[output_rows=3, <slt:ignore>bytes_scanned=146<slt:ignore>]++query TT+explain analyze select s, s['y'] from full_schema;+----+Plan with Metrics DataSourceExec: <slt:ignore>metrics=[output_rows=3, <slt:ignore>bytes_scanned=219<slt:ignore>]+++# `SELECT *` goes through the same clipped read as an explicit projection.+query I??+SELECT * FROM narrow ORDER BY id;+----+1 [{x: 10, y: a1, z: NULL}] {x: 100, y: s1}+2 [{x: 20, y: b1, z: NULL}, {x: 21, y: b2, z: NULL}] {x: 200, y: s2}+3 NULL NULL++# Referencing the narrowed column as a whole *and* through a field access in+# the same query.+query I?T+SELECT id, s, s['y'] FROM narrow ORDER BY id;+----+1 {x: 100, y: s1} s1+2 {x: 200, y: s2} s2+3 NULL NULL++# Aggregating over a clipped nested column.+query IIT+SELECT count(*), sum(s['x']), string_agg(s['y'], ',' ORDER BY id) FROM narrow;+----+3 300 s1,s2++# Filtering on a field of a clipped nested column.+query I?+SELECT id, s FROM narrow WHERE s['x'] = 200 ORDER BY id;+----+2 {x: 200, y: s2}++# A declared schema whose fields are in a different order from the file's:+# the values follow the declared order, not the physical one.+statement ok+CREATE EXTERNAL TABLE reordered (+ id INT,+ events ARRAY<STRUCT<y VARCHAR, x BIGINT>>,+ s STRUCT<y VARCHAR, x BIGINT>+)+STORED AS PARQUET+LOCATION 'test_files/scratch/parquet_nested_schema_pruning/wide.parquet';++query I??+SELECT id, events, s FROM reordered ORDER BY id;+----+1 [{y: a1, x: 10}] {y: s1, x: 100}+2 [{y: b1, x: 20}, {y: b2, x: 21}] {y: s2, x: 200}+3 NULL NULL++statement ok+DROP TABLE reordered;++# A declared struct sharing no field name with the file's is rejected rather+# than silently null-filled: `clip_for_cast` never sees a zero-overlap cast.+statement ok+CREATE EXTERNAL TABLE no_overlap (+ id INT,+ s STRUCT<not_in_file BIGINT>+)+STORED AS PARQUET+LOCATION 'test_files/scratch/parquet_nested_schema_pruning/wide.parquet';++statement error DataFusion error: Execution error: Cannot cast column 's'+SELECT s FROM no_overlap;++statement ok+DROP TABLE no_overlap;++##########+# Struct nested inside a struct: both levels are clipped, and the reader's+# reconstruction of struct validity survives at both levels.+##########++statement ok+COPY (+ SELECT id, n+ FROM (VALUES+ (1, named_struct('inner', named_struct('a', 1, 'pad_i', 'pi1'), 'c', 'c1', 'pad_o', 'po1')),+ (2, named_struct('inner', named_struct('a', 2, 'pad_i', 'pi2'), 'c', 'c2', 'pad_o', 'po2')),+ (3, NULL)+ ) AS t(id, n)+) TO 'test_files/scratch/parquet_nested_schema_pruning/nested_struct.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE nested_narrow (+ id INT,+ n STRUCT<inner STRUCT<a BIGINT>, c VARCHAR>+)+STORED AS PARQUET+LOCATION 'test_files/scratch/parquet_nested_schema_pruning/nested_struct.parquet';++query I?+SELECT id, n FROM nested_narrow ORDER BY id;+----+1 {inner: {a: 1}, c: c1}+2 {inner: {a: 2}, c: c2}+3 NULL++query IBB+SELECT id, n IS NULL, n['inner'] IS NULL FROM nested_narrow ORDER BY id;+----+1 false false+2 false false+3 true true++statement ok+DROP TABLE nested_narrow;++##########+# The exact shape reported in datafusion-comet#4859: a two-level+# `ARRAY<STRUCT<..., items: ARRAY<STRUCT<...>>>>` column with a dropped+# struct sibling (`latency_parts`), a dropped map sibling (`feature_map`), a+# dropped nested-struct sibling (`diagnostics`), and dropped top-level+# sibling columns (`dimension_id`, `region_code`, `raw_payload`).+# Structurally the same ReadSchema/InputSchema pair as the issue (field names+# representative, not verbatim), which let Comet's production query read+# 1.35 TB where plain Spark, given the same pruned ReadSchema, read 30.9 GB.+#+# Every dropped sibling carries real data rather than NULLs, so the+# bytes_scanned gap below is attributable to the clip and not to NULL columns+# being cheap.+##########++statement ok+COPY (+ SELECT id, is_flagged, dimension_id, region_code, events, raw_payload+ FROM (VALUES+ (1, true, 1001, 'us-east', [named_struct(+ 'is_available', true,+ 'event_time_ms', 10,+ 'event_token', 'token-0',+ 'latency_parts', named_struct('queue_time_ms', 5, 'retry_count', 1),+ 'items', [named_struct('group_id', 1, 'entity_id', 101, 'metric_value', 1.5,+ 'feature_map', MAP {'f1': 0.25},+ 'diagnostics', named_struct('module_id', 'm1', 'trace_id', 't1'),+ 'pad', 'pad-0000'),+ named_struct('group_id', 2, 'entity_id', 102, 'metric_value', 3.0,+ 'feature_map', MAP {'f2': 0.5},+ 'diagnostics', named_struct('module_id', 'm2', 'trace_id', 't2'),+ 'pad', 'pad-0001')])],+ 'payload-0'),+ (2, false, 1002, 'us-west', [named_struct(+ 'is_available', false,+ 'event_time_ms', 20,+ 'event_token', 'token-1',+ 'latency_parts', named_struct('queue_time_ms', 7, 'retry_count', 2),+ 'items', [named_struct('group_id', 3, 'entity_id', 103, 'metric_value', 4.5,+ 'feature_map', MAP {'f3': 0.75},+ 'diagnostics', named_struct('module_id', 'm3', 'trace_id', 't3'),+ 'pad', 'pad-0002')])],+ 'payload-1')+ ) AS t(id, is_flagged, dimension_id, region_code, events, raw_payload)+) TO 'test_files/scratch/parquet_nested_schema_pruning/two_level.parquet'+STORED AS PARQUET;++# Declares neither the dropped top-level columns nor, inside `events`,+# `event_token`/`latency_parts`, nor, inside `items`, the map, the nested+# struct, or the pad.+statement ok+CREATE EXTERNAL TABLE two_level_narrow (+ id INT,+ is_flagged BOOLEAN,+ events ARRAY<STRUCT<+ is_available BOOLEAN,+ event_time_ms BIGINT,+ items ARRAY<STRUCT<group_id BIGINT, entity_id BIGINT, metric_value DOUBLE>>+ >>+)+STORED AS PARQUET+LOCATION 'test_files/scratch/parquet_nested_schema_pruning/two_level.parquet';++# The file's own physical schema, so no cast is inserted: the same-context+# baseline for the bytes comparison.+statement ok+CREATE EXTERNAL TABLE two_level_full+STORED AS PARQUET+LOCATION 'test_files/scratch/parquet_nested_schema_pruning/two_level.parquet';++# Only the declared subfields survive, at *both* nesting levels: the printed+# structs are the emitted Arrow type.+query I?+SELECT id, events FROM two_level_narrow ORDER BY id;+----+1 [{is_available: true, event_time_ms: 10, items: [{group_id: 1, entity_id: 101, metric_value: 1.5}, {group_id: 2, entity_id: 102, metric_value: 3.0}]}]+2 [{is_available: false, event_time_ms: 20, items: [{group_id: 3, entity_id: 103, metric_value: 4.5}]}]++# Unnesting twice reaches the inner list's surviving leaves.+query III+SELECT id, i['group_id'], i['entity_id']+FROM (SELECT id, unnest(e['items']) AS i+ FROM (SELECT id, unnest(events) AS e FROM two_level_narrow))+ORDER BY id, i['group_id'];+----+1 1 101+1 2 102+2 3 103++query TT+explain analyze select events from two_level_narrow;+----+Plan with Metrics DataSourceExec: <slt:ignore>metrics=[output_rows=2, <slt:ignore>bytes_scanned=381<slt:ignore>]++query TT+explain analyze select events from two_level_full;+----+Plan with Metrics DataSourceExec: <slt:ignore>metrics=[output_rows=2, <slt:ignore>bytes_scanned=1.05 K<slt:ignore>]++statement ok+DROP TABLE two_level_narrow;++statement ok+DROP TABLE two_level_full;++##########+# A MAP column is never clipped (the runtime cast routes maps through Arrow's+# positional struct cast, which needs every child), but it must not stop a+# struct sibling from being clipped. The declared schema below omits the map+# entirely, leaving it in the file as an unprojected root.+##########++statement ok+COPY (+ SELECT id, m, s+ FROM (VALUES+ (1, MAP {'k1': 1, 'k2': 2}, named_struct('x', 10, 'pad', 'p1')),+ (2, MAP {'k1': 3}, named_struct('x', 20, 'pad', 'p2'))+ ) AS t(id, m, s)+) TO 'test_files/scratch/parquet_nested_schema_pruning/with_map.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE map_sibling (+ id INT,+ s STRUCT<x BIGINT>+)+STORED AS PARQUET+LOCATION 'test_files/scratch/parquet_nested_schema_pruning/with_map.parquet';++query I?+SELECT id, s FROM map_sibling ORDER BY id;+----+1 {x: 10}+2 {x: 20}++statement ok+DROP TABLE map_sibling;++##########+# One table over two files, one physically narrow (no cast inserted) and one+# wide (clipped). Both must read correctly in the same scan.+##########++statement ok+COPY (+ SELECT id, s+ FROM (VALUES+ (10, named_struct('x', 1000, 'y', 'w1', 'pad', 'wp1'))+ ) AS t(id, s)+) TO 'test_files/scratch/parquet_nested_schema_pruning/mixed/wide.parquet'+STORED AS PARQUET;++statement ok+COPY (+ SELECT id, s+ FROM (VALUES+ (20, named_struct('x', 2000, 'y', 'n1'))+ ) AS t(id, s)+) TO 'test_files/scratch/parquet_nested_schema_pruning/mixed/narrow.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE mixed_files (+ id INT,+ s STRUCT<x BIGINT, y VARCHAR>+)+STORED AS PARQUET+LOCATION 'test_files/scratch/parquet_nested_schema_pruning/mixed/';++query I?+SELECT id, s FROM mixed_files ORDER BY id;+----+10 {x: 1000, y: w1}+20 {x: 2000, y: n1}++statement ok+DROP TABLE mixed_files;++##########+# A predicate on a primitive column with filter pushdown enabled, while the+# projected nested column is clipped: the clip and the row filter have to+# coexist on the same scan.+#+# The predicate is deliberately on `id` and not on a field of the clipped+# column: `WHERE s['x'] = ...` with pushdown enabled is silently dropped+# today (apache/datafusion#24109), which is a pre-existing row-filter bug+# rather than anything this feature does.+##########++statement ok+set datafusion.execution.parquet.pushdown_filters = true;++query I?+SELECT id, s FROM narrow WHERE id >= 2 ORDER BY id;+----+2 {x: 200, y: s2}+3 NULL++query TT+explain analyze select s from narrow where id >= 2;+----+Plan with Metrics DataSourceExec: <slt:ignore>metrics=[output_rows=2, <slt:ignore>bytes_scanned=219<slt:ignore>]++query TT+explain analyze select s from full_schema where id >= 2;+----+Plan with Metrics DataSourceExec: <slt:ignore>metrics=[output_rows=2, <slt:ignore>bytes_scanned=292<slt:ignore>]++statement ok+set datafusion.execution.parquet.pushdown_filters = false;++##########+# Query-level casts. `ProjectionExec` is merged into the scan, so a `CAST`+# written in the query reaches the same read-plan analysis as an+# adapter-inserted one — including one column consumed through two *different*+# cast targets, which no single clipped read can serve. Clipping to one+# target's leaves would leave the other cast reading a struct that is missing+# the fields it names, which `cast_column` either null-fills (wrong results)+# or, for disjoint targets, rejects outright.+#+# `exact` infers its schema from the file, so no adapter cast is interposed+# and the casts below are the only ones the scan sees.+##########++statement ok+CREATE EXTERNAL TABLE exact+STORED AS PARQUET+LOCATION 'test_files/scratch/parquet_nested_schema_pruning/wide.parquet';++# A single query-level cast is clipped like an adapter-inserted one.+query ?+SELECT CAST(s AS STRUCT<y VARCHAR>) FROM exact ORDER BY id;+----+{y: s1}+{y: s2}+NULL++# Disjoint targets.+query ??+SELECT CAST(s AS STRUCT<x BIGINT>) AS q0,+ CAST(s AS STRUCT<pad VARCHAR>) AS q1+FROM exact ORDER BY id;+----+{x: 100} {pad: sp1}+{x: 200} {pad: sp2}+NULL NULL++# Overlapping targets: q1 needs a leaf q0's clip would have dropped.+query ??+SELECT CAST(s AS STRUCT<x BIGINT>) AS q0,+ CAST(s AS STRUCT<x BIGINT, y VARCHAR>) AS q1+FROM exact ORDER BY id;+----+{x: 100} {x: 100, y: s1}+{x: 200} {x: 200, y: s2}+NULL NULL++# Repeated identical targets still clip.+query ??+SELECT CAST(s AS STRUCT<x BIGINT>) AS q0,+ CAST(s AS STRUCT<x BIGINT>) AS q1+FROM exact ORDER BY id;+----+{x: 100} {x: 100}+{x: 200} {x: 200}+NULL NULL++# A cast alongside a whole-column reference: the whole-column read wins.+query ??+SELECT CAST(s AS STRUCT<x BIGINT>) AS q0, s+FROM exact ORDER BY id;+----+{x: 100} {x: 100, y: s1, pad: sp1}+{x: 200} {x: 200, y: s2, pad: sp2}+NULL NULL++# The conflicting-target fallback reads the whole column -- exactly what a+# scan with no clipping at all reads, and never more. These two must match:+# the first falls back, the second never clips in the first place.+query TT+explain analyze select CAST(s AS STRUCT<x BIGINT>) AS q0, CAST(s AS STRUCT<pad VARCHAR>) AS q1 from exact;+----+Plan with Metrics DataSourceExec: <slt:ignore>metrics=[output_rows=3, <slt:ignore>bytes_scanned=219<slt:ignore>]++query TT+explain analyze select s from exact;+----+Plan with Metrics DataSourceExec: <slt:ignore>metrics=[output_rows=3, <slt:ignore>bytes_scanned=219<slt:ignore>]++statement ok+DROP TABLE exact;++# A query cast stacked on top of the adapter's cast for a narrowed table.+query ?+SELECT CAST(s AS STRUCT<y VARCHAR>) FROM narrow ORDER BY id;+----+{y: s1}+{y: s2}+NULL++statement ok+DROP TABLE narrow;++statement ok+DROP TABLE full_schema;