Apache DataFusion SQL Query Engine
refactor: de-duplicate parquet read plan construction (#23426)
Adrian Garcia Badaraccob9968a06eadc
diff --git a/datafusion/datasource-parquet/src/projection_read_plan.rs b/datafusion/datasource-parquet/src/projection_read_plan.rsindex 46caed6616..c9d8beab14 100644--- a/datafusion/datasource-parquet/src/projection_read_plan.rs+++ b/datafusion/datasource-parquet/src/projection_read_plan.rs@@ -363,18 +363,7 @@ pub(crate) fn build_projection_read_plan(root_indices.sort_unstable();root_indices.dedup();- let projection_mask =- ProjectionMask::roots(schema_descr, root_indices.iter().copied());- let projected_schema = Arc::new(- file_schema- .project(&root_indices)- .expect("valid column indices"),- );-- return ParquetReadPlan {- projection_mask,- projected_schema,- };+ return root_level_plan(&root_indices, file_schema, schema_descr);}// secondary fast path: if the schema has no struct columns, we can skip@@ -393,19 +382,7 @@ pub(crate) fn build_projection_read_plan(root_indices.sort_unstable();root_indices.dedup();- let projection_mask =- ProjectionMask::roots(schema_descr, root_indices.iter().copied());-- let projected_schema = Arc::new(- file_schema- .project(&root_indices)- .expect("valid column indices"),- );-- return ParquetReadPlan {- projection_mask,- projected_schema,- };+ return root_level_plan(&root_indices, file_schema, schema_descr);}let mut all_root_indices = Vec::new();@@ -426,38 +403,74 @@ pub(crate) fn build_projection_read_plan(// when no struct field accesses were found, fall back to root-level projection// to match the performance of the simple pathif all_struct_accesses.is_empty() {- let projection_mask =- ProjectionMask::roots(schema_descr, all_root_indices.iter().copied());- let projected_schema = Arc::new(- file_schema- .project(&all_root_indices)- .expect("valid column indices"),- );-- return ParquetReadPlan {- projection_mask,- projected_schema,- };+ return root_level_plan(&all_root_indices, file_schema, schema_descr);}- let leaf_indices = {- let mut out =- leaf_indices_for_roots(all_root_indices.iter().copied(), schema_descr);- let struct_leaf_indices =- resolve_struct_field_leaves(&all_struct_accesses, file_schema, schema_descr);+ let (read_plan, _leaf_indices) = assemble_read_plan(+ &all_root_indices,+ &all_struct_accesses,+ file_schema,+ schema_descr,+ );- out.extend_from_slice(&struct_leaf_indices);- out.sort_unstable();- out.dedup();+ read_plan+}- out- };+/// Builds a leaf-level [`ParquetReadPlan`] covering `root_indices` in full plus+/// the individual leaves reached by `struct_field_accesses`.+///+/// `root_indices` must be sorted, deduplicated indices into `file_schema`.+///+/// Also returns the resolved Parquet leaf indices, sorted and deduplicated, so+/// callers can size the columns the decoder will read.+pub(crate) fn assemble_read_plan(+ root_indices: &[usize],+ struct_field_accesses: &[StructFieldAccess],+ file_schema: &Schema,+ schema_descr: &SchemaDescriptor,+) -> (ParquetReadPlan, Vec<usize>) {+ let mut leaf_indices =+ leaf_indices_for_roots(root_indices.iter().copied(), schema_descr);+ leaf_indices.extend_from_slice(&resolve_struct_field_leaves(+ struct_field_accesses,+ file_schema,+ schema_descr,+ ));+ leaf_indices.sort_unstable();+ leaf_indices.dedup();let projection_mask =ProjectionMask::leaves(schema_descr, leaf_indices.iter().copied());-let projected_schema =- build_filter_schema(file_schema, &all_root_indices, &all_struct_accesses);+ build_filter_schema(file_schema, root_indices, struct_field_accesses);++ (+ ParquetReadPlan {+ projection_mask,+ projected_schema,+ },+ leaf_indices,+ )+}++/// Builds a [`ParquetReadPlan`] that decodes whole root columns.+///+/// `root_indices` must be sorted, deduplicated indices into `file_schema`. Every+/// leaf below each root is decoded, and the projected schema keeps each root+/// field's full type. Callers that need to decode only some leaves of a struct+/// root must build the plan from leaf indices instead.+fn root_level_plan(+ root_indices: &[usize],+ file_schema: &Schema,+ schema_descr: &SchemaDescriptor,+) -> ParquetReadPlan {+ let projection_mask =+ ProjectionMask::roots(schema_descr, root_indices.iter().copied());+ let projected_schema = Arc::new(+ file_schema+ .project(root_indices)+ .expect("valid column indices"),+ );ParquetReadPlan {projection_mask,@@ -465,7 +478,7 @@ pub(crate) fn build_projection_read_plan(}}-pub(crate) fn leaf_indices_for_roots<I>(+fn leaf_indices_for_roots<I>(root_indices: I,schema_descr: &SchemaDescriptor,) -> Vec<usize>@@ -491,7 +504,7 @@ where/// For every `StructFieldAccess`, finds the leaf columns in the Parquet schema/// whose path matches the struct root name + field path. This avoids reading all/// leaves of a struct when only specific fields are needed-pub(crate) fn resolve_struct_field_leaves(+fn resolve_struct_field_leaves(accesses: &[StructFieldAccess],file_schema: &Schema,schema_descr: &SchemaDescriptor,@@ -530,7 +543,7 @@ pub(crate) fn resolve_struct_field_leaves(/// For struct columns accessed via `get_field`, a pruned struct type is created/// containing only the fields along the access path. Note: it must match the schema/// that the Parquet reader produces when projecting specific struct leaves-pub(crate) fn build_filter_schema(+fn build_filter_schema(file_schema: &Schema,regular_indices: &[usize],struct_field_accesses: &[StructFieldAccess],diff --git a/datafusion/datasource-parquet/src/row_filter.rs b/datafusion/datasource-parquet/src/row_filter.rsindex 4505f7fd62..a375e6611e 100644--- a/datafusion/datasource-parquet/src/row_filter.rs+++ b/datafusion/datasource-parquet/src/row_filter.rs@@ -86,8 +86,7 @@ use datafusion_physical_plan::metrics;use super::ParquetFileMetrics;use super::supported_predicates::supports_list_predicates;use crate::projection_read_plan::{- ParquetReadPlan, PushdownChecker, PushdownColumns, build_filter_schema,- leaf_indices_for_roots, resolve_struct_field_leaves,+ ParquetReadPlan, PushdownChecker, PushdownColumns, assemble_read_plan,};/// A "compiled" predicate passed to `ParquetRecordBatchStream` to perform@@ -269,38 +268,16 @@ pub(crate) fn build_parquet_read_plan(return Ok(None);};- let root_indices = &required_columns.required_columns;-- let mut leaf_indices =- leaf_indices_for_roots(root_indices.iter().copied(), schema_descr);-- let struct_leaf_indices = resolve_struct_field_leaves(+ let (read_plan, leaf_indices) = assemble_read_plan(+ &required_columns.required_columns,&required_columns.struct_field_accesses,file_schema,schema_descr,);- leaf_indices.extend_from_slice(&struct_leaf_indices);- leaf_indices.sort_unstable();- leaf_indices.dedup();let required_bytes = size_of_columns(&leaf_indices, metadata)?;- let projection_mask =- ProjectionMask::leaves(schema_descr, leaf_indices.iter().copied());-- let projected_schema = build_filter_schema(- file_schema,- root_indices,- &required_columns.struct_field_accesses,- );-- Ok(Some((- ParquetReadPlan {- projection_mask,- projected_schema,- },- required_bytes,- )))+ Ok(Some((read_plan, required_bytes)))}/// Checks if a predicate expression can be pushed down to the parquet decoder.