Apache DataFusion SQL Query Engine
perf: resolve parquet leaf columns in O(1) during statistics collection (#26355)
Peter L2ff1f76f3d8c
diff --git a/datafusion/datasource-parquet/benches/parquet_metadata_statistics.rs b/datafusion/datasource-parquet/benches/parquet_metadata_statistics.rsindex 82147abfd9..b4d1a6644a 100644--- a/datafusion/datasource-parquet/benches/parquet_metadata_statistics.rs+++ b/datafusion/datasource-parquet/benches/parquet_metadata_statistics.rs@@ -72,7 +72,9 @@ fn parquet_metadata_statistics(c: &mut Criterion) {MetadataState::Mixed,MetadataState::None,];- let column_counts = [8, 64, 256];+ // 1024 covers wide schemas, where the cost of resolving each field to a+ // parquet leaf column dominates.+ let column_counts = [8, 64, 256, 1024];let row_group_counts = [1, 32, 128];let mut group = c.benchmark_group("parquet_metadata_statistics");diff --git a/datafusion/datasource-parquet/src/leaf_resolver.rs b/datafusion/datasource-parquet/src/leaf_resolver.rsnew file mode 100644index 0000000000..192cb6c679--- /dev/null+++ b/datafusion/datasource-parquet/src/leaf_resolver.rs@@ -0,0 +1,293 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! O(1) resolution of an Arrow field name to its Parquet leaf column index.++use arrow::datatypes::Schema;+use datafusion_common::HashMap;+use parquet::arrow::arrow_reader::statistics::StatisticsConverter;+use parquet::schema::types::SchemaDescriptor;++/// Resolves Arrow field names to Parquet leaf column indices in O(1), after+/// an O(N) build.+///+/// [`parquet::arrow::parquet_column`], and [`StatisticsConverter::try_new`]+/// which calls it, is O(N) per lookup, so resolving every field of a file+/// with it is O(N²). This resolver builds the lookup maps once per file+/// instead.+///+/// Results match `parquet_column` exactly, including its tie-breaks (see+/// [`Self::leaf_index`]). A mismatch would attribute one column's statistics+/// to another, so the tests compare the two directly.+pub(crate) struct LeafResolver<'a> {+ parquet_schema: &'a SchemaDescriptor,+ schema: &'a Schema,+ /// Arrow field name to root field index. The first field with a name+ /// wins, as in `Fields::find`.+ name_to_root: HashMap<&'a str, usize>,+ /// Parquet root index to its lowest leaf index, or `None` for a root+ /// with no leaves (an empty group).+ root_to_first_leaf: Vec<Option<usize>>,+}++impl<'a> LeafResolver<'a> {+ pub(crate) fn new(parquet_schema: &'a SchemaDescriptor, schema: &'a Schema) -> Self {+ let fields = schema.fields();+ let mut name_to_root = HashMap::with_capacity(fields.len());+ for (idx, field) in fields.iter().enumerate() {+ name_to_root.entry(field.name().as_str()).or_insert(idx);+ }++ let num_roots = parquet_schema.root_schema().get_fields().len();+ let mut root_to_first_leaf = vec![None; num_roots];+ // Iterate in reverse so the lowest leaf index for each root is kept.+ for leaf in (0..parquet_schema.columns().len()).rev() {+ let root = parquet_schema.get_column_root_idx(leaf);+ if root < num_roots {+ root_to_first_leaf[root] = Some(leaf);+ }+ }++ Self {+ parquet_schema,+ schema,+ name_to_root,+ root_to_first_leaf,+ }+ }++ /// The Parquet leaf column index for `name`.+ ///+ /// Returns `None` in the same cases as [`parquet::arrow::parquet_column`]:+ /// * `name` is not in the Arrow schema,+ /// * the matched field is nested,+ /// * the matched root has no Parquet leaf columns.+ pub(crate) fn leaf_index(&self, name: &str) -> Option<usize> {+ self.resolve(name).map(|(_, leaf)| leaf)+ }++ /// Build a [`StatisticsConverter`] for `name`.+ ///+ /// When `name` has no leaf this falls back to+ /// [`StatisticsConverter::try_new`], which is O(N). The fallback gives the+ /// same error as `try_new` for an unknown column, and is needed because+ /// [`StatisticsConverter::from_column_index`] cannot express a column with+ /// no Parquet leaf. Flat schemas never take it.+ pub(crate) fn converter(+ &self,+ name: &str,+ ) -> parquet::errors::Result<StatisticsConverter<'a>> {+ match self.resolve(name) {+ Some((root, leaf)) => StatisticsConverter::from_column_index(+ leaf,+ self.schema.field(root),+ self.parquet_schema,+ ),+ None => StatisticsConverter::try_new(name, self.schema, self.parquet_schema),+ }+ }++ /// The Arrow root index and Parquet leaf index for `name`.+ fn resolve(&self, name: &str) -> Option<(usize, usize)> {+ let root = *self.name_to_root.get(name)?;+ if self.schema.field(root).data_type().is_nested() {+ return None;+ }+ let leaf = self.root_to_first_leaf.get(root).copied().flatten()?;+ Some((root, leaf))+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use arrow::datatypes::{DataType, Field, Fields};+ use parquet::arrow::{ArrowSchemaConverter, parquet_column};+ use parquet::schema::parser::parse_message_type;+ use std::sync::Arc;++ /// Primitives, a nested struct, a nested list, and a duplicate field name+ /// with a different type from the first.+ fn test_schema() -> Schema {+ Schema::new(vec![+ Field::new("a", DataType::Int64, true),+ Field::new("b", DataType::Utf8, true),+ Field::new(+ "s",+ DataType::Struct(Fields::from(vec![+ Field::new("x", DataType::Int32, true),+ Field::new("y", DataType::Int32, true),+ ])),+ true,+ ),+ Field::new(+ "l",+ DataType::List(Arc::new(Field::new("item", DataType::Int32, true))),+ true,+ ),+ Field::new("c", DataType::Float64, true),+ // Unreachable by name: "a" resolves to index 0.+ Field::new("a", DataType::Utf8, true),+ ])+ }++ fn descriptor(message: &str) -> SchemaDescriptor {+ SchemaDescriptor::new(Arc::new(parse_message_type(message).unwrap()))+ }++ /// Asserts `leaf_index` and `converter` agree with `parquet_column` and+ /// `try_new` for every field name in `schema`, plus some absent names.+ fn assert_matches_upstream(descr: &SchemaDescriptor, schema: &Schema) {+ let resolver = LeafResolver::new(descr, schema);++ let mut names: Vec<&str> =+ schema.fields().iter().map(|f| f.name().as_str()).collect();+ names.extend(["missing", "", "A"]);++ for name in names {+ let expected = parquet_column(descr, schema, name).map(|(idx, _)| idx);+ assert_eq!(+ resolver.leaf_index(name),+ expected,+ "leaf_index disagreed with parquet_column for {name:?}"+ );++ match (+ resolver.converter(name),+ StatisticsConverter::try_new(name, schema, descr),+ ) {+ (Ok(actual), Ok(expected)) => {+ assert_eq!(+ actual.parquet_column_index(),+ expected.parquet_column_index(),+ "converter targeted a different column for {name:?}"+ );+ assert_eq!(+ actual.arrow_field(),+ expected.arrow_field(),+ "converter used a different Arrow field for {name:?}"+ );+ }+ (Err(actual), Err(expected)) => {+ assert_eq!(actual.to_string(), expected.to_string());+ }+ (actual, expected) => panic!(+ "converter for {name:?} returned {:?}, try_new returned {:?}",+ actual.map(|c| c.parquet_column_index()),+ expected.map(|c| c.parquet_column_index()),+ ),+ }+ }+ }++ #[test]+ fn matches_upstream() {+ let schema = test_schema();+ let descr = ArrowSchemaConverter::new().convert(&schema).unwrap();+ assert_matches_upstream(&descr, &schema);+ }++ #[test]+ fn resolves_expected_leaves() {+ let schema = test_schema();+ let descr = ArrowSchemaConverter::new().convert(&schema).unwrap();+ let resolver = LeafResolver::new(&descr, &schema);++ assert_eq!(resolver.leaf_index("a"), Some(0));+ assert_eq!(resolver.leaf_index("b"), Some(1));+ // "s" has two leaves and "l" one, so "c" is leaf 5.+ assert_eq!(resolver.leaf_index("c"), Some(5));+ assert_eq!(resolver.leaf_index("s"), None);+ assert_eq!(resolver.leaf_index("l"), None);+ assert_eq!(resolver.leaf_index("missing"), None);+ }++ /// The converter for a duplicated name must use the first field's type.+ #[test]+ fn duplicate_name_uses_first_field() {+ let schema = test_schema();+ let descr = ArrowSchemaConverter::new().convert(&schema).unwrap();+ let resolver = LeafResolver::new(&descr, &schema);++ let converter = resolver.converter("a").unwrap();+ assert_eq!(converter.arrow_field().data_type(), &DataType::Int64);+ }++ /// A root with no leaves must resolve to no leaf, even when the Arrow+ /// field at that index is a primitive.+ #[test]+ fn empty_group_root() {+ let descr = descriptor(+ "message schema {+ required int64 a;+ optional group g {}+ required int64 c;+ }",+ );+ let schema = Schema::new(vec![+ Field::new("a", DataType::Int64, false),+ Field::new("g", DataType::Int64, true),+ Field::new("c", DataType::Int64, false),+ ]);+ assert_eq!(LeafResolver::new(&descr, &schema).leaf_index("g"), None);+ assert_matches_upstream(&descr, &schema);+ }++ /// A primitive Arrow field over a multi-leaf Parquet root must resolve to+ /// the root's lowest leaf.+ #[test]+ fn multi_leaf_root_uses_lowest_leaf() {+ let descr = descriptor(+ "message schema {+ required int64 a;+ optional group g {+ required int64 x;+ required int64 y;+ }+ }",+ );+ let schema = Schema::new(vec![+ Field::new("a", DataType::Int64, false),+ Field::new("g", DataType::Int64, true),+ ]);+ assert_eq!(LeafResolver::new(&descr, &schema).leaf_index("g"), Some(1));+ assert_matches_upstream(&descr, &schema);+ }++ /// Arrow fields beyond the last Parquet root must resolve to no leaf.+ #[test]+ fn more_arrow_fields_than_parquet_roots() {+ let descr = descriptor("message schema { required int64 a; }");+ let schema = Schema::new(vec![+ Field::new("a", DataType::Int64, false),+ Field::new("b", DataType::Int64, true),+ ]);+ assert_eq!(LeafResolver::new(&descr, &schema).leaf_index("b"), None);+ assert_matches_upstream(&descr, &schema);+ }++ #[test]+ fn empty_schema() {+ let schema = Schema::empty();+ let descr = ArrowSchemaConverter::new().convert(&schema).unwrap();+ assert_eq!(+ LeafResolver::new(&descr, &schema).leaf_index("anything"),+ None+ );+ assert_matches_upstream(&descr, &schema);+ }+}diff --git a/datafusion/datasource-parquet/src/metadata.rs b/datafusion/datasource-parquet/src/metadata.rsindex f1299c0a97..0cfcb980a7 100644--- a/datafusion/datasource-parquet/src/metadata.rs+++ b/datafusion/datasource-parquet/src/metadata.rs@@ -19,6 +19,7 @@//! and schema information.use crate::file_format::ObjectStoreFetch;+use crate::leaf_resolver::LeafResolver;use crate::{Int96Coercer, apply_file_schema_type_coercions};use arrow::array::{Array, ArrayRef, BooleanArray};use arrow::compute::kernels::cmp::eq;@@ -42,7 +43,7 @@ use object_store::path::Path;use object_store::{ObjectMeta, ObjectStore};use parquet::DecodeResult;use parquet::arrow::arrow_reader::statistics::StatisticsConverter;-use parquet::arrow::{parquet_column, parquet_to_arrow_schema};+use parquet::arrow::parquet_to_arrow_schema;use parquet::basic::{ColumnOrder, SortOrder, Type as PhysicalType};use parquet::file::metadata::{PageIndexPolicy, ParquetMetaData, ParquetMetaDataPushDecoder, ParquetMetaDataReader,@@ -611,26 +612,27 @@ impl<'a> DFParquetMetadata<'a> {physical_file_schema = merged;}- statistics.column_statistics =- if has_statistics {- let (mut max_accs, mut min_accs) =- create_max_min_accs(logical_file_schema);- let mut null_counts_array =- vec![Precision::Absent; logical_file_schema.fields().len()];- let mut column_byte_sizes =- vec![Precision::Absent; logical_file_schema.fields().len()];- let mut is_max_value_exact =- vec![Some(true); logical_file_schema.fields().len()];- let mut is_min_value_exact =- vec![Some(true); logical_file_schema.fields().len()];- let mut distinct_counts_array =- vec![Precision::Absent; logical_file_schema.fields().len()];- logical_file_schema.fields().iter().enumerate().for_each(- |(idx, field)| match StatisticsConverter::try_new(- field.name(),- &physical_file_schema,- file_metadata.schema_descr(),- ) {+ let leaf_resolver =+ LeafResolver::new(file_metadata.schema_descr(), &physical_file_schema);++ statistics.column_statistics = if has_statistics {+ let (mut max_accs, mut min_accs) = create_max_min_accs(logical_file_schema);+ let mut null_counts_array =+ vec![Precision::Absent; logical_file_schema.fields().len()];+ let mut column_byte_sizes =+ vec![Precision::Absent; logical_file_schema.fields().len()];+ let mut is_max_value_exact =+ vec![Some(true); logical_file_schema.fields().len()];+ let mut is_min_value_exact =+ vec![Some(true); logical_file_schema.fields().len()];+ let mut distinct_counts_array =+ vec![Precision::Absent; logical_file_schema.fields().len()];+ logical_file_schema+ .fields()+ .iter()+ .enumerate()+ .for_each(|(idx, field)| {+ match leaf_resolver.converter(field.name()) {Ok(stats_converter) => {// An omitted count must not become an exact zero in// file statistics used for pruning and aggregates.@@ -676,44 +678,39 @@ impl<'a> DFParquetMetadata<'a> {debug!("Failed to create statistics converter: {e}");null_counts_array[idx] = Precision::Exact(num_rows);}- },- );-- let mut accumulators = StatisticsAccumulators {- min_accs: &mut min_accs,- max_accs: &mut max_accs,- null_counts_array: &mut null_counts_array,- is_min_value_exact: &mut is_min_value_exact,- is_max_value_exact: &mut is_max_value_exact,- column_byte_sizes: &mut column_byte_sizes,- distinct_counts_array: &mut distinct_counts_array,- };- accumulators.build_column_statistics(logical_file_schema)- } else {- // Record column sizes- logical_file_schema- .fields()- .iter()- .enumerate()- .map(|(logical_file_schema_index, field)| {- let arrow_field =- logical_file_schema.field(logical_file_schema_index);- let parquet_idx = parquet_column(- file_metadata.schema_descr(),- &physical_file_schema,- arrow_field.name(),- )- .map(|(idx, _)| idx);- let byte_size = compute_arrow_column_size(- field.data_type(),- row_groups_metadata,- parquet_idx,- num_rows,- );- ColumnStatistics::new_unknown().with_byte_size(byte_size)- })- .collect()+ }+ });++ let mut accumulators = StatisticsAccumulators {+ min_accs: &mut min_accs,+ max_accs: &mut max_accs,+ null_counts_array: &mut null_counts_array,+ is_min_value_exact: &mut is_min_value_exact,+ is_max_value_exact: &mut is_max_value_exact,+ column_byte_sizes: &mut column_byte_sizes,+ distinct_counts_array: &mut distinct_counts_array,};+ accumulators.build_column_statistics(logical_file_schema)+ } else {+ // Record column sizes+ logical_file_schema+ .fields()+ .iter()+ .enumerate()+ .map(|(logical_file_schema_index, field)| {+ let arrow_field =+ logical_file_schema.field(logical_file_schema_index);+ let parquet_idx = leaf_resolver.leaf_index(arrow_field.name());+ let byte_size = compute_arrow_column_size(+ field.data_type(),+ row_groups_metadata,+ parquet_idx,+ num_rows,+ );+ ColumnStatistics::new_unknown().with_byte_size(byte_size)+ })+ .collect()+ };#[cfg(debug_assertions)]{diff --git a/datafusion/datasource-parquet/src/mod.rs b/datafusion/datasource-parquet/src/mod.rsindex 80d2205a1f..6837e05194 100644--- a/datafusion/datasource-parquet/src/mod.rs+++ b/datafusion/datasource-parquet/src/mod.rs@@ -28,6 +28,7 @@ pub mod access_plan;mod bloom_filter;mod decoder_projection;pub mod file_format;+mod leaf_resolver;pub mod metadata;mod metrics;mod nested_schema_pruning;