Apache DataFusion SQL Query Engine
fix: honor Parquet byte-array statistics ordering (#24525)
Chao Sun26b40dd781d2
diff --git a/datafusion/datasource-parquet/benches/parquet_metadata_statistics.rs b/datafusion/datasource-parquet/benches/parquet_metadata_statistics.rsindex 46ebd100fd..82147abfd9 100644--- a/datafusion/datasource-parquet/benches/parquet_metadata_statistics.rs+++ b/datafusion/datasource-parquet/benches/parquet_metadata_statistics.rs@@ -28,6 +28,7 @@ use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use criterion::{BatchSize, BenchmarkId, Criterion, criterion_group, criterion_main};use datafusion_datasource_parquet::metadata::DFParquetMetadata;use parquet::arrow::ArrowSchemaConverter;+use parquet::basic::ColumnOrder;use parquet::data_type::ByteArray;use parquet::file::metadata::{ColumnChunkMetaData, FileMetaData, ParquetMetaData, RowGroupMetaData,@@ -169,13 +170,19 @@ fn make_synthetic_metadata(}).collect::<Vec<_>>();+ // Model a modern writer so byte-array bounds have a trustworthy order.+ let column_orders = schema_descr+ .columns()+ .iter()+ .map(|column| ColumnOrder::TYPE_DEFINED_ORDER(column.sort_order()))+ .collect();let file_metadata = FileMetaData::new(1,(spec.row_groups * ROWS_PER_GROUP) as i64,Some("datafusion parquet metadata benchmark".to_string()),None,schema_descr,- None,+ Some(column_orders),);ParquetMetaData::new(file_metadata, row_groups)diff --git a/datafusion/datasource-parquet/src/metadata.rs b/datafusion/datasource-parquet/src/metadata.rsindex c01811c087..213cac24a8 100644--- a/datafusion/datasource-parquet/src/metadata.rs+++ b/datafusion/datasource-parquet/src/metadata.rs@@ -43,12 +43,13 @@ use object_store::{ObjectMeta, ObjectStore};use parquet::DecodeResult;use parquet::arrow::arrow_reader::statistics::StatisticsConverter;use parquet::arrow::{parquet_column, parquet_to_arrow_schema};+use parquet::basic::{ColumnOrder, SortOrder, Type as PhysicalType};use parquet::file::metadata::{PageIndexPolicy, ParquetMetaData, ParquetMetaDataPushDecoder, ParquetMetaDataReader,RowGroupMetaData, SortingColumn,};use parquet::file::statistics::Statistics as ParquetStatistics;-use parquet::schema::types::SchemaDescriptor;+use parquet::schema::types::{ColumnDescriptor, SchemaDescriptor};use std::any::Any;use std::sync::Arc;@@ -57,6 +58,60 @@ use std::sync::Arc;/// would be too unreliable otherwise.const PARTIAL_NDV_THRESHOLD: f64 = 0.75;+fn requires_unsigned_byte_array_order(column: &ColumnDescriptor) -> bool {+ matches!(+ column.physical_type(),+ PhysicalType::BYTE_ARRAY | PhysicalType::FIXED_LEN_BYTE_ARRAY+ ) && column.sort_order() != SortOrder::SIGNED+}++/// Whether a column's min/max bounds lack a comparison order matching Arrow's.+///+/// The deprecated Parquet `min`/`max` fields use signed comparison, unlike+/// Arrow's string and binary comparisons. Even the modern bounds cannot be+/// interpreted without the corresponding footer `column_orders` entry.+/// Signed logical types, such as decimals, retain their existing behavior.+/// Columns with undefined sort orders, such as `INT96`, never have usable+/// min/max bounds regardless of their physical type. The `INT96` check is+/// defensive because parquet-rs does not currently expose those bounds.+pub(crate) fn has_untrusted_min_max_order(+ parquet_schema: &SchemaDescriptor,+ column_orders: Option<&[ColumnOrder]>,+ parquet_column_index: usize,+) -> bool {+ let column = parquet_schema.column(parquet_column_index);+ if column.sort_order() == SortOrder::UNDEFINED {+ return true;+ }+ requires_unsigned_byte_array_order(&column)+ && (column.sort_order() != SortOrder::UNSIGNED+ || column_orders+ .and_then(|orders| orders.get(parquet_column_index))+ .copied()+ != Some(ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::UNSIGNED)))+}++/// Whether any row group provides byte-array bounds in the deprecated signed+/// order, or the column's logical order is undefined.+pub(crate) fn has_untrusted_byte_array_stats<'a>(+ parquet_schema: &SchemaDescriptor,+ parquet_column_index: Option<usize>,+ row_groups: impl IntoIterator<Item = &'a RowGroupMetaData>,+) -> bool {+ parquet_column_index.is_some_and(|index| {+ let column = parquet_schema.column(index);+ requires_unsigned_byte_array_order(&column)+ && (column.sort_order() != SortOrder::UNSIGNED+ || row_groups.into_iter().any(|group| {+ group.column(index).statistics().is_some_and(|stats| {+ stats.is_min_max_deprecated()+ && (stats.min_bytes_opt().is_some()+ || stats.max_bytes_opt().is_some())+ })+ }))+ })+}+/// Handles fetching Parquet file schema, metadata and statistics/// from object store.///@@ -496,6 +551,23 @@ impl<'a> DFParquetMetadata<'a> {file_metadata.schema_descr(),) {Ok(stats_converter) => {+ let parquet_index = stats_converter.parquet_column_index();+ if parquet_index.is_some_and(|index| {+ has_untrusted_min_max_order(+ file_metadata.schema_descr(),+ file_metadata.column_orders().map(Vec::as_slice),+ index,+ )+ }) || has_untrusted_byte_array_stats(+ file_metadata.schema_descr(),+ parquet_index,+ row_groups_metadata,+ ) {+ // The remaining row groups cannot establish bounds+ // for the whole file. Keep unrelated statistics.+ min_accs[idx] = None;+ max_accs[idx] = None;+ }let mut accumulators = StatisticsAccumulators {min_accs: &mut min_accs,max_accs: &mut max_accs,diff --git a/datafusion/datasource-parquet/src/mod.rs b/datafusion/datasource-parquet/src/mod.rsindex 35f831230b..23348f94aa 100644--- a/datafusion/datasource-parquet/src/mod.rs+++ b/datafusion/datasource-parquet/src/mod.rs@@ -42,6 +42,8 @@ mod schema_coercion;mod sink;mod sort;pub mod source;+#[cfg(test)]+mod statistics_order_tests;mod supported_predicates;#[cfg(test)]mod test_util;diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rsindex 5e004cf017..0b3d29c27a 100644--- a/datafusion/datasource-parquet/src/opener/mod.rs+++ b/datafusion/datasource-parquet/src/opener/mod.rs@@ -1129,10 +1129,9 @@ impl FiltersPreparedParquetOpen {// If there is a predicate that can be evaluated against the metadataif let Some(predicate) = self.pruning_predicate.as_ref().map(|p| p.as_ref()) {if prepared.enable_row_group_stats_pruning {- row_groups.prune_by_statistics(+ row_groups.prune_by_statistics_with_metadata(&prepared.physical_file_schema,- loaded.reader_metadata.parquet_schema(),- rg_metadata,+ &file_metadata,predicate,&prepared.file_metrics,);diff --git a/datafusion/datasource-parquet/src/page_filter.rs b/datafusion/datasource-parquet/src/page_filter.rsindex 6bc1aca667..5372ae0a00 100644--- a/datafusion/datasource-parquet/src/page_filter.rs+++ b/datafusion/datasource-parquet/src/page_filter.rs@@ -22,6 +22,7 @@ use std::sync::Arc;use super::metrics::ParquetFileMetrics;use crate::ParquetAccessPlan;+use crate::metadata::has_untrusted_min_max_order;use arrow::array::BooleanArray;use arrow::{@@ -490,6 +491,7 @@ struct PagesPruningStatistics<'a> {column_index: &'a ParquetColumnIndex,offset_index: &'a ParquetOffsetIndex,page_offsets: &'a Vec<PageLocation>,+ trusted_min_max: bool,}impl<'a> PagesPruningStatistics<'a> {@@ -529,6 +531,12 @@ impl<'a> PagesPruningStatistics<'a> {return None;};let page_offsets = offset_index_metadata.page_locations();+ let file_metadata = parquet_metadata.file_metadata();+ let trusted_min_max = !has_untrusted_min_max_order(+ file_metadata.schema_descr(),+ file_metadata.column_orders().map(Vec::as_slice),+ parquet_column_index,+ );Some(Self {row_group_index,@@ -537,6 +545,7 @@ impl<'a> PagesPruningStatistics<'a> {column_index,offset_index,page_offsets,+ trusted_min_max,})}@@ -563,6 +572,9 @@ impl<'a> PagesPruningStatistics<'a> {}impl PruningStatistics for PagesPruningStatistics<'_> {fn min_values(&self, _column: &datafusion_common::Column) -> Option<ArrayRef> {+ if !self.trusted_min_max {+ return None;+ }match self.converter.data_page_mins(self.column_index,self.offset_index,@@ -577,6 +589,9 @@ impl PruningStatistics for PagesPruningStatistics<'_> {}fn max_values(&self, _column: &datafusion_common::Column) -> Option<ArrayRef> {+ if !self.trusted_min_max {+ return None;+ }match self.converter.data_page_maxes(self.column_index,self.offset_index,diff --git a/datafusion/datasource-parquet/src/push_decoder.rs b/datafusion/datasource-parquet/src/push_decoder.rsindex 8238f13b00..03b7d99c28 100644--- a/datafusion/datasource-parquet/src/push_decoder.rs+++ b/datafusion/datasource-parquet/src/push_decoder.rs@@ -232,6 +232,11 @@ impl RowGroupPruner {.collect::<Vec<_>>();let stats = RowGroupPruningStatistics {parquet_schema: self.parquet_metadata.file_metadata().schema_descr(),+ column_orders: self+ .parquet_metadata+ .file_metadata()+ .column_orders()+ .map(Vec::as_slice),row_group_metadatas,arrow_schema: self.arrow_schema.as_ref(),// Match the existing static row-group pruning behavior: when adiff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rsindex df02355c0f..2db4cb718d 100644--- a/datafusion/datasource-parquet/src/row_group_filter.rs+++ b/datafusion/datasource-parquet/src/row_group_filter.rs@@ -20,7 +20,9 @@ use std::sync::Arc;use super::{ParquetAccessPlan, ParquetFileMetrics, RowGroupAccess};use crate::bloom_filter::BloomFilterStatistics;+use crate::metadata::{has_untrusted_byte_array_stats, has_untrusted_min_max_order};use arrow::array::{ArrayRef, BooleanArray, UInt64Array};+use arrow::compute::nullif;use arrow::datatypes::Schema;use datafusion_common::pruning::PruningStatistics;use datafusion_common::{Column, Result, ScalarValue};@@ -31,7 +33,8 @@ use datafusion_physical_expr::utils::collect_columns;use datafusion_physical_expr::{PhysicalExpr, PhysicalExprSimplifier};use datafusion_pruning::{PruningPredicate, PruningPredicateBuilder};use parquet::arrow::arrow_reader::statistics::StatisticsConverter;-use parquet::file::metadata::RowGroupMetaData;+use parquet::basic::ColumnOrder;+use parquet::file::metadata::{ParquetMetaData, RowGroupMetaData};use parquet::schema::types::SchemaDescriptor;/// Reduces the [`ParquetAccessPlan`] based on row group level metadata.@@ -259,8 +262,10 @@ impl RowGroupAccessPlanFilter {////// Updates this set to mark row groups that should not be scanned///- /// Note: This method currently ignores ColumnOrder- /// <https://github.com/apache/datafusion/issues/8335>+ /// This method has no file footer, so it cannot establish the sort order+ /// of string or binary min/max statistics. Such bounds are ignored. Use+ /// [`Self::prune_by_statistics_with_metadata`] when the full metadata is+ /// available. Null counts and other columns' statistics remain usable.////// # Panics/// if `groups.len() != self.len()`@@ -271,6 +276,51 @@ impl RowGroupAccessPlanFilter {groups: &[RowGroupMetaData],predicate: &PruningPredicate,metrics: &ParquetFileMetrics,+ ) {+ self.prune_by_statistics_inner(+ arrow_schema,+ parquet_schema,+ groups,+ None,+ predicate,+ metrics,+ );+ }++ /// Prune row groups using statistics and the comparison orders recorded+ /// in the Parquet file footer.+ ///+ /// Unlike [`Self::prune_by_statistics`], this method can use string and+ /// binary bounds when their ordering is known to match Arrow's ordering.+ ///+ /// # Panics+ /// if `metadata.num_row_groups() != self.len()`+ pub fn prune_by_statistics_with_metadata(+ &mut self,+ arrow_schema: &Schema,+ metadata: &ParquetMetaData,+ predicate: &PruningPredicate,+ metrics: &ParquetFileMetrics,+ ) {+ let file_metadata = metadata.file_metadata();+ self.prune_by_statistics_inner(+ arrow_schema,+ file_metadata.schema_descr(),+ metadata.row_groups(),+ file_metadata.column_orders().map(Vec::as_slice),+ predicate,+ metrics,+ );+ }++ fn prune_by_statistics_inner(+ &mut self,+ arrow_schema: &Schema,+ parquet_schema: &SchemaDescriptor,+ groups: &[RowGroupMetaData],+ column_orders: Option<&[ColumnOrder]>,+ predicate: &PruningPredicate,+ metrics: &ParquetFileMetrics,) {// scoped timer updates on droplet _timer_guard = metrics.statistics_eval_time.timer();@@ -285,6 +335,7 @@ impl RowGroupAccessPlanFilter {let pruning_stats = RowGroupPruningStatistics {parquet_schema,+ column_orders,row_group_metadatas,arrow_schema,// Preserve the existing row-group pruning behavior. This path only@@ -311,8 +362,7 @@ impl RowGroupAccessPlanFilter {// Check if any of the matched row groups are fully contained by the predicateself.identify_fully_matched_row_groups(&fully_contained_candidates_original_idx,- arrow_schema,- parquet_schema,+ &pruning_stats,groups,predicate,metrics,@@ -337,8 +387,7 @@ impl RowGroupAccessPlanFilter {fn identify_fully_matched_row_groups(&mut self,candidate_row_group_indices: &[usize],- arrow_schema: &Schema,- parquet_schema: &SchemaDescriptor,+ pruning_stats: &RowGroupPruningStatistics<'_>,groups: &[RowGroupMetaData],predicate: &PruningPredicate,metrics: &ParquetFileMetrics,@@ -346,6 +395,7 @@ impl RowGroupAccessPlanFilter {if candidate_row_group_indices.is_empty() {return;}+ let arrow_schema = pruning_stats.arrow_schema;let mut inverted_expr: Arc<dyn PhysicalExpr> =Arc::new(NotExpr::new(Arc::clone(predicate.orig_expr())));@@ -388,7 +438,8 @@ impl RowGroupAccessPlanFilter {};let inverted_pruning_stats = RowGroupPruningStatistics {- parquet_schema,+ parquet_schema: pruning_stats.parquet_schema,+ column_orders: pruning_stats.column_orders,row_group_metadatas: candidate_row_group_indices.iter().map(|&i| &groups[i])@@ -479,6 +530,7 @@ impl RowGroupAccessPlanFilter {/// duplicating the statistics-to-`PruningStatistics` plumbing.pub(crate) struct RowGroupPruningStatistics<'a> {pub(crate) parquet_schema: &'a SchemaDescriptor,+ pub(crate) column_orders: Option<&'a [ColumnOrder]>,pub(crate) row_group_metadatas: Vec<&'a RowGroupMetaData>,pub(crate) arrow_schema: &'a Schema,pub(crate) missing_null_counts_as_zero: bool,@@ -486,14 +538,11 @@ pub(crate) struct RowGroupPruningStatistics<'a> {impl<'a> RowGroupPruningStatistics<'a> {/// Return an iterator over the row group metadata- fn metadata_iter(&'a self) -> impl Iterator<Item = &'a RowGroupMetaData> + 'a {+ fn metadata_iter(&self) -> impl Iterator<Item = &'a RowGroupMetaData> + '_ {self.row_group_metadatas.iter().copied()}- fn statistics_converter<'b>(- &'a self,- column: &'b Column,- ) -> Result<StatisticsConverter<'a>> {+ fn statistics_converter(&self, column: &Column) -> Result<StatisticsConverter<'a>> {Ok(StatisticsConverter::try_new(&column.name,self.arrow_schema,@@ -501,19 +550,53 @@ impl<'a> RowGroupPruningStatistics<'a> {)?.with_missing_null_counts_as_zero(self.missing_null_counts_as_zero))}++ fn min_max_statistics_converter(+ &self,+ column: &Column,+ ) -> Option<StatisticsConverter<'a>> {+ let converter = self.statistics_converter(column).ok()?;+ let parquet_index = converter.parquet_column_index();+ if parquet_index.is_some_and(|index| {+ has_untrusted_min_max_order(self.parquet_schema, self.column_orders, index)+ }) {+ return None;+ }+ Some(converter)+ }++ fn mask_untrusted_byte_array_stats(+ &self,+ parquet_index: Option<usize>,+ values: ArrayRef,+ ) -> Option<ArrayRef> {+ if !has_untrusted_byte_array_stats(+ self.parquet_schema,+ parquet_index,+ self.metadata_iter(),+ ) {+ return Some(values);+ }+ // A file may mix legacy and modern row-group statistics. Keep the+ // modern bounds usable rather than discarding this entire column.+ let mask = BooleanArray::from_iter(self.metadata_iter().map(|group| {+ has_untrusted_byte_array_stats(self.parquet_schema, parquet_index, [group])+ }));+ nullif(values.as_ref(), &mask).ok()+ }}impl PruningStatistics for RowGroupPruningStatistics<'_> {fn min_values(&self, column: &Column) -> Option<ArrayRef> {- self.statistics_converter(column)- .and_then(|c| Ok(c.row_group_mins(self.metadata_iter())?))- .ok()+ let converter = self.min_max_statistics_converter(column)?;+ let values = converter.row_group_mins(self.metadata_iter()).ok()?;+ self.mask_untrusted_byte_array_stats(converter.parquet_column_index(), values)}fn max_values(&self, column: &Column) -> Option<ArrayRef> {- self.statistics_converter(column)- .and_then(|c| Ok(c.row_group_maxes(self.metadata_iter())?))- .ok()+ let converter = self.min_max_statistics_converter(column)?;+ let values = converter.row_group_maxes(self.metadata_iter()).ok()?;+ self.mask_untrusted_byte_array_stats(converter.parquet_column_index(), values)}fn num_containers(&self) -> usize {diff --git a/datafusion/datasource-parquet/src/statistics_order_tests.rs b/datafusion/datasource-parquet/src/statistics_order_tests.rsnew file mode 100644index 0000000000..bf9df532f1--- /dev/null+++ b/datafusion/datasource-parquet/src/statistics_order_tests.rs@@ -0,0 +1,749 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Regression tests for interpreting Parquet byte-array statistics orders.++use std::io::Write;+use std::sync::Arc;++use arrow::array::{BooleanArray, record_batch};+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};+use bytes::Bytes;+use datafusion_common::pruning::{PrunableStatistics, PruningStatistics};+use datafusion_common::stats::Precision;+use datafusion_common::{Column, ScalarValue, Statistics};+use datafusion_expr::{Expr, col, lit};+use datafusion_physical_expr::PhysicalExpr;+use datafusion_physical_expr::planner::logical2physical;+use datafusion_physical_plan::metrics::{Count, ExecutionPlanMetricsSet};+use datafusion_pruning::{MAX_IN_LIST_SIZE, PruningPredicate, PruningPredicateBuilder};+use parquet::arrow::ArrowWriter;+use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;+use parquet::basic::{ColumnOrder, LogicalType, SortOrder, Type as PhysicalType};+use parquet::data_type::{ByteArray, FixedLenByteArray};+use parquet::file::metadata::{+ ColumnChunkMetaData, ColumnIndexBuilder, FileMetaData, OffsetIndexBuilder,+ PageIndexPolicy, ParquetMetaData, ParquetMetaDataReader, ParquetMetaDataWriter,+ RowGroupMetaData,+};+use parquet::file::properties::{EnabledStatistics, WriterProperties};+use parquet::file::statistics::Statistics as ParquetStatistics;+use parquet::file::writer::TrackedWrite;+use parquet::schema::types::{SchemaDescriptor, Type as ParquetType};++use crate::RowGroupAccessPlanFilter;+use crate::metadata::{DFParquetMetadata, has_untrusted_min_max_order};+use crate::push_decoder::RowGroupPruner;+use crate::row_group_filter::RowGroupPruningStatistics;+use crate::{PagePruningAccessPlanFilter, ParquetAccessPlan, ParquetFileMetrics};++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+enum StatisticsOrder {+ Modern,+ Deprecated,+ Missing,+ Unknown,+}++struct TestFile {+ bytes: Bytes,+ schema: SchemaRef,+ metadata: Arc<ParquetMetaData>,+}++impl TestFile {+ fn new(order: StatisticsOrder) -> Self {+ let batch = record_batch!(+ (+ "s",+ Utf8,+ vec![+ Some("aé"),+ Some("az"),+ Some("b"),+ None,+ None,+ None,+ Some("d"),+ Some("e"),+ Some("f"),+ ]+ ),+ ("n", Int32, [1, 2, 3, 10, 11, 12, 20, 21, 22])+ )+ .unwrap();+ let schema = batch.schema();+ let properties = WriterProperties::builder()+ .set_max_row_group_row_count(Some(3))+ .set_data_page_row_count_limit(3)+ .set_write_batch_size(3)+ .set_dictionary_enabled(false)+ .set_statistics_enabled(EnabledStatistics::Page)+ .build();+ let mut original = Vec::new();+ let mut writer =+ ArrowWriter::try_new(&mut original, Arc::clone(&schema), Some(properties))+ .unwrap();+ writer.write(&batch).unwrap();+ writer.close().unwrap();+ let original = Bytes::from(original);+ let metadata = read_metadata(&original);+ assert_eq!(metadata.num_row_groups(), 3);++ if order == StatisticsOrder::Modern {+ return Self {+ bytes: original,+ schema,+ metadata: Arc::new(metadata),+ };+ }++ // Signed-byte comparison gives ["aé", "b"] for the first row+ // group's ["aé", "az", "b"]. The endpoints are not inverted in+ // unsigned order, but the interval wrongly excludes "az".+ let mut row_groups = metadata.row_groups().to_vec();+ let mut columns = row_groups[0].columns().to_vec();+ columns[0] = columns[0]+ .clone()+ .into_builder()+ .set_statistics(ParquetStatistics::byte_array(+ Some(ByteArray::from("aé")),+ Some(ByteArray::from("b")),+ None,+ Some(0),+ order == StatisticsOrder::Deprecated,+ ))+ .build()+ .unwrap();+ row_groups[0] = row_groups[0]+ .clone()+ .into_builder()+ .set_column_metadata(columns)+ .build()+ .unwrap();++ let mut column_index = metadata.column_index().unwrap().clone();+ if matches!(order, StatisticsOrder::Missing | StatisticsOrder::Unknown) {+ let mut index = ColumnIndexBuilder::new(PhysicalType::BYTE_ARRAY);+ index.append(false, "aé".as_bytes().to_vec(), b"b".to_vec(), 0);+ column_index[0][0] = index.build().unwrap();+ }+ let metadata = metadata+ .into_builder()+ .set_row_groups(row_groups)+ .set_column_index(Some(column_index))+ .build();++ // Keep the real data pages, and serialize the replacement statistics+ // and page indexes at their actual file offsets.+ let mut bytes = Vec::new();+ let mut tracked = TrackedWrite::new(&mut bytes);+ tracked+ .write_all(&original[..footer_start(&original)])+ .unwrap();+ ParquetMetaDataWriter::new_with_tracked(tracked, &metadata)+ .finish()+ .unwrap();++ // parquet-rs always writes TYPEORDER and cannot write an unknown+ // ColumnOrder. The final Thrift field is column_orders (field 7): a+ // two-element list of unions, followed by the FileMetaData STOP.+ // Alter just that field to model old and future writers, then verify+ // the decoded footer below. No data or page-index offsets change.+ if matches!(order, StatisticsOrder::Missing | StatisticsOrder::Unknown) {+ let end = bytes.len() - 8;+ let encoded_orders = [0x19, 0x2c, 0x1c, 0, 0, 0x1c, 0, 0, 0];+ let start = end - encoded_orders.len();+ assert_eq!(&bytes[start..end], &encoded_orders);+ let metadata_start = footer_start(&bytes);+ if order == StatisticsOrder::Missing {+ bytes.drain(start..end - 1);+ let new_end = bytes.len() - 8;+ let metadata_len = (new_end - metadata_start) as u32;+ bytes[new_end..new_end + 4].copy_from_slice(&metadata_len.to_le_bytes());+ } else {+ // Change the first union member from field 1 (TYPEORDER) to+ // an unrecognized field 2. The numeric column stays known.+ bytes[start + 2] = 0x2c;+ }+ }++ let bytes = Bytes::from(bytes);+ let metadata = read_metadata(&bytes);+ let expected_order = match order {+ StatisticsOrder::Missing => ColumnOrder::UNDEFINED,+ StatisticsOrder::Unknown => ColumnOrder::UNKNOWN,+ _ => ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::UNSIGNED),+ };+ assert_eq!(metadata.file_metadata().column_order(0), expected_order);+ assert_eq!(+ metadata+ .row_group(0)+ .column(0)+ .statistics()+ .unwrap()+ .is_min_max_deprecated(),+ order == StatisticsOrder::Deprecated,+ );+ Self {+ bytes,+ schema,+ metadata: Arc::new(metadata),+ }+ }++ fn predicate(&self, expr: &Expr) -> (Arc<dyn PhysicalExpr>, PruningPredicate) {+ let physical = logical2physical(expr, &self.schema);+ let pruning = PruningPredicateBuilder::new()+ .with_file_schema(Arc::clone(&self.schema))+ .try_build(Arc::clone(&physical))+ .unwrap();+ (physical, pruning)+ }++ fn statistics(&self) -> Statistics {+ DFParquetMetadata::statistics_from_parquet_metadata(&self.metadata, &self.schema)+ .unwrap()+ }++ fn file_matches(&self, predicate: &PruningPredicate) -> bool {+ let stats = PrunableStatistics::new(+ vec![Arc::new(self.statistics())],+ Arc::clone(&self.schema),+ );+ predicate.prune(&stats).unwrap()[0]+ }++ fn row_group_plan(&self, predicate: &PruningPredicate) -> ParquetAccessPlan {+ let mut filter = RowGroupAccessPlanFilter::new(ParquetAccessPlan::new_all(+ self.metadata.num_row_groups(),+ ));+ filter.prune_by_statistics_with_metadata(+ &self.schema,+ &self.metadata,+ predicate,+ &metrics(),+ );+ filter.build()+ }++ fn page_plan(+ &self,+ physical: &Arc<dyn PhysicalExpr>,+ plan: ParquetAccessPlan,+ ) -> ParquetAccessPlan {+ PagePruningAccessPlanFilter::new(physical, Arc::clone(&self.schema))+ .prune_plan_with_page_index(+ plan,+ &self.schema,+ self.metadata.file_metadata().schema_descr(),+ &self.metadata,+ &metrics(),+ )+ }++ fn matching_rows(+ &self,+ physical: &Arc<dyn PhysicalExpr>,+ plan: ParquetAccessPlan,+ ) -> usize {+ let mut builder = ParquetRecordBatchReaderBuilder::try_new(self.bytes.clone())+ .unwrap()+ .with_row_groups(plan.row_group_indexes());+ if let Some(selection) = plan+ .into_overall_row_selection(self.metadata.row_groups())+ .unwrap()+ {+ builder = builder.with_row_selection(selection);+ }+ builder+ .build()+ .unwrap()+ .map(|batch| {+ let batch = batch.unwrap();+ let matches = physical+ .evaluate(&batch)+ .unwrap()+ .into_array(batch.num_rows())+ .unwrap();+ matches+ .as_any()+ .downcast_ref::<BooleanArray>()+ .unwrap()+ .true_count()+ })+ .sum()+ }+}++fn footer_start(bytes: &[u8]) -> usize {+ let end = bytes.len() - 8;+ let metadata_len = u32::from_le_bytes(bytes[end..end + 4].try_into().unwrap());+ end - metadata_len as usize+}++fn read_metadata(bytes: &Bytes) -> ParquetMetaData {+ ParquetMetaDataReader::new()+ .with_page_index_policy(PageIndexPolicy::Required)+ .parse_and_finish(bytes)+ .unwrap()+}++fn metrics() -> ParquetFileMetrics {+ ParquetFileMetrics::new(+ 0,+ "statistics-order.parquet",+ &ExecutionPlanMetricsSet::new(),+ )+}++#[test]+fn byte_array_order_preserves_matching_rows_at_every_pruning_level() {+ for order in [+ StatisticsOrder::Modern,+ StatisticsOrder::Deprecated,+ StatisticsOrder::Missing,+ StatisticsOrder::Unknown,+ ] {+ let file = TestFile::new(order);+ let (physical, predicate) = file.predicate(&col("s").eq(lit("az")));+ let all = ParquetAccessPlan::new_all(file.metadata.num_row_groups());+ assert_eq!(file.matching_rows(&physical, all.clone()), 1);+ assert!(file.file_matches(&predicate), "order={order:?}");+ let row_groups = file.row_group_plan(&predicate);+ assert!(row_groups.should_scan(0), "order={order:?}");+ assert!(!row_groups.is_fully_matched(0), "order={order:?}");+ assert_eq!(+ row_groups.row_group_indexes(),+ if matches!(order, StatisticsOrder::Modern | StatisticsOrder::Deprecated) {+ vec![0]+ } else {+ vec![0, 2]+ },+ );+ assert_eq!(file.matching_rows(&physical, row_groups.clone()), 1);+ assert_eq!(+ file.matching_rows(&physical, file.page_plan(&physical, all)),+ 1+ );+ assert_eq!(+ file.matching_rows(&physical, file.page_plan(&physical, row_groups)),+ 1,+ "order={order:?}",+ );++ let mut runtime_pruner = RowGroupPruner::new(+ physical,+ Arc::clone(&file.schema),+ Arc::clone(&file.metadata),+ Count::new(),+ Count::new(),+ MAX_IN_LIST_SIZE,+ );+ assert!(!runtime_pruner.should_prune(&[0]), "order={order:?}");+ assert!(runtime_pruner.should_prune(&[1]), "all-null row group");+ }+}++#[test]+fn byte_array_order_keeps_null_counts_and_unrelated_column_bounds() {+ for order in [+ StatisticsOrder::Deprecated,+ StatisticsOrder::Missing,+ StatisticsOrder::Unknown,+ ] {+ let file = TestFile::new(order);+ let statistics = file.statistics();+ let string = &statistics.column_statistics[0];+ assert_eq!(string.min_value, Precision::Absent, "order={order:?}");+ assert_eq!(string.max_value, Precision::Absent, "order={order:?}");+ assert_eq!(string.null_count, Precision::Exact(3));+ assert_eq!(+ statistics.column_statistics[1].min_value,+ Precision::Exact(ScalarValue::Int32(Some(1))),+ );+ assert_eq!(+ statistics.column_statistics[1].max_value,+ Precision::Exact(ScalarValue::Int32(Some(22))),+ );++ let (physical, predicate) = file.predicate(&col("s").is_null());+ assert_eq!(file.row_group_plan(&predicate).row_group_indexes(), vec![1]);+ let page_plan = file.page_plan(&physical, ParquetAccessPlan::new_all(3));+ assert_eq!(page_plan.row_group_indexes(), vec![1]);+ assert_eq!(file.matching_rows(&physical, page_plan), 3);++ let expr = col("s").eq(lit("az")).and(col("n").eq(lit(99)));+ let (physical, predicate) = file.predicate(&expr);+ assert!(!file.file_matches(&predicate));+ assert!(+ file.row_group_plan(&predicate)+ .row_group_indexes()+ .is_empty()+ );+ assert!(+ file.page_plan(&physical, ParquetAccessPlan::new_all(3))+ .row_group_indexes()+ .is_empty(),+ );+ }+}++#[test]+fn byte_array_order_modern_bounds_and_null_only_groups_remain_usable() {+ let file = TestFile::new(StatisticsOrder::Modern);+ let statistics = file.statistics();+ assert_eq!(+ statistics.column_statistics[0].min_value,+ Precision::Exact(ScalarValue::Utf8(Some("az".to_owned()))),+ );+ let (physical, predicate) = file.predicate(&col("s").eq(lit("az")));+ assert_eq!(file.row_group_plan(&predicate).row_group_indexes(), vec![0]);+ assert_eq!(+ file.page_plan(&physical, ParquetAccessPlan::new_all(3))+ .row_group_indexes(),+ vec![0],+ );++ // The compatibility API has no footer. It still uses null counts, but+ // cannot assume that even modern-looking byte-array bounds are unsigned.+ let mut filter = RowGroupAccessPlanFilter::new(ParquetAccessPlan::new_all(3));+ filter.prune_by_statistics(+ &file.schema,+ file.metadata.file_metadata().schema_descr(),+ file.metadata.row_groups(),+ &predicate,+ &metrics(),+ );+ assert_eq!(filter.build().row_group_indexes(), vec![0, 2]);+}++#[test]+fn byte_array_order_guard_follows_parquet_type_not_arrow_representation() {+ let file = TestFile::new(StatisticsOrder::Deprecated);+ let metadata = file.metadata.file_metadata();+ let column = Column::from_name("s");+ for data_type in [+ DataType::Utf8,+ DataType::LargeUtf8,+ DataType::Utf8View,+ DataType::Binary,+ DataType::LargeBinary,+ DataType::BinaryView,+ DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)),+ ] {+ let schema = Schema::new(vec![Field::new("s", data_type, true)]);+ let stats = RowGroupPruningStatistics {+ parquet_schema: metadata.schema_descr(),+ column_orders: metadata.column_orders().map(Vec::as_slice),+ row_group_metadatas: file.metadata.row_groups().iter().collect(),+ arrow_schema: &schema,+ missing_null_counts_as_zero: true,+ };+ for values in [stats.min_values(&column), stats.max_values(&column)] {+ let values = values.unwrap();+ assert!(values.is_null(0));+ assert!(values.is_null(1));+ assert!(!values.is_null(2));+ }+ assert!(stats.null_counts(&column).is_some());+ }+}++fn single_column_metadata(+ parquet_type: ParquetType,+ statistics: ParquetStatistics,+ order: Option<ColumnOrder>,+) -> ParquetMetaData {+ let physical_type = parquet_type.get_physical_type();+ let schema = Arc::new(SchemaDescriptor::new(Arc::new(+ ParquetType::group_type_builder("schema")+ .with_fields(vec![Arc::new(parquet_type)])+ .build()+ .unwrap(),+ )));+ let mut column_index = ColumnIndexBuilder::new(physical_type);+ column_index.append(+ false,+ statistics.min_bytes_opt().unwrap().to_vec(),+ statistics.max_bytes_opt().unwrap().to_vec(),+ 0,+ );+ let mut offset_index = OffsetIndexBuilder::new();+ offset_index.append_row_count(3);+ offset_index.append_offset_and_size(0, 1);+ let column = ColumnChunkMetaData::builder(schema.column(0))+ .set_num_values(3)+ .set_statistics(statistics)+ .build()+ .unwrap();+ let group = RowGroupMetaData::builder(Arc::clone(&schema))+ .set_num_rows(3)+ .set_column_metadata(vec![column])+ .build()+ .unwrap();+ ParquetMetaData::new(+ FileMetaData::new(1, 3, None, None, schema, order.map(|order| vec![order])),+ vec![group],+ )+ .into_builder()+ .set_column_index(Some(vec![vec![column_index.build().unwrap()]]))+ .set_offset_index(Some(vec![vec![offset_index.build()]]))+ .build()+}++#[test]+fn fixed_byte_array_and_uuid_orders_guard_bounds_but_not_null_counts() {+ let schema = Arc::new(Schema::new(vec![Field::new(+ "s",+ DataType::FixedSizeBinary(16),+ true,+ )]));+ let padded = |prefix: &[u8]| {+ let mut value = vec![0; 16];+ value[..prefix.len()].copy_from_slice(prefix);+ value+ };+ let min = padded("aé".as_bytes());+ let max = padded(b"b");+ let value = ScalarValue::FixedSizeBinary(16, Some(padded(b"az")));+ let physical = logical2physical(&col("s").eq(lit(value)), &schema);+ let predicate = PruningPredicateBuilder::new()+ .with_file_schema(Arc::clone(&schema))+ .try_build(Arc::clone(&physical))+ .unwrap();++ for logical_type in [None, Some(LogicalType::Uuid)] {+ for deprecated in [false, true] {+ for order in [+ None,+ Some(ColumnOrder::UNKNOWN),+ Some(ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::SIGNED)),+ Some(ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::UNSIGNED)),+ ] {+ let parquet_type = ParquetType::primitive_type_builder(+ "s",+ PhysicalType::FIXED_LEN_BYTE_ARRAY,+ )+ .with_length(16)+ .with_logical_type(logical_type.clone())+ .build()+ .unwrap();+ let statistics = ParquetStatistics::fixed_len_byte_array(+ Some(FixedLenByteArray::from(min.clone())),+ Some(FixedLenByteArray::from(max.clone())),+ None,+ Some(0),+ deprecated,+ );+ let metadata = single_column_metadata(parquet_type, statistics, order);+ let trusted =+ order == Some(ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::UNSIGNED));+ let statistics = DFParquetMetadata::statistics_from_parquet_metadata(+ &metadata, &schema,+ )+ .unwrap();+ assert_eq!(+ statistics.column_statistics[0].min_value,+ if trusted && !deprecated {+ Precision::Exact(ScalarValue::FixedSizeBinary(+ 16,+ Some(min.clone()),+ ))+ } else {+ Precision::Absent+ },+ );+ assert_eq!(+ statistics.column_statistics[0].null_count,+ Precision::Exact(0),+ );++ let mut row_groups =+ RowGroupAccessPlanFilter::new(ParquetAccessPlan::new_all(1));+ row_groups.prune_by_statistics_with_metadata(+ &schema,+ &metadata,+ &predicate,+ &metrics(),+ );+ assert_eq!(row_groups.build().should_scan(0), !trusted || deprecated);+ let pages =+ PagePruningAccessPlanFilter::new(&physical, Arc::clone(&schema))+ .prune_plan_with_page_index(+ ParquetAccessPlan::new_all(1),+ &schema,+ metadata.file_metadata().schema_descr(),+ &metadata,+ &metrics(),+ );+ // Modern page indexes are independent of legacy row-group+ // bounds, but still need a recognized footer order.+ assert_eq!(pages.should_scan(0), !trusted);+ }+ }+ }+}++#[test]+fn signed_decimal_byte_array_statistics_remain_usable() {+ let schema = Arc::new(Schema::new(vec![Field::new(+ "s",+ DataType::Decimal128(10, 0),+ true,+ )]));+ for physical_type in [PhysicalType::BYTE_ARRAY, PhysicalType::FIXED_LEN_BYTE_ARRAY] {+ let parquet_type = ParquetType::primitive_type_builder("s", physical_type)+ .with_length(16)+ .with_logical_type(Some(LogicalType::decimal(0, 10)))+ .with_precision(10)+ .with_scale(0)+ .build()+ .unwrap();+ let min = (-2_i128).to_be_bytes().to_vec();+ let max = 3_i128.to_be_bytes().to_vec();+ let statistics = match physical_type {+ PhysicalType::BYTE_ARRAY => ParquetStatistics::byte_array(+ Some(ByteArray::from(min)),+ Some(ByteArray::from(max)),+ None,+ Some(0),+ true,+ ),+ _ => ParquetStatistics::fixed_len_byte_array(+ Some(FixedLenByteArray::from(min)),+ Some(FixedLenByteArray::from(max)),+ None,+ Some(0),+ true,+ ),+ };+ let metadata = single_column_metadata(parquet_type, statistics, None);+ let statistics =+ DFParquetMetadata::statistics_from_parquet_metadata(&metadata, &schema)+ .unwrap();+ assert_eq!(+ statistics.column_statistics[0].min_value,+ Precision::Exact(ScalarValue::Decimal128(Some(-2), 10, 0)),+ );+ let physical = logical2physical(+ &col("s").eq(lit(ScalarValue::Decimal128(Some(4), 10, 0))),+ &schema,+ );+ let predicate = PruningPredicateBuilder::new()+ .with_file_schema(Arc::clone(&schema))+ .try_build(physical)+ .unwrap();+ let mut row_groups = RowGroupAccessPlanFilter::new(ParquetAccessPlan::new_all(1));+ row_groups.prune_by_statistics_with_metadata(+ &schema,+ &metadata,+ &predicate,+ &metrics(),+ );+ assert!(!row_groups.build().should_scan(0));+ }+}++#[test]+fn undefined_int96_order_is_never_trusted() {+ let parquet_type = ParquetType::primitive_type_builder("s", PhysicalType::INT96)+ .build()+ .unwrap();+ let schema = SchemaDescriptor::new(Arc::new(+ ParquetType::group_type_builder("schema")+ .with_fields(vec![Arc::new(parquet_type)])+ .build()+ .unwrap(),+ ));+ assert_eq!(schema.column(0).sort_order(), SortOrder::UNDEFINED);++ for order in [+ None,+ Some(ColumnOrder::UNDEFINED),+ Some(ColumnOrder::UNKNOWN),+ Some(ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::UNDEFINED)),+ Some(ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::SIGNED)),+ Some(ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::UNSIGNED)),+ ] {+ assert!(+ has_untrusted_min_max_order(+ &schema,+ order.as_ref().map(std::slice::from_ref),+ 0+ ),+ "order={order:?}",+ );+ }+}++#[test]+fn undefined_logical_byte_array_order_is_not_a_bound() {+ let parquet_type = ParquetType::primitive_type_builder("s", PhysicalType::BYTE_ARRAY)+ .with_logical_type(Some(LogicalType::_Unknown { field_id: 100 }))+ .build()+ .unwrap();+ let statistics = ParquetStatistics::byte_array(+ Some(ByteArray::from("aé")),+ Some(ByteArray::from("b")),+ None,+ Some(0),+ false,+ );+ let metadata = single_column_metadata(+ parquet_type,+ statistics,+ Some(ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::UNDEFINED)),+ );+ let schema = Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, true)]));+ let file_statistics =+ DFParquetMetadata::statistics_from_parquet_metadata(&metadata, &schema).unwrap();+ assert_eq!(+ file_statistics.column_statistics[0].min_value,+ Precision::Absent+ );+ assert_eq!(+ file_statistics.column_statistics[0].max_value,+ Precision::Absent+ );+ assert_eq!(+ file_statistics.column_statistics[0].null_count,+ Precision::Exact(0)+ );+ let physical = logical2physical(&col("s").eq(lit("az")), &schema);+ let predicate = PruningPredicateBuilder::new()+ .with_file_schema(Arc::clone(&schema))+ .try_build(Arc::clone(&physical))+ .unwrap();+ let mut row_groups = RowGroupAccessPlanFilter::new(ParquetAccessPlan::new_all(1));+ row_groups.prune_by_statistics_with_metadata(+ &schema,+ &metadata,+ &predicate,+ &metrics(),+ );+ assert!(row_groups.build().should_scan(0));+ let pages = PagePruningAccessPlanFilter::new(&physical, Arc::clone(&schema))+ .prune_plan_with_page_index(+ ParquetAccessPlan::new_all(1),+ &schema,+ metadata.file_metadata().schema_descr(),+ &metadata,+ &metrics(),+ );+ assert!(pages.should_scan(0));+}