Apache DataFusion SQL Query Engine
fix: prune row groups when file statistics collapse the predicate to a constant (#24770)
Jens Holdgaard Pedersenbe5a96e76df9
diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rsindex ec24462db0..d430d81e44 100644--- a/datafusion/datasource-parquet/src/opener/mod.rs+++ b/datafusion/datasource-parquet/src/opener/mod.rs@@ -58,8 +58,9 @@ use datafusion_common::{ColumnStatistics, HashSet, Result, ScalarValue, Statistics, exec_err, internal_err,};use datafusion_datasource::{PartitionedFile, TableSchema};-use datafusion_physical_expr::expressions::{Column, DynamicFilterTracking};+use datafusion_physical_expr::expressions::{Column, DynamicFilterTracking, Literal};use datafusion_physical_expr::simplifier::PhysicalExprSimplifier;+use datafusion_physical_expr::utils::collect_columns;use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use datafusion_physical_expr_common::sort_expr::LexOrdering;@@ -460,6 +461,27 @@ struct PreparedParquetOpen {enable_page_index: bool,enable_bloom_filter: bool,enable_row_group_stats_pruning: bool,+ /// True when substituting the columns `constant_columns_from_stats`+ /// proved constant for this file is on its own enough to collapse the+ /// predicate to a constant no row can satisfy — see `prune_row_groups`.+ ///+ /// Provisional until `prepare_filters` has the physical file schema: see+ /// `stats_constant_columns_in_predicate`.+ stats_prove_unsatisfiable: bool,+ /// The stats-derived constant columns the predicate actually references,+ /// recorded only when `stats_prove_unsatisfiable` is set.+ ///+ /// Collected file statistics cannot distinguish a column that is+ /// physically absent from the Parquet file from one that is present and+ /// all NULL: `statistics_from_parquet_metadata` documents that a column+ /// in the Arrow schema but not the Parquet schema gets+ /// `Precision::Exact(null)` min/max and an `Exact(num_rows)` null count —+ /// exactly what a present all-NULL column produces. So an absent column+ /// can enter the constants as a NULL and collapse the predicate, which+ /// would prune the very missing-column case this flag exists to leave+ /// alone. `prepare_filters` clears the flag once the physical schema+ /// shows whether these columns are really in the file.+ stats_constant_columns_in_predicate: Vec<String>,limit: Option<usize>,coerce_int96: Option<TimeUnit>,coerce_int96_tz: Option<Arc<str>>,@@ -794,10 +816,42 @@ impl ParquetMorselizer {// Note that if there are statistics for partition columns there will be overlap,// but since we use a HashMap, we'll just overwrite the partition values with the// constant values from statistics (which should be the same).- literal_columns.extend(constant_columns_from_stats(+ let stats_constants = constant_columns_from_stats(partitioned_file.statistics.as_deref(),&logical_file_schema,- ));+ );+ // Whether the statistics substitution *itself* proves the predicate+ // unsatisfiable has to be decided here, before the partition-value+ // and missing-column rewriting below: a predicate can collapse to a+ // constant through either of those too, and only a collapse the+ // file's own statistics produced is proof for `prune_row_groups`.+ // The referenced columns are recorded alongside the flag because this+ // is also too early to know whether they exist in the file at all —+ // `prepare_filters` re-checks them against the physical file schema.+ let stats_referenced: Vec<String> =+ match (&self.predicate, stats_constants.is_empty()) {+ (Some(predicate), false) => collect_columns(predicate)+ .iter()+ .filter(|c| stats_constants.contains_key(c.name()))+ .map(|c| c.name().to_string())+ .collect(),+ _ => Vec::new(),+ };+ // `stats_referenced` is the cheap short-circuit so the simplifier only+ // runs when a stats-derived constant is actually referenced.+ let stats_prove_unsatisfiable = !stats_referenced.is_empty()+ && self.predicate.as_ref().is_some_and(|p| {+ stats_alone_unsatisfiable(+ p,+ &stats_constants,+ self.table_schema.table_schema(),+ )+ });+ let stats_constant_columns_in_predicate = match stats_prove_unsatisfiable {+ true => stats_referenced,+ false => Vec::new(),+ };+ literal_columns.extend(stats_constants);let mut projection = self.projection.clone();let mut predicate = self.predicate.clone();@@ -867,6 +921,8 @@ impl ParquetMorselizer {enable_page_index: self.enable_page_index,enable_bloom_filter: self.enable_bloom_filter,enable_row_group_stats_pruning: self.enable_row_group_stats_pruning,+ stats_prove_unsatisfiable,+ stats_constant_columns_in_predicate,limit: self.limit,coerce_int96: self.coerce_int96,coerce_int96_tz: self.coerce_int96_tz.clone(),@@ -1073,6 +1129,24 @@ impl MetadataLoadedParquetOpen {}prepared.physical_file_schema = Arc::clone(&physical_file_schema);+ // The physical schema is only known here, so this is the first point+ // that can tell a stats-derived NULL constant for a column genuinely+ // present in the file from one synthesized for a column the file does+ // not have (see `stats_constant_columns_in_predicate`). Withdraw the+ // proof if any referenced column turns out to be absent: pruning on a+ // synthesized NULL would skip the missing-column case this flag is+ // meant to leave alone, and a custom `PhysicalExprAdapter` may fill a+ // missing column with a non-NULL default, so the skip could drop rows+ // that default would have matched.+ if prepared.stats_prove_unsatisfiable+ && prepared+ .stats_constant_columns_in_predicate+ .iter()+ .any(|name| physical_file_schema.field_with_name(name).is_err())+ {+ prepared.stats_prove_unsatisfiable = false;+ }+// Build predicates for this specific filelet pruning_predicate = build_pruning_predicates(prepared.predicate.as_ref(),@@ -1128,6 +1202,37 @@ impl FiltersPreparedParquetOpen {row_groups.prune_by_range(rg_metadata, range);}+ // Substituting the columns file statistics proved constant+ // (`constant_columns_from_stats`) collapsed the predicate to a+ // constant that can never be true (`false`, or NULL — filters treat+ // NULL as false), so the file's own statistics have proven that no+ // row can match: skip every remaining row group, credited to+ // statistics pruning. Without this, the substitution is strictly+ // counterproductive for such files: it *removes* the column+ // references the pruning predicate would need+ // (`build_pruning_predicates` returns `None` for a bare literal), so+ // a file that was previously pruned via its `null_count` statistics+ // is instead scanned in full.+ //+ // The flag is deliberately computed from the statistics substitution+ // alone (see `ParquetMorselizer::prepare`). A predicate can also+ // collapse via the missing-column adapter or partition-value+ // folding, and those paths keep their existing behaviour (files+ // pruned by partition values are `FilePruner`'s job); a mixed+ // predicate that only collapses once one of them has run is not+ // treated as proof here.+ if prepared.enable_row_group_stats_pruning && prepared.stats_prove_unsatisfiable {+ prepared+ .file_metrics+ .row_groups_pruned_statistics+ .add_pruned(row_groups.remaining_row_group_count());+ row_groups.skip_all();+ return Ok(RowGroupsPrunedParquetOpen {+ prepared: self,+ row_groups,+ });+ }+// If there is a predicate that can be evaluated against the metadataif let Some(predicate) = self.pruning_predicate.as_ref().map(|p| p.as_ref()) {if prepared.enable_row_group_stats_pruning {@@ -1720,6 +1825,38 @@ fn row_group_bytes(rg_meta: &RowGroupMetaData) -> u64 {type ConstantColumns = HashMap<String, ScalarValue>;+/// True when substituting `stats_constants` — the columns this file's+/// statistics prove constant — is on its own enough to collapse `predicate`+/// to a constant no row can satisfy (`false`, or NULL, which filters treat+/// as false).+///+/// Only the statistics substitution is applied, so a predicate that needs+/// the missing-column adapter or partition-value folding to collapse does+/// not qualify: those are separate proofs with their own pruning paths.+///+/// `schema` must resolve every column the predicate can reference — pass the+/// full table schema, since the simplifier types each node as it walks and+/// the substitution leaves non-constant columns in place. Substitution or+/// simplification failing is not proof, so both answer `false` and leave the+/// caller's normal pruning paths untouched.+fn stats_alone_unsatisfiable(+ predicate: &Arc<dyn PhysicalExpr>,+ stats_constants: &ConstantColumns,+ schema: &Schema,+) -> bool {+ let Ok(substituted) =+ replace_columns_with_literals(Arc::clone(predicate), stats_constants)+ else {+ return false;+ };+ let Ok(simplified) = PhysicalExprSimplifier::new(schema).simplify(substituted) else {+ return false;+ };+ simplified.downcast_ref::<Literal>().is_some_and(|l| {+ l.value().is_null() || l.value() == &ScalarValue::Boolean(Some(false))+ })+}+/// Extract constant column values from statistics, keyed by column name in the logical file schema.fn constant_columns_from_stats(statistics: Option<&Statistics>,@@ -1886,6 +2023,7 @@ async fn load_page_index<T: AsyncFileReader>(mod test {use super::*;use super::{ConstantColumns, ParquetMorselizer, constant_columns_from_stats};+ use crate::metadata::DFParquetMetadata;use crate::{CachedParquetFileReaderFactory, DefaultParquetFileReaderFactory,ParquetFileReaderFactory, ParquetRowSelection, RowGroupAccess,@@ -2512,6 +2650,43 @@ mod test {write_parquet_batches(store, filename, vec![batch], None).await}+ /// Write `batch` and return the statistics a collecting `ListingTable`+ /// would attach for `logical_file_schema`, produced by the same+ /// `statistics_from_parquet_metadata` the read path uses.+ ///+ /// Tests about schema evolution have to go through this rather than+ /// hand-build `Statistics`: the shape that function synthesizes for a+ /// logical column the file does not physically contain is the whole+ /// subject, so a hand-built approximation can silently stop matching it.+ async fn write_parquet_with_collected_statistics(+ store: Arc<dyn ObjectStore>,+ filename: &str,+ batch: RecordBatch,+ logical_file_schema: &SchemaRef,+ ) -> (usize, Statistics) {+ use parquet::file::metadata::ParquetMetaDataReader;++ let mut out = BytesMut::new().writer();+ {+ let mut writer =+ ArrowWriter::try_new(&mut out, batch.schema(), None).unwrap();+ writer.write(&batch).unwrap();+ writer.finish().unwrap();+ }+ let data = out.into_inner().freeze();+ let data_len = data.len();+ let metadata = ParquetMetaDataReader::new()+ .parse_and_finish(&data)+ .unwrap();+ let statistics = DFParquetMetadata::statistics_from_parquet_metadata(+ &metadata,+ logical_file_schema,+ )+ .unwrap();+ store.put(&Path::from(filename), data.into()).await.unwrap();+ (data_len, statistics)+ }+/// Write multiple batches to a parquet file with optional writer propertiesasync fn write_parquet_batches(store: Arc<dyn ObjectStore>,@@ -3040,6 +3215,302 @@ mod test {assert_eq!(num_rows, 0);}+ /// Row groups this scan skipped through statistics pruning.+ ///+ /// The row count alone cannot tell "pruned" from "scanned, then+ /// row-filtered" — both yield zero rows — so the tests around+ /// constant-column substitution assert on this metric instead.+ fn pruned_row_groups_statistics(metrics: &ExecutionPlanMetricsSet) -> usize {+ use datafusion_physical_plan::metrics::MetricValue;+ metrics+ .clone_inner()+ .iter()+ .find_map(|m| match m.value() {+ MetricValue::PruningMetrics {+ name,+ pruning_metrics,+ } if name == "row_groups_pruned_statistics" => {+ Some(pruning_metrics.pruned())+ }+ _ => None,+ })+ .expect("row_groups_pruned_statistics metric is emitted")+ }++ #[tokio::test]+ async fn test_prune_all_null_column_equality_from_file_statistics() {+ // Regression: a column whose file statistics say every value is+ // NULL cannot satisfy `col = <literal>`, so the file's row groups+ // must be pruned. Constant-column substitution folds the all-NULL+ // column to a NULL literal, collapsing the predicate to a+ // constant; the opener has to recognise that and skip the row+ // groups, rather than lose the pruning because the substituted+ // predicate no longer references any column.+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;++ let batch = record_batch!(+ ("a", Int32, vec![Some(1), Some(2), Some(3)]),+ ("b", Utf8, vec![None::<&str>, None, None])+ )+ .unwrap();+ let data_size =+ write_parquet(Arc::clone(&store), "file.parquet", batch.clone()).await;+ let file_schema = batch.schema();+ let mut file = PartitionedFile::new(+ "file.parquet".to_string(),+ u64::try_from(data_size).unwrap(),+ );+ // Statistics as a collecting `ListingTable` would supply them: the+ // `b` column's null count equals the row count, i.e. every value is+ // NULL. (`new_unknown` pre-fills one entry per field, so overwrite+ // rather than append.)+ let mut statistics = Statistics::new_unknown(&file_schema);+ statistics.num_rows = Precision::Exact(3);+ statistics.column_statistics[1].null_count = Precision::Exact(3);+ file.statistics = Some(Arc::new(statistics));+ let table_schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int32, false),+ Field::new("b", DataType::Utf8, true),+ ]));+ let table_schema_for_opener = TableSchemaBuilder::from(&file_schema).build();+ let make_opener = |predicate, metrics: ExecutionPlanMetricsSet| {+ ParquetMorselizerBuilder::new()+ .with_store(Arc::clone(&store))+ .with_table_schema(table_schema_for_opener.clone())+ .with_projection_indices(&[0])+ .with_predicate(predicate)+ .with_row_group_stats_pruning(true)+ .with_metrics(metrics)+ .build()+ };++ // `b = 'x'` cannot match: every `b` is NULL, so the file's one row+ // group is pruned from statistics rather than read and filtered.+ let metrics = ExecutionPlanMetricsSet::new();+ let expr = col("b").eq(lit("x"));+ let predicate = logical2physical(&expr, &table_schema);+ let opener = make_opener(predicate, metrics.clone());+ let stream = open_file(&opener, file.clone()).await.unwrap();+ let (_, num_rows) = count_batches_and_rows(stream).await;+ assert_eq!(num_rows, 0);+ assert_eq!(+ pruned_row_groups_statistics(&metrics),+ 1,+ "an all-NULL column cannot satisfy equality: the row group must be pruned"+ );++ // A predicate the statistics cannot disprove still reads the file,+ // so the skip above is not simply "prune everything".+ let metrics = ExecutionPlanMetricsSet::new();+ let expr = col("a").eq(lit(2));+ let predicate = logical2physical(&expr, &table_schema);+ let opener = make_opener(predicate, metrics.clone());+ let stream = open_file(&opener, file).await.unwrap();+ let (num_batches, num_rows) = count_batches_and_rows(stream).await;+ assert_eq!(num_batches, 1);+ assert_eq!(num_rows, 3);+ assert_eq!(pruned_row_groups_statistics(&metrics), 0);+ }++ #[tokio::test]+ async fn test_prune_exact_constant_column_false_predicate_from_file_statistics() {+ // The `false` half of the branch above, which the all-NULL case+ // only covers for NULL: exact min == max statistics prove `a`+ // constant, so `a = <other literal>` folds to `false` and the row+ // group is pruned on that proof.+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;++ let batch = record_batch!(("a", Int32, vec![Some(7), Some(7), Some(7)])).unwrap();+ let data_size =+ write_parquet(Arc::clone(&store), "file.parquet", batch.clone()).await;+ let file_schema = batch.schema();+ let mut file = PartitionedFile::new(+ "file.parquet".to_string(),+ u64::try_from(data_size).unwrap(),+ );+ let mut statistics = Statistics::new_unknown(&file_schema);+ statistics.num_rows = Precision::Exact(3);+ statistics.column_statistics[0].null_count = Precision::Exact(0);+ statistics.column_statistics[0].min_value =+ Precision::Exact(ScalarValue::Int32(Some(7)));+ statistics.column_statistics[0].max_value =+ Precision::Exact(ScalarValue::Int32(Some(7)));+ file.statistics = Some(Arc::new(statistics));+ let table_schema =+ Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));+ let table_schema_for_opener = TableSchemaBuilder::from(&file_schema).build();++ let metrics = ExecutionPlanMetricsSet::new();+ let expr = col("a").eq(lit(8));+ let predicate = logical2physical(&expr, &table_schema);+ let opener = ParquetMorselizerBuilder::new()+ .with_store(Arc::clone(&store))+ .with_table_schema(table_schema_for_opener)+ .with_projection_indices(&[0])+ .with_predicate(predicate)+ .with_row_group_stats_pruning(true)+ .with_metrics(metrics.clone())+ .build();+ let stream = open_file(&opener, file).await.unwrap();+ let (_, num_rows) = count_batches_and_rows(stream).await;+ assert_eq!(num_rows, 0);+ assert_eq!(+ pruned_row_groups_statistics(&metrics),+ 1,+ "a column statistics prove constant cannot equal a different literal: \+ the row group must be pruned"+ );+ }++ #[tokio::test]+ async fn test_no_prune_when_missing_column_collapses_mixed_predicate() {+ // A stats-derived constant appearing in the predicate is not proof+ // that the statistics are what made it unsatisfiable. Here `a = 1`+ // is proven true by statistics and folds away, and the collapse to+ // NULL comes from `b = 2` — a column this file does not have. That+ // is the schema-evolution path, which deliberately scans and+ // row-filters rather than pruning, so the statistics branch must not+ // fire.+ //+ // The statistics come from `statistics_from_parquet_metadata` rather+ // than being hand-built, because the shape that matters is the one it+ // synthesizes for a missing column: `Exact(null)` min/max and an+ // `Exact(num_rows)` null count, indistinguishable from a present+ // all-NULL column. Hand-building anything weaker (leaving `b`+ // unknown, say) stops `b` from ever becoming a constant, and the test+ // then passes without exercising this path at all.+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;++ // The file has only `a`; `b` is in the table schema but missing+ // from this file.+ let batch = record_batch!(("a", Int32, vec![Some(1), Some(1), Some(1)])).unwrap();+ let physical_schema = batch.schema();+ let logical_file_schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int32, false),+ Field::new("b", DataType::Int32, true),+ ]));+ let (data_size, statistics) = write_parquet_with_collected_statistics(+ Arc::clone(&store),+ "file.parquet",+ batch,+ &logical_file_schema,+ )+ .await;+ let mut file = PartitionedFile::new(+ "file.parquet".to_string(),+ u64::try_from(data_size).unwrap(),+ );+ // Assert the premise instead of trusting it: if collection ever stops+ // representing the absent `b` as an all-NULL column, this test is no+ // longer covering the case it is named for and should say so loudly.+ assert_eq!(+ statistics.column_statistics[1].null_count,+ Precision::Exact(3),+ "collection must still report the absent column as all NULL for \+ this test to exercise the missing-column path"+ );+ file.statistics = Some(Arc::new(statistics));+ let table_schema_for_opener =+ TableSchemaBuilder::from(&logical_file_schema).build();++ let metrics = ExecutionPlanMetricsSet::new();+ let expr = col("a").eq(lit(1)).and(col("b").eq(lit(2)));+ let predicate = logical2physical(&expr, &logical_file_schema);+ let opener = ParquetMorselizerBuilder::new()+ .with_store(Arc::clone(&store))+ .with_table_schema(table_schema_for_opener)+ .with_projection_indices(&[0])+ .with_predicate(predicate)+ .with_row_group_stats_pruning(true)+ .with_metrics(metrics.clone())+ .build();+ let stream = open_file(&opener, file).await.unwrap();+ let (_, num_rows) = count_batches_and_rows(stream).await;+ // The row group is read and its rows handed up for the filter above+ // to apply, which is the missing-column path this test protects.+ assert_eq!(+ num_rows, 3,+ "the file must be scanned, not pruned, on the missing-column path"+ );+ assert_eq!(+ pruned_row_groups_statistics(&metrics),+ 0,+ "the collapse came from the missing column, not from file \+ statistics: the statistics branch must not claim the prune"+ );+ // Guard against the file schema drifting: `b` really is absent.+ assert!(physical_schema.field_with_name("b").is_err());+ }++ #[tokio::test]+ async fn test_prune_when_present_all_null_column_collapses_mixed_predicate() {+ // The mirror of the test above, and the reason its fix cannot simply+ // be "never trust a NULL constant". Identical predicate, identical+ // statistics source, one difference: `b` is physically in the file+ // and genuinely all NULL. The collapse is then real proof about this+ // file's contents, so the row group must still be pruned.+ //+ // Collected statistics describe these two files identically — an+ // `Exact(null)` min/max and an `Exact(num_rows)` null count for `b`+ // either way — which is exactly why the decision has to wait for the+ // physical file schema.+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;++ let batch = record_batch!(+ ("a", Int32, vec![Some(1), Some(1), Some(1)]),+ ("b", Int32, vec![None::<i32>, None, None])+ )+ .unwrap();+ let physical_schema = batch.schema();+ let logical_file_schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int32, false),+ Field::new("b", DataType::Int32, true),+ ]));+ let (data_size, statistics) = write_parquet_with_collected_statistics(+ Arc::clone(&store),+ "file.parquet",+ batch,+ &logical_file_schema,+ )+ .await;+ let mut file = PartitionedFile::new(+ "file.parquet".to_string(),+ u64::try_from(data_size).unwrap(),+ );+ assert_eq!(+ statistics.column_statistics[1].null_count,+ Precision::Exact(3),+ "the present `b` must collect as all NULL, so that it is \+ indistinguishable from the absent `b` in the test above"+ );+ file.statistics = Some(Arc::new(statistics));+ let table_schema_for_opener =+ TableSchemaBuilder::from(&logical_file_schema).build();++ let metrics = ExecutionPlanMetricsSet::new();+ let expr = col("a").eq(lit(1)).and(col("b").eq(lit(2)));+ let predicate = logical2physical(&expr, &logical_file_schema);+ let opener = ParquetMorselizerBuilder::new()+ .with_store(Arc::clone(&store))+ .with_table_schema(table_schema_for_opener)+ .with_projection_indices(&[0])+ .with_predicate(predicate)+ .with_row_group_stats_pruning(true)+ .with_metrics(metrics.clone())+ .build();+ let stream = open_file(&opener, file).await.unwrap();+ let (_, num_rows) = count_batches_and_rows(stream).await;+ assert_eq!(num_rows, 0);+ assert_eq!(+ pruned_row_groups_statistics(&metrics),+ 1,+ "the all-NULL column is really in the file: the statistics do \+ prove the predicate unsatisfiable and must prune"+ );+ // Guard against the file schema drifting: `b` really is present.+ assert!(physical_schema.field_with_name("b").is_ok());+ }+#[tokio::test]async fn test_prune_on_partition_value_and_data_value() {let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;diff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rsindex ab690e345d..7d4048df75 100644--- a/datafusion/datasource-parquet/src/row_group_filter.rs+++ b/datafusion/datasource-parquet/src/row_group_filter.rs@@ -69,6 +69,17 @@ impl RowGroupAccessPlanFilter {Self { access_plan }}+ /// Skip every remaining row group: used when the predicate has been+ /// proven unsatisfiable for the whole file (for example, when+ /// constant-column substitution from file statistics collapsed it to a+ /// constant `false`/NULL), so no row group can contain a matching row.+ pub fn skip_all(&mut self) {+ let indexes: Vec<usize> = self.access_plan.row_group_indexes();+ for idx in indexes {+ self.access_plan.skip(idx);+ }+ }+/// Return true if there are no row groupspub fn is_empty(&self) -> bool {self.access_plan.is_empty()