Apache DataFusion SQL Query Engine
Conditionally build page pruning predicates (#21480)
Filip Petkovskiec001124829a
diff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rsindex 35900e16c1..5307cb15ee 100644--- a/datafusion/datasource-parquet/src/opener.rs+++ b/datafusion/datasource-parquet/src/opener.rs@@ -920,12 +920,22 @@ impl MetadataLoadedParquetOpen {prepared.physical_file_schema = Arc::clone(&physical_file_schema);// Build predicates for this specific file- let (pruning_predicate, page_pruning_predicate) = build_pruning_predicates(+ let pruning_predicate = build_pruning_predicates(prepared.predicate.as_ref(),&physical_file_schema,&prepared.predicate_creation_errors,);+ // Only build page pruning predicate if page index is enabled+ let page_pruning_predicate = if prepared.enable_page_index {+ prepared.predicate.as_ref().and_then(|predicate| {+ let p = build_page_pruning_predicate(predicate, &physical_file_schema);+ (p.filter_number() > 0).then_some(p)+ })+ } else {+ None+ };+Ok(FiltersPreparedParquetOpen {loaded: MetadataLoadedParquetOpen {prepared,@@ -945,10 +955,7 @@ impl FiltersPreparedParquetOpen {// metadata load above may not have read the page index structures yet.// If we need them for reading and they aren't yet loaded, we need to// load them now.- if should_enable_page_index(- self.loaded.prepared.enable_page_index,- &self.page_pruning_predicate,- ) {+ if self.page_pruning_predicate.is_some() {self.loaded.reader_metadata = load_page_index(self.loaded.reader_metadata,&mut self.loaded.prepared.async_file_reader,@@ -1661,20 +1668,13 @@ pub(crate) fn build_pruning_predicates(predicate: Option<&Arc<dyn PhysicalExpr>>,file_schema: &SchemaRef,predicate_creation_errors: &Count,-) -> (- Option<Arc<PruningPredicate>>,- Option<Arc<PagePruningAccessPlanFilter>>,-) {- let Some(predicate) = predicate.as_ref() else {- return (None, None);- };- let pruning_predicate = build_pruning_predicate(+) -> Option<Arc<PruningPredicate>> {+ let predicate = predicate.as_ref()?;+ build_pruning_predicate(Arc::clone(predicate),file_schema,predicate_creation_errors,- );- let page_pruning_predicate = build_page_pruning_predicate(predicate, file_schema);- (pruning_predicate, Some(page_pruning_predicate))+ )}/// Returns a `ArrowReaderMetadata` with the page index loaded, loading@@ -1708,18 +1708,6 @@ async fn load_page_index<T: AsyncFileReader>(}}-fn should_enable_page_index(- enable_page_index: bool,- page_pruning_predicate: &Option<Arc<PagePruningAccessPlanFilter>>,-) -> bool {- enable_page_index- && page_pruning_predicate.is_some()- && page_pruning_predicate- .as_ref()- .map(|p| p.filter_number() > 0)- .unwrap_or(false)-}-#[cfg(test)]mod test {use std::sync::Arc;@@ -1849,6 +1837,12 @@ mod test {self}+ /// Enable page index.+ fn with_enable_page_index(mut self, enable: bool) -> Self {+ self.enable_page_index = enable;+ self+ }+/// Set reverse row groups flag.fn with_reverse_row_groups(mut self, enable: bool) -> Self {self.reverse_row_groups = enable;@@ -2713,4 +2707,71 @@ mod test {"Reverse scan with non-contiguous row groups should correctly map RowSelection");}++ /// Test that page pruning predicates are only built and applied when `enable_page_index` is true.+ ///+ /// The file has a single row group with 10 pages (10 rows each, values 1..100).+ /// With page index enabled, pages whose max value <= 90 are pruned, returning only+ /// the last page (rows 91..100). With page index disabled, all 100 rows are returned+ /// since neither pushdown nor row-group pruning is active.+ #[tokio::test]+ async fn test_page_pruning_predicate_respects_enable_page_index() {+ use parquet::file::properties::WriterProperties;++ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;++ // 100 rows with values 1..=100, written as a single row group with 10 rows per page+ let values: Vec<i32> = (1..=100).collect();+ let batch = record_batch!((+ "a",+ Int32,+ values.iter().map(|v| Some(*v)).collect::<Vec<_>>()+ ))+ .unwrap();+ let props = WriterProperties::builder()+ .set_data_page_row_count_limit(10)+ .set_write_batch_size(10)+ .build();+ let schema = batch.schema();+ let data_size = write_parquet_batches(+ Arc::clone(&store),+ "test.parquet",+ vec![batch],+ Some(props),+ )+ .await;++ let file = PartitionedFile::new("test.parquet".to_string(), data_size as u64);++ // predicate: a > 90 — should allow page index to prune first 9 pages+ let predicate = logical2physical(&col("a").gt(lit(90i32)), &schema);++ let make_opener = |enable_page_index| {+ ParquetOpenerBuilder::new()+ .with_store(Arc::clone(&store))+ .with_schema(Arc::clone(&schema))+ .with_predicate(Arc::clone(&predicate))+ .with_enable_page_index(enable_page_index)+ // disable pushdown and row-group pruning so the only pruning path is page index+ .with_pushdown_filters(false)+ .with_row_group_stats_pruning(false)+ .build()+ };+ let (_, rows_with_page_index) = count_batches_and_rows(+ make_opener(true).open(file.clone()).unwrap().await.unwrap(),+ )+ .await;+ let (_, rows_without_page_index) =+ count_batches_and_rows(make_opener(false).open(file).unwrap().await.unwrap())+ .await;++ assert_eq!(+ rows_with_page_index, 10,+ "page index should prune 9 of 10 pages"+ );+ assert_eq!(+ rows_without_page_index, 100,+ "without page index all rows are returned"+ );+ }}diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rsindex 1e54e98dfd..7fad6862fa 100644--- a/datafusion/datasource-parquet/src/source.rs+++ b/datafusion/datasource-parquet/src/source.rs@@ -629,17 +629,17 @@ impl FileSource for ParquetSource {write!(f, ", reverse_row_groups=true")?;}- // Try to build a the pruning predicates.+ // Try to build the pruning predicates.// These are only generated here because it's useful to have *some*// idea of what pushdown is happening when viewing plans.- // However it is important to note that these predicates are *not*+ // However, it is important to note that these predicates are *not*// necessarily the predicates that are actually evaluated:// the actual predicates are built in reference to the physical schema of// each file, which we do not have at this point and hence cannot use.- // Instead we use the logical schema of the file (the table schema without partition columns).+ // Instead, we use the logical schema of the file (the table schema without partition columns).if let Some(predicate) = &self.predicate {let predicate_creation_errors = Count::new();- if let (Some(pruning_predicate), _) = build_pruning_predicates(+ if let Some(pruning_predicate) = build_pruning_predicates(Some(predicate),self.table_schema.table_schema(),&predicate_creation_errors,