Apache DataFusion SQL Query Engine
perf: don't build a parquet stream for a file range with no row group left (#26476)
Braedon Woodingc0ec5127ac99
diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rsindex b048a23eea..3a1766869d 100644--- a/datafusion/datasource-parquet/src/opener/mod.rs+++ b/datafusion/datasource-parquet/src/opener/mod.rs@@ -660,6 +660,9 @@ impl ParquetOpenState {}ParquetOpenState::PruneWithStatistics(prepared) => {let mut prepared_row_groups = (*prepared).prune_row_groups()?;+ if prepared_row_groups.row_groups.is_fully_pruned() {+ return Ok(ParquetOpenState::Done);+ }if prepared_row_groups.should_load_page_index()? {Ok(ParquetOpenState::LoadPageIndex(prepared_row_groups.load_page_index().boxed(),@@ -690,9 +693,13 @@ impl ParquetOpenState {ParquetOpenState::LoadBloomFilters(future) => {Ok(ParquetOpenState::LoadBloomFilters(future))}- ParquetOpenState::PruneWithBloomFilters(loaded) => Ok(- ParquetOpenState::BuildStream(Box::new(loaded.prune_bloom_filters())),- ),+ ParquetOpenState::PruneWithBloomFilters(loaded) => {+ let pruned = loaded.prune_bloom_filters();+ if pruned.row_groups.is_fully_pruned() {+ return Ok(ParquetOpenState::Done);+ }+ Ok(ParquetOpenState::BuildStream(Box::new(pruned)))+ }ParquetOpenState::BuildStream(prepared) => {Ok(ParquetOpenState::Ready(prepared.build_stream()?))}@@ -2690,6 +2697,26 @@ mod test {}}+ /// Drives a [`ParquetMorselizer`] to completion and returns how many+ /// morsels it produced.+ async fn count_morsels(+ morselizer: &dyn Morselizer,+ file: PartitionedFile,+ ) -> Result<usize> {+ let mut planners = VecDeque::from([morselizer.plan_file(file)?]);+ let mut morsels = 0;+ while let Some(planner) = planners.pop_front() {+ if let Some(mut plan) = planner.plan()? {+ morsels += plan.take_morsels().len();+ planners.extend(plan.take_ready_planners());+ if let Some(pending_planner) = plan.take_pending_planner() {+ planners.push_front(pending_planner.await?);+ }+ }+ }+ Ok(morsels)+ }+fn constant_int_stats() -> (Statistics, SchemaRef) {let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false),@@ -3234,6 +3261,47 @@ mod test {assert_eq!(num_rows, 0);}+ #[tokio::test]+ async fn test_no_morsel_when_every_row_group_is_pruned() {+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;+ let batches = vec![+ record_batch!(("a", Int32, vec![0, 2, 4])).unwrap(),+ record_batch!(("a", Int32, vec![0, 2, 4])).unwrap(),+ ];+ let schema = batches[0].schema();+ let props = WriterProperties::builder()+ .set_max_row_group_row_count(Some(3))+ .set_bloom_filter_enabled(true)+ .set_statistics_enabled(EnabledStatistics::Chunk)+ .build();+ let data_size = write_parquet_batches(+ Arc::clone(&store),+ "test.parquet",+ batches,+ Some(props),+ )+ .await;+ let file = PartitionedFile::new("test.parquet".to_string(), data_size as u64);++ let morsels = |expr: Expr| {+ let morselizer = ParquetMorselizerBuilder::new()+ .with_store(Arc::clone(&store))+ .with_schema(Arc::clone(&schema))+ .with_predicate(logical2physical(&expr, &schema))+ .with_row_group_stats_pruning(true)+ .with_enable_bloom_filter(true)+ .build();+ let file = file.clone();+ async move { count_morsels(&morselizer, file).await.unwrap() }+ };++ // Statistics prune both row groups.+ assert_eq!(morsels(col("a").gt(lit(10))).await, 0);+ // Statistics keep both row groups; their Bloom filters prune both.+ assert_eq!(morsels(col("a").eq(lit(1))).await, 0);+ assert_eq!(morsels(col("a").eq(lit(2))).await, 1);+ }+#[tokio::test]async fn test_prune_on_partition_statistics_with_dynamic_expression() {let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;diff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rsindex 02fc2700d1..feba768955 100644--- a/datafusion/datasource-parquet/src/row_group_filter.rs+++ b/datafusion/datasource-parquet/src/row_group_filter.rs@@ -99,6 +99,12 @@ impl RowGroupAccessPlanFilter {self.access_plan.is_empty()}+ /// Return true if no row groups remain to be scanned, either because the+ /// file has none or because pruning has skipped every one of them.+ pub fn is_fully_pruned(&self) -> bool {+ self.access_plan.row_group_index_iter().next().is_none()+ }+/// Return the number of row groups that are currently expected to be scannedpub fn remaining_row_group_count(&self) -> usize {self.access_plan.row_group_index_iter().count()@@ -706,6 +712,25 @@ mod tests {assert_eq!(filter.remaining_row_group_count(), 2);}+ #[test]+ fn is_fully_pruned_only_when_every_row_group_is_skipped() {+ let mut filter = RowGroupAccessPlanFilter::new(ParquetAccessPlan::new_all(2));+ assert!(!filter.is_fully_pruned());++ filter.access_plan.skip(0);+ assert!(!filter.is_fully_pruned());++ filter.access_plan.skip(1);+ assert!(filter.is_fully_pruned());+ // Skipped row groups are still part of the plan+ assert!(!filter.is_empty());++ assert!(+ RowGroupAccessPlanFilter::new(ParquetAccessPlan::new_all(0))+ .is_fully_pruned()+ );+ }+/// A row group with one column chunk of `size` bytes at the given offsetsfn row_group_at(dictionary_page_offset: Option<i64>,