Skip to content

Apache DataFusion SQL Query Engine

perf: don't build a parquet stream for a file range with no row group left (#26476)

Braedon Woodingc0ec5127ac99
diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rs
index b048a23eea..3a1766869d 100644
--- a/datafusion/datasource-parquet/src/opener/mod.rs
+++ b/datafusion/datasource-parquet/src/opener/mod.rs
@@ -660,6 +660,9 @@ impl ParquetOpenState {
}
ParquetOpenState::PruneWithStatistics(prepared) => {
let mut prepared_row_groups = (*prepared).prune_row_groups()?;
+ if prepared_row_groups.row_groups.is_fully_pruned() {
+ return Ok(ParquetOpenState::Done);
+ }
if prepared_row_groups.should_load_page_index()? {
Ok(ParquetOpenState::LoadPageIndex(
prepared_row_groups.load_page_index().boxed(),
@@ -690,9 +693,13 @@ impl ParquetOpenState {
ParquetOpenState::LoadBloomFilters(future) => {
Ok(ParquetOpenState::LoadBloomFilters(future))
}
- ParquetOpenState::PruneWithBloomFilters(loaded) => Ok(
- ParquetOpenState::BuildStream(Box::new(loaded.prune_bloom_filters())),
- ),
+ ParquetOpenState::PruneWithBloomFilters(loaded) => {
+ let pruned = loaded.prune_bloom_filters();
+ if pruned.row_groups.is_fully_pruned() {
+ return Ok(ParquetOpenState::Done);
+ }
+ Ok(ParquetOpenState::BuildStream(Box::new(pruned)))
+ }
ParquetOpenState::BuildStream(prepared) => {
Ok(ParquetOpenState::Ready(prepared.build_stream()?))
}
@@ -2690,6 +2697,26 @@ mod test {
}
}
+ /// Drives a [`ParquetMorselizer`] to completion and returns how many
+ /// morsels it produced.
+ async fn count_morsels(
+ morselizer: &dyn Morselizer,
+ file: PartitionedFile,
+ ) -> Result<usize> {
+ let mut planners = VecDeque::from([morselizer.plan_file(file)?]);
+ let mut morsels = 0;
+ while let Some(planner) = planners.pop_front() {
+ if let Some(mut plan) = planner.plan()? {
+ morsels += plan.take_morsels().len();
+ planners.extend(plan.take_ready_planners());
+ if let Some(pending_planner) = plan.take_pending_planner() {
+ planners.push_front(pending_planner.await?);
+ }
+ }
+ }
+ Ok(morsels)
+ }
+
fn constant_int_stats() -> (Statistics, SchemaRef) {
let schema = Arc::new(Schema::new(vec![
Field::new("a", DataType::Int32, false),
@@ -3234,6 +3261,47 @@ mod test {
assert_eq!(num_rows, 0);
}
+ #[tokio::test]
+ async fn test_no_morsel_when_every_row_group_is_pruned() {
+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+ let batches = vec![
+ record_batch!(("a", Int32, vec![0, 2, 4])).unwrap(),
+ record_batch!(("a", Int32, vec![0, 2, 4])).unwrap(),
+ ];
+ let schema = batches[0].schema();
+ let props = WriterProperties::builder()
+ .set_max_row_group_row_count(Some(3))
+ .set_bloom_filter_enabled(true)
+ .set_statistics_enabled(EnabledStatistics::Chunk)
+ .build();
+ let data_size = write_parquet_batches(
+ Arc::clone(&store),
+ "test.parquet",
+ batches,
+ Some(props),
+ )
+ .await;
+ let file = PartitionedFile::new("test.parquet".to_string(), data_size as u64);
+
+ let morsels = |expr: Expr| {
+ let morselizer = ParquetMorselizerBuilder::new()
+ .with_store(Arc::clone(&store))
+ .with_schema(Arc::clone(&schema))
+ .with_predicate(logical2physical(&expr, &schema))
+ .with_row_group_stats_pruning(true)
+ .with_enable_bloom_filter(true)
+ .build();
+ let file = file.clone();
+ async move { count_morsels(&morselizer, file).await.unwrap() }
+ };
+
+ // Statistics prune both row groups.
+ assert_eq!(morsels(col("a").gt(lit(10))).await, 0);
+ // Statistics keep both row groups; their Bloom filters prune both.
+ assert_eq!(morsels(col("a").eq(lit(1))).await, 0);
+ assert_eq!(morsels(col("a").eq(lit(2))).await, 1);
+ }
+
#[tokio::test]
async fn test_prune_on_partition_statistics_with_dynamic_expression() {
let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
diff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rs
index 02fc2700d1..feba768955 100644
--- a/datafusion/datasource-parquet/src/row_group_filter.rs
+++ b/datafusion/datasource-parquet/src/row_group_filter.rs
@@ -99,6 +99,12 @@ impl RowGroupAccessPlanFilter {
self.access_plan.is_empty()
}
+ /// Return true if no row groups remain to be scanned, either because the
+ /// file has none or because pruning has skipped every one of them.
+ pub fn is_fully_pruned(&self) -> bool {
+ self.access_plan.row_group_index_iter().next().is_none()
+ }
+
/// Return the number of row groups that are currently expected to be scanned
pub fn remaining_row_group_count(&self) -> usize {
self.access_plan.row_group_index_iter().count()
@@ -706,6 +712,25 @@ mod tests {
assert_eq!(filter.remaining_row_group_count(), 2);
}
+ #[test]
+ fn is_fully_pruned_only_when_every_row_group_is_skipped() {
+ let mut filter = RowGroupAccessPlanFilter::new(ParquetAccessPlan::new_all(2));
+ assert!(!filter.is_fully_pruned());
+
+ filter.access_plan.skip(0);
+ assert!(!filter.is_fully_pruned());
+
+ filter.access_plan.skip(1);
+ assert!(filter.is_fully_pruned());
+ // Skipped row groups are still part of the plan
+ assert!(!filter.is_empty());
+
+ assert!(
+ RowGroupAccessPlanFilter::new(ParquetAccessPlan::new_all(0))
+ .is_fully_pruned()
+ );
+ }
+
/// A row group with one column chunk of `size` bytes at the given offsets
fn row_group_at(
dictionary_page_offset: Option<i64>,