Apache DataFusion SQL Query Engine
[Parquet] Skip bloom filter predicate evaluation when no bloom filters are loaded (#24101)
Renan GEHANbbb552f71c93
diff --git a/datafusion/datasource-parquet/src/bloom_filter.rs b/datafusion/datasource-parquet/src/bloom_filter.rsindex 8e55c12ba0..2384f95df9 100644--- a/datafusion/datasource-parquet/src/bloom_filter.rs+++ b/datafusion/datasource-parquet/src/bloom_filter.rs@@ -61,6 +61,11 @@ impl BloomFilterStatistics {}}+ /// Returns true if no per-column Bloom filters have been loaded.+ pub fn is_empty(&self) -> bool {+ self.column_sbbf.is_empty()+ }+/// Add a Bloom filter for the specified column, along with the column's/// Parquet physical [`Type`] and type length from the column descriptor.pub fn insert(diff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rsindex ddf71bb7e6..9231d02325 100644--- a/datafusion/datasource-parquet/src/row_group_filter.rs+++ b/datafusion/datasource-parquet/src/row_group_filter.rs@@ -434,6 +434,15 @@ impl RowGroupAccessPlanFilter {continue;}+ // A row group without any loaded bloom filter cannot be pruned by this pass: with no+ // bloom statistics to consult, evaluation can only conclude "may match". Skip the+ // evaluation in that case: it runs once per row group and can be expensive for wide+ // predicates, a cost files written without bloom filters would pay for nothing.+ if stats.is_empty() {+ metrics.row_groups_pruned_bloom_filter.add_matched(1);+ continue;+ }+// Can this group be pruned?let prune_group = match predicate.prune(stats) {Ok(values) => !values[0],@@ -544,6 +553,7 @@ mod tests {use parquet::arrow::ArrowSchemaConverter;use parquet::arrow::arrow_reader::{RowSelection, RowSelector};use parquet::basic::LogicalType;+ use parquet::bloom_filter::Sbbf;use parquet::data_type::{ByteArray, FixedLenByteArray};use parquet::file::metadata::ColumnChunkMetaData;use parquet::{@@ -1382,6 +1392,40 @@ mod tests {assert_pruned(row_groups, ExpectedPruning::Some(vec![1, 2]));}+ #[test]+ fn bloom_filter_pruning_evaluates_row_groups_with_bloom_filters() {+ // c1 = 15+ let schema =+ Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)]));+ let expr = logical2physical(&col("c1").eq(lit(15)), &schema);+ let pruning_predicate = PruningPredicateBuilder::new()+ .with_file_schema(schema)+ .build(expr)+ .unwrap();++ // Row group 0 has no bloom filter; row group 1 has a bloom filter+ // whose values do not include 15, so it can be pruned.+ let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap();+ sbbf.insert(&1_i32);+ sbbf.insert(&2_i32);+ let mut with_bloom_filter = BloomFilterStatistics::new();+ with_bloom_filter.insert("c1", sbbf, PhysicalType::INT32, 4);++ let metrics = parquet_file_metrics();+ let mut row_groups = RowGroupAccessPlanFilter::new(ParquetAccessPlan::new_all(2));+ row_groups.prune_by_bloom_filters(+ &pruning_predicate,+ &metrics,+ &[BloomFilterStatistics::new(), with_bloom_filter],+ );++ // The bloom-filter-less row group is kept without evaluation while+ // the row group with a bloom filter is still evaluated and pruned.+ assert_pruned(row_groups, ExpectedPruning::Some(vec![0]));+ assert_eq!(metrics.row_groups_pruned_bloom_filter.pruned(), 1);+ assert_eq!(metrics.row_groups_pruned_bloom_filter.matched(), 1);+ }+fn get_row_group_meta_data(schema_descr: &SchemaDescPtr,column_statistics: Vec<ParquetStatistics>,