Apache DataFusion SQL Query Engine
feat: Better parquet row-group/page pruning metrics display (#18321)
Yongting You9e9eb944c062
diff --git a/datafusion-examples/examples/json_shredding.rs b/datafusion-examples/examples/json_shredding.rsindex a2e83bc951..5ef8b59b64 100644--- a/datafusion-examples/examples/json_shredding.rs+++ b/datafusion-examples/examples/json_shredding.rs@@ -142,7 +142,7 @@ async fn main() -> Result<()> {.await?;let plan = format!("{}", arrow::util::pretty::pretty_format_batches(&batches)?);println!("{plan}");- assert_contains!(&plan, "row_groups_pruned_statistics=1");+ assert_contains!(&plan, "row_groups_pruned_statistics=2 total → 1 matched");assert_contains!(&plan, "pushdown_rows_pruned=1");Ok(())diff --git a/datafusion/core/src/datasource/physical_plan/parquet.rs b/datafusion/core/src/datasource/physical_plan/parquet.rsindex 18b855cec5..0ffb252a66 100644--- a/datafusion/core/src/datasource/physical_plan/parquet.rs+++ b/datafusion/core/src/datasource/physical_plan/parquet.rs@@ -65,7 +65,7 @@ mod tests {use datafusion_physical_plan::analyze::AnalyzeExec;use datafusion_physical_plan::collect;use datafusion_physical_plan::metrics::{- ExecutionPlanMetricsSet, MetricType, MetricsSet,+ ExecutionPlanMetricsSet, MetricType, MetricValue, MetricsSet,};use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties};@@ -1175,8 +1175,10 @@ mod tests {// There are 4 rows pruned in each of batch2, batch3, and// batch4 for a total of 12. batch1 had no pruning as c2 was// filled in as null- assert_eq!(get_value(&metrics, "page_index_rows_pruned"), 12);- assert_eq!(get_value(&metrics, "page_index_rows_matched"), 6);+ let (page_index_pruned, page_index_matched) =+ get_pruning_metric(&metrics, "page_index_rows_pruned");+ assert_eq!(page_index_pruned, 12);+ assert_eq!(page_index_matched, 6);}#[tokio::test]@@ -1776,8 +1778,10 @@ mod tests {| 5 |+-----+"###);- assert_eq!(get_value(&metrics, "page_index_rows_pruned"), 4);- assert_eq!(get_value(&metrics, "page_index_rows_matched"), 2);+ let (page_index_pruned, page_index_matched) =+ get_pruning_metric(&metrics, "page_index_rows_pruned");+ assert_eq!(page_index_pruned, 4);+ assert_eq!(page_index_matched, 2);assert!(get_value(&metrics, "page_index_eval_time") > 0,"no eval time in metrics: {metrics:#?}"@@ -1866,8 +1870,10 @@ mod tests {assert_contains!(&explain, "predicate=c1@0 != bar");// there's a single row group, but we can check that it matched- // if no pruning was done this would be 0 instead of 1- assert_contains!(&explain, "row_groups_matched_statistics=1");+ assert_contains!(+ &explain,+ "row_groups_pruned_statistics=1 total \u{2192} 1 matched"+ );// check the projectionassert_contains!(&explain, "projection=[c1]");@@ -1898,8 +1904,10 @@ mod tests {// When both matched and pruned are 0, it means that the pruning predicate// was not used at all.- assert_contains!(&explain, "row_groups_matched_statistics=0");- assert_contains!(&explain, "row_groups_pruned_statistics=0");+ assert_contains!(+ &explain,+ "row_groups_pruned_statistics=1 total \u{2192} 1 matched"+ );// But pushdown predicate should be presentassert_contains!(@@ -1952,7 +1960,12 @@ mod tests {/// Panics if no such metric.fn get_value(metrics: &MetricsSet, metric_name: &str) -> usize {match metrics.sum_by_name(metric_name) {- Some(v) => v.as_usize(),+ Some(v) => match v {+ MetricValue::PruningMetrics {+ pruning_metrics, ..+ } => pruning_metrics.pruned(),+ _ => v.as_usize(),+ },_ => {panic!("Expected metric not found. Looking for '{metric_name}' in\n\n{metrics:#?}"@@ -1961,6 +1974,20 @@ mod tests {}}+ fn get_pruning_metric(metrics: &MetricsSet, metric_name: &str) -> (usize, usize) {+ match metrics.sum_by_name(metric_name) {+ Some(MetricValue::PruningMetrics {+ pruning_metrics, ..+ }) => (pruning_metrics.pruned(), pruning_metrics.matched()),+ Some(_) => panic!(+ "Metric '{metric_name}' is not a pruning metric in\n\n{metrics:#?}"+ ),+ None => panic!(+ "Expected metric not found. Looking for '{metric_name}' in\n\n{metrics:#?}"+ ),+ }+ }+fn populate_csv_partitions(tmp_dir: &TempDir,partition_count: usize,diff --git a/datafusion/core/tests/parquet/external_access_plan.rs b/datafusion/core/tests/parquet/external_access_plan.rsindex a5397c5a39..5135f95685 100644--- a/datafusion/core/tests/parquet/external_access_plan.rs+++ b/datafusion/core/tests/parquet/external_access_plan.rs@@ -33,7 +33,7 @@ use datafusion_common::{assert_contains, DFSchema};use datafusion_datasource_parquet::{ParquetAccessPlan, RowGroupAccess};use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_expr::{col, lit, Expr};-use datafusion_physical_plan::metrics::MetricsSet;+use datafusion_physical_plan::metrics::{MetricValue, MetricsSet};use datafusion_physical_plan::ExecutionPlan;use datafusion_datasource::file_scan_config::FileScanConfigBuilder;@@ -178,12 +178,21 @@ async fn plan_and_filter() {.unwrap();// Verify that row group pruning still happens for just that group- let row_groups_pruned_statistics =- metric_value(&parquet_metrics, "row_groups_pruned_statistics").unwrap();- assert_eq!(- row_groups_pruned_statistics, 1,- "metrics : {parquet_metrics:#?}",- );+ let row_groups_pruned_statistics = parquet_metrics+ .sum_by_name("row_groups_pruned_statistics")+ .unwrap();+ if let MetricValue::PruningMetrics {+ pruning_metrics, ..+ } = row_groups_pruned_statistics+ {+ assert_eq!(+ pruning_metrics.pruned(),+ 1,+ "metrics : {parquet_metrics:#?}",+ );+ } else {+ unreachable!("metrics `row_groups_pruned_statistics` should exist")+ }}#[tokio::test]diff --git a/datafusion/core/tests/parquet/filter_pushdown.rs b/datafusion/core/tests/parquet/filter_pushdown.rsindex 226497fe58..966f251613 100644--- a/datafusion/core/tests/parquet/filter_pushdown.rs+++ b/datafusion/core/tests/parquet/filter_pushdown.rs@@ -29,7 +29,7 @@use arrow::compute::concat_batches;use arrow::record_batch::RecordBatch;use datafusion::physical_plan::collect;-use datafusion::physical_plan::metrics::MetricsSet;+use datafusion::physical_plan::metrics::{MetricValue, MetricsSet};use datafusion::prelude::{col, lit, lit_timestamp_nano, Expr, ParquetReadOptions, SessionContext,};@@ -563,9 +563,9 @@ impl<'a> TestCase<'a> {}};- let page_index_rows_pruned = get_value(&metrics, "page_index_rows_pruned");+ let (page_index_rows_pruned, page_index_rows_matched) =+ get_pruning_metrics(&metrics, "page_index_rows_pruned");println!(" page_index_rows_pruned: {page_index_rows_pruned}");- let page_index_rows_matched = get_value(&metrics, "page_index_rows_matched");println!(" page_index_rows_matched: {page_index_rows_matched}");let page_index_filtering_expected = if scan_options.enable_page_index {@@ -592,14 +592,29 @@ impl<'a> TestCase<'a> {}}+fn get_pruning_metrics(metrics: &MetricsSet, metric_name: &str) -> (usize, usize) {+ match metrics.sum_by_name(metric_name) {+ Some(MetricValue::PruningMetrics {+ pruning_metrics, ..+ }) => (pruning_metrics.pruned(), pruning_metrics.matched()),+ Some(_) => {+ panic!("Metric '{metric_name}' is not a pruning metric in\n\n{metrics:#?}")+ }+ None => panic!(+ "Expected metric not found. Looking for '{metric_name}' in\n\n{metrics:#?}"+ ),+ }+}+fn get_value(metrics: &MetricsSet, metric_name: &str) -> usize {match metrics.sum_by_name(metric_name) {+ Some(MetricValue::PruningMetrics {+ pruning_metrics, ..+ }) => pruning_metrics.pruned(),Some(v) => v.as_usize(),- _ => {- panic!(- "Expected metric not found. Looking for '{metric_name}' in\n\n{metrics:#?}"- );- }+ None => panic!(+ "Expected metric not found. Looking for '{metric_name}' in\n\n{metrics:#?}"+ ),}}diff --git a/datafusion/core/tests/parquet/mod.rs b/datafusion/core/tests/parquet/mod.rsindex 34a48cdae3..097600e45e 100644--- a/datafusion/core/tests/parquet/mod.rs+++ b/datafusion/core/tests/parquet/mod.rs@@ -126,79 +126,97 @@ struct TestOutput {impl TestOutput {/// retrieve the value of the named metric, if anyfn metric_value(&self, metric_name: &str) -> Option<usize> {+ if let Some((pruned, _matched)) = self.pruning_metric(metric_name) {+ return Some(pruned);+ }+self.parquet_metrics.sum(|metric| metric.value().name() == metric_name)- .map(|v| v.as_usize())- }-- /// The number of times the pruning predicate evaluation errors- fn predicate_evaluation_errors(&self) -> Option<usize> {- self.metric_value("predicate_evaluation_errors")- }-- /// The number of row_groups matched by bloom filter- fn row_groups_matched_bloom_filter(&self) -> Option<usize> {- self.metric_value("row_groups_matched_bloom_filter")- }-- /// The number of row_groups pruned by bloom filter- fn row_groups_pruned_bloom_filter(&self) -> Option<usize> {- self.metric_value("row_groups_pruned_bloom_filter")- }-- /// The number of row_groups matched by statistics- fn row_groups_matched_statistics(&self) -> Option<usize> {- self.metric_value("row_groups_matched_statistics")- }-- /// The number of row_groups pruned by statistics- fn row_groups_pruned_statistics(&self) -> Option<usize> {- self.metric_value("row_groups_pruned_statistics")+ .map(|v| match v {+ MetricValue::PruningMetrics {+ pruning_metrics, ..+ } => pruning_metrics.pruned(),+ _ => v.as_usize(),+ })}- /// Metric `files_ranges_pruned_statistics` tracks both pruned and matched count,- /// for testing purpose, here it only aggregate the `pruned` count.- fn files_ranges_pruned_statistics(&self) -> Option<usize> {+ fn pruning_metric(&self, metric_name: &str) -> Option<(usize, usize)> {let mut total_pruned = 0;+ let mut total_matched = 0;let mut found = false;for metric in self.parquet_metrics.iter() {let metric = metric.as_ref();- if metric.value().name() == "files_ranges_pruned_statistics" {+ if metric.value().name() == metric_name {if let MetricValue::PruningMetrics {pruning_metrics, ..} = metric.value(){total_pruned += pruning_metrics.pruned();+ total_matched += pruning_metrics.matched();found = true;}}}if found {- Some(total_pruned)+ Some((total_pruned, total_matched))} else {None}}+ /// The number of times the pruning predicate evaluation errors+ fn predicate_evaluation_errors(&self) -> Option<usize> {+ self.metric_value("predicate_evaluation_errors")+ }++ /// The number of row_groups pruned / matched by bloom filter+ fn row_groups_bloom_filter(&self) -> Option<(usize, usize)> {+ self.pruning_metric("row_groups_pruned_bloom_filter")+ }++ /// The number of row_groups matched by statistics+ fn row_groups_matched_statistics(&self) -> Option<usize> {+ self.pruning_metric("row_groups_pruned_statistics")+ .map(|(_pruned, matched)| matched)+ }++ /// The number of row_groups pruned by statistics+ fn row_groups_pruned_statistics(&self) -> Option<usize> {+ self.pruning_metric("row_groups_pruned_statistics")+ .map(|(pruned, _matched)| pruned)+ }++ /// Metric `files_ranges_pruned_statistics` tracks both pruned and matched count,+ /// for testing purpose, here it only aggregate the `pruned` count.+ fn files_ranges_pruned_statistics(&self) -> Option<usize> {+ self.pruning_metric("files_ranges_pruned_statistics")+ .map(|(pruned, _matched)| pruned)+ }+/// The number of row_groups matched by bloom filter or statistics+ ///+ /// E.g. starting with 10 row groups, statistics: 10 total -> 7 matched, bloom+ /// filter: 7 total -> 3 matched, this function returns 3 for the final matched+ /// count.fn row_groups_matched(&self) -> Option<usize> {- self.row_groups_matched_bloom_filter()- .zip(self.row_groups_matched_statistics())- .map(|(a, b)| a + b)+ self.row_groups_bloom_filter()+ .map(|(_pruned, matched)| matched)}/// The number of row_groups prunedfn row_groups_pruned(&self) -> Option<usize> {- self.row_groups_pruned_bloom_filter()+ self.row_groups_bloom_filter()+ .map(|(pruned, _matched)| pruned).zip(self.row_groups_pruned_statistics()).map(|(a, b)| a + b)}/// The number of row pages prunedfn row_pages_pruned(&self) -> Option<usize> {- self.metric_value("page_index_rows_pruned")+ self.pruning_metric("page_index_rows_pruned")+ .map(|(pruned, _matched)| pruned)}fn description(&self) -> String {diff --git a/datafusion/core/tests/parquet/row_group_pruning.rs b/datafusion/core/tests/parquet/row_group_pruning.rsindex 44409166d3..0411298055 100644--- a/datafusion/core/tests/parquet/row_group_pruning.rs+++ b/datafusion/core/tests/parquet/row_group_pruning.rs@@ -133,13 +133,14 @@ impl RowGroupPruningTest {self.expected_files_pruned_by_statistics,"mismatched files_ranges_pruned_statistics",);+ let bloom_filter_metrics = output.row_groups_bloom_filter();assert_eq!(- output.row_groups_matched_bloom_filter(),+ bloom_filter_metrics.map(|(_pruned, matched)| matched),self.expected_row_group_matched_by_bloom_filter,"mismatched row_groups_matched_bloom_filter",);assert_eq!(- output.row_groups_pruned_bloom_filter(),+ bloom_filter_metrics.map(|(pruned, _matched)| pruned),self.expected_row_group_pruned_by_bloom_filter,"mismatched row_groups_pruned_bloom_filter",);@@ -163,7 +164,7 @@ async fn prune_timestamps_nanos() {.with_matched_by_stats(Some(3)).with_pruned_by_stats(Some(1)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(3)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(10).test_row_group_prune()@@ -181,7 +182,7 @@ async fn prune_timestamps_micros() {.with_matched_by_stats(Some(3)).with_pruned_by_stats(Some(1)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(3)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(10).test_row_group_prune()@@ -199,7 +200,7 @@ async fn prune_timestamps_millis() {.with_matched_by_stats(Some(3)).with_pruned_by_stats(Some(1)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(3)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(10).test_row_group_prune()@@ -217,7 +218,7 @@ async fn prune_timestamps_seconds() {.with_matched_by_stats(Some(3)).with_pruned_by_stats(Some(1)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(3)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(10).test_row_group_prune()@@ -233,7 +234,7 @@ async fn prune_date32() {.with_matched_by_stats(Some(1)).with_pruned_by_stats(Some(3)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(1)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(1).test_row_group_prune()@@ -262,8 +263,9 @@ async fn prune_date64() {println!("{}", output.description());// This should prune out groups without errorassert_eq!(output.predicate_evaluation_errors(), Some(0));- assert_eq!(output.row_groups_matched(), Some(1));- assert_eq!(output.row_groups_pruned(), Some(3));+ // 'dates' table has 4 row groups, and only the first one is matched by the predicate+ assert_eq!(output.row_groups_matched_statistics(), Some(1));+ assert_eq!(output.row_groups_pruned_statistics(), Some(3));assert_eq!(output.result_rows, 1, "{}", output.description());}@@ -276,7 +278,7 @@ async fn prune_disabled() {.with_matched_by_stats(Some(3)).with_pruned_by_stats(Some(1)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(3)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(10).test_row_group_prune()@@ -296,7 +298,7 @@ async fn prune_disabled() {// This should not prune anyassert_eq!(output.predicate_evaluation_errors(), Some(0));- assert_eq!(output.row_groups_matched(), Some(0));+ assert_eq!(output.row_groups_matched(), Some(4));assert_eq!(output.row_groups_pruned(), Some(0));assert_eq!(output.result_rows,@@ -322,7 +324,7 @@ macro_rules! int_tests {.with_matched_by_stats(Some(3)).with_pruned_by_stats(Some(1)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(3)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(11).test_row_group_prune()@@ -337,7 +339,7 @@ macro_rules! int_tests {.with_matched_by_stats(Some(3)).with_pruned_by_stats(Some(1)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(3)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(11).test_row_group_prune()@@ -381,10 +383,10 @@ macro_rules! int_tests {.with_scenario(Scenario::Int).with_query(&format!("SELECT * FROM t where abs(i{}) = 1", $bits)).with_expected_errors(Some(0))- .with_matched_by_stats(Some(0))+ .with_matched_by_stats(Some(4)).with_pruned_by_stats(Some(0)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(4)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(3).test_row_group_prune()@@ -397,10 +399,10 @@ macro_rules! int_tests {.with_scenario(Scenario::Int).with_query(&format!("SELECT * FROM t where i{}+1 = 1", $bits)).with_expected_errors(Some(0))- .with_matched_by_stats(Some(0))+ .with_matched_by_stats(Some(4)).with_pruned_by_stats(Some(0)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(4)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(2).test_row_group_prune()@@ -413,10 +415,10 @@ macro_rules! int_tests {.with_scenario(Scenario::Int).with_query(&format!("SELECT * FROM t where 1-i{} > 1", $bits)).with_expected_errors(Some(0))- .with_matched_by_stats(Some(0))+ .with_matched_by_stats(Some(4)).with_pruned_by_stats(Some(0)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(4)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(9).test_row_group_prune()@@ -498,7 +500,7 @@ macro_rules! uint_tests {.with_matched_by_stats(Some(3)).with_pruned_by_stats(Some(1)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(3)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(11).test_row_group_prune()@@ -542,10 +544,10 @@ macro_rules! uint_tests {.with_scenario(Scenario::UInt).with_query(&format!("SELECT * FROM t where power(u{}, 2) = 25", $bits)).with_expected_errors(Some(0))- .with_matched_by_stats(Some(0))+ .with_matched_by_stats(Some(4)).with_pruned_by_stats(Some(0)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(4)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(2).test_row_group_prune()@@ -558,10 +560,10 @@ macro_rules! uint_tests {.with_scenario(Scenario::UInt).with_query(&format!("SELECT * FROM t where u{}+1 = 6", $bits)).with_expected_errors(Some(0))- .with_matched_by_stats(Some(0))+ .with_matched_by_stats(Some(4)).with_pruned_by_stats(Some(0)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(4)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(2).test_row_group_prune()@@ -682,7 +684,7 @@ async fn prune_f64_lt() {.with_matched_by_stats(Some(3)).with_pruned_by_stats(Some(1)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(3)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(11).test_row_group_prune()@@ -694,7 +696,7 @@ async fn prune_f64_lt() {.with_matched_by_stats(Some(3)).with_pruned_by_stats(Some(1)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(3)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(11).test_row_group_prune()@@ -712,7 +714,7 @@ async fn prune_f64_scalar_fun_and_gt() {.with_matched_by_stats(Some(2)).with_pruned_by_stats(Some(2)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(2)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(1).test_row_group_prune()@@ -726,10 +728,10 @@ async fn prune_f64_scalar_fun() {.with_scenario(Scenario::Float64).with_query("SELECT * FROM t where abs(f-1) <= 0.000001").with_expected_errors(Some(0))- .with_matched_by_stats(Some(0))+ .with_matched_by_stats(Some(4)).with_pruned_by_stats(Some(0)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(4)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(1).test_row_group_prune()@@ -743,10 +745,10 @@ async fn prune_f64_complex_expr() {.with_scenario(Scenario::Float64).with_query("SELECT * FROM t where f+1 > 1.1").with_expected_errors(Some(0))- .with_matched_by_stats(Some(0))+ .with_matched_by_stats(Some(4)).with_pruned_by_stats(Some(0)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(4)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(9).test_row_group_prune()@@ -760,10 +762,10 @@ async fn prune_f64_complex_expr_subtract() {.with_scenario(Scenario::Float64).with_query("SELECT * FROM t where 1-f > 1").with_expected_errors(Some(0))- .with_matched_by_stats(Some(0))+ .with_matched_by_stats(Some(4)).with_pruned_by_stats(Some(0)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(4)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(9).test_row_group_prune()@@ -782,7 +784,7 @@ async fn prune_decimal_lt() {.with_matched_by_stats(Some(2)).with_pruned_by_stats(Some(1)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(2)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(6).test_row_group_prune()@@ -794,7 +796,7 @@ async fn prune_decimal_lt() {.with_matched_by_stats(Some(2)).with_pruned_by_stats(Some(1)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(2)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(8).test_row_group_prune()@@ -806,7 +808,7 @@ async fn prune_decimal_lt() {.with_matched_by_stats(Some(2)).with_pruned_by_stats(Some(1)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(2)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(6).test_row_group_prune()@@ -818,7 +820,7 @@ async fn prune_decimal_lt() {.with_matched_by_stats(Some(2)).with_pruned_by_stats(Some(1)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(2)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(8).test_row_group_prune()@@ -894,7 +896,7 @@ async fn prune_decimal_in_list() {.with_matched_by_stats(Some(2)).with_pruned_by_stats(Some(1)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(2)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(5).test_row_group_prune()@@ -906,7 +908,7 @@ async fn prune_decimal_in_list() {.with_matched_by_stats(Some(2)).with_pruned_by_stats(Some(1)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(2)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(6).test_row_group_prune()@@ -918,7 +920,7 @@ async fn prune_decimal_in_list() {.with_matched_by_stats(Some(2)).with_pruned_by_stats(Some(1)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(2)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(5).test_row_group_prune()@@ -930,7 +932,7 @@ async fn prune_decimal_in_list() {.with_matched_by_stats(Some(2)).with_pruned_by_stats(Some(1)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(2)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(6).test_row_group_prune()@@ -1064,7 +1066,7 @@ async fn prune_string_lt() {.with_matched_by_stats(Some(1)).with_pruned_by_stats(Some(2)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(1)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(3).test_row_group_prune()@@ -1079,7 +1081,7 @@ async fn prune_string_lt() {.with_matched_by_stats(Some(2)).with_pruned_by_stats(Some(1)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(2)).with_pruned_by_bloom_filter(Some(0))// all backends from 'mixed' and 'all backends'.with_expected_rows(8)@@ -1172,7 +1174,7 @@ async fn prune_binary_lt() {.with_matched_by_stats(Some(1)).with_pruned_by_stats(Some(2)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(1)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(3).test_row_group_prune()@@ -1187,7 +1189,7 @@ async fn prune_binary_lt() {.with_matched_by_stats(Some(2)).with_pruned_by_stats(Some(1)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(2)).with_pruned_by_bloom_filter(Some(0))// all backends from 'mixed' and 'all backends'.with_expected_rows(8)@@ -1279,7 +1281,7 @@ async fn prune_fixedsizebinary_lt() {.with_matched_by_stats(Some(1)).with_pruned_by_stats(Some(2)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(1)).with_pruned_by_bloom_filter(Some(0)).with_expected_rows(2).test_row_group_prune()@@ -1294,7 +1296,7 @@ async fn prune_fixedsizebinary_lt() {.with_matched_by_stats(Some(2)).with_pruned_by_stats(Some(1)).with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(2)).with_pruned_by_bloom_filter(Some(0))// all backends from 'mixed' and 'all backends'.with_expected_rows(8)@@ -1362,7 +1364,7 @@ async fn test_row_group_with_null_values() {.with_pruned_files(Some(0)).with_pruned_by_stats(Some(2)).with_expected_rows(5)- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(1)).with_pruned_by_bloom_filter(Some(0)).test_row_group_prune().await;@@ -1376,7 +1378,7 @@ async fn test_row_group_with_null_values() {.with_pruned_files(Some(0)).with_pruned_by_stats(Some(1)).with_expected_rows(10)- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(2)).with_pruned_by_bloom_filter(Some(0)).test_row_group_prune().await;@@ -1390,7 +1392,7 @@ async fn test_row_group_with_null_values() {.with_pruned_files(Some(0)).with_pruned_by_stats(Some(2)).with_expected_rows(5)- .with_matched_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(Some(1)).with_pruned_by_bloom_filter(Some(0)).test_row_group_prune().await;diff --git a/datafusion/core/tests/sql/explain_analyze.rs b/datafusion/core/tests/sql/explain_analyze.rsindex a7cc30a948..b3e8dac111 100644--- a/datafusion/core/tests/sql/explain_analyze.rs+++ b/datafusion/core/tests/sql/explain_analyze.rs@@ -852,10 +852,14 @@ async fn parquet_explain_analyze() {// should contain aggregated statsassert_contains!(&formatted, "output_rows=8");- assert_contains!(&formatted, "row_groups_matched_bloom_filter=0");- assert_contains!(&formatted, "row_groups_pruned_bloom_filter=0");- assert_contains!(&formatted, "row_groups_matched_statistics=1");- assert_contains!(&formatted, "row_groups_pruned_statistics=0");+ assert_contains!(+ &formatted,+ "row_groups_pruned_bloom_filter=1 total \u{2192} 1 matched"+ );+ assert_contains!(+ &formatted,+ "row_groups_pruned_statistics=1 total \u{2192} 1 matched"+ );}// This test reproduces the behavior described in@@ -995,9 +999,7 @@ async fn parquet_explain_analyze_verbose() {.to_string();// should contain the raw per file stats (with the label)- assert_contains!(&formatted, "row_groups_matched_bloom_filter{partition=0");assert_contains!(&formatted, "row_groups_pruned_bloom_filter{partition=0");- assert_contains!(&formatted, "row_groups_matched_statistics{partition=0");assert_contains!(&formatted, "row_groups_pruned_statistics{partition=0");}diff --git a/datafusion/datasource-parquet/src/metrics.rs b/datafusion/datasource-parquet/src/metrics.rsindex 9d86a3ae9f..306bc9e6b0 100644--- a/datafusion/datasource-parquet/src/metrics.rs+++ b/datafusion/datasource-parquet/src/metrics.rs@@ -44,14 +44,10 @@ pub struct ParquetFileMetrics {pub files_ranges_pruned_statistics: PruningMetrics,/// Number of times the predicate could not be evaluatedpub predicate_evaluation_errors: Count,- /// Number of row groups whose bloom filters were checked and matched (not pruned)- pub row_groups_matched_bloom_filter: Count,- /// Number of row groups pruned by bloom filters- pub row_groups_pruned_bloom_filter: Count,- /// Number of row groups whose statistics were checked and matched (not pruned)- pub row_groups_matched_statistics: Count,- /// Number of row groups pruned by statistics- pub row_groups_pruned_statistics: Count,+ /// Number of row groups whose bloom filters were checked, tracked with matched/pruned counts+ pub row_groups_pruned_bloom_filter: PruningMetrics,+ /// Number of row groups whose statistics were checked, tracked with matched/pruned counts+ pub row_groups_pruned_statistics: PruningMetrics,/// Total number of bytes scannedpub bytes_scanned: Count,/// Total rows filtered out by predicates pushed into parquet scan@@ -64,10 +60,8 @@ pub struct ParquetFileMetrics {pub statistics_eval_time: Time,/// Total time spent evaluating row group Bloom Filterspub bloom_filter_eval_time: Time,- /// Total rows filtered out by parquet page index- pub page_index_rows_pruned: Count,- /// Total rows passed through the parquet page index- pub page_index_rows_matched: Count,+ /// Total rows filtered or matched by parquet page index+ pub page_index_rows_pruned: PruningMetrics,/// Total time spent evaluating parquet page index filterspub page_index_eval_time: Time,/// Total time spent reading and parsing metadata from the footer@@ -91,34 +85,20 @@ impl ParquetFileMetrics {// -----------------------// 'summary' level metrics// ------------------------ let row_groups_matched_bloom_filter = MetricBuilder::new(metrics)- .with_new_label("filename", filename.to_string())- .with_type(MetricType::SUMMARY)- .counter("row_groups_matched_bloom_filter", partition);-let row_groups_pruned_bloom_filter = MetricBuilder::new(metrics).with_new_label("filename", filename.to_string()).with_type(MetricType::SUMMARY)- .counter("row_groups_pruned_bloom_filter", partition);-- let row_groups_matched_statistics = MetricBuilder::new(metrics)- .with_new_label("filename", filename.to_string())- .with_type(MetricType::SUMMARY)- .counter("row_groups_matched_statistics", partition);+ .pruning_metrics("row_groups_pruned_bloom_filter", partition);let row_groups_pruned_statistics = MetricBuilder::new(metrics).with_new_label("filename", filename.to_string()).with_type(MetricType::SUMMARY)- .counter("row_groups_pruned_statistics", partition);+ .pruning_metrics("row_groups_pruned_statistics", partition);let page_index_rows_pruned = MetricBuilder::new(metrics).with_new_label("filename", filename.to_string()).with_type(MetricType::SUMMARY)- .counter("page_index_rows_pruned", partition);- let page_index_rows_matched = MetricBuilder::new(metrics)- .with_new_label("filename", filename.to_string())- .with_type(MetricType::SUMMARY)- .counter("page_index_rows_matched", partition);+ .pruning_metrics("page_index_rows_pruned", partition);let bytes_scanned = MetricBuilder::new(metrics).with_new_label("filename", filename.to_string())@@ -173,16 +153,13 @@ impl ParquetFileMetrics {Self {files_ranges_pruned_statistics,predicate_evaluation_errors,- row_groups_matched_bloom_filter,row_groups_pruned_bloom_filter,- row_groups_matched_statistics,row_groups_pruned_statistics,bytes_scanned,pushdown_rows_pruned,pushdown_rows_matched,row_pushdown_eval_time,page_index_rows_pruned,- page_index_rows_matched,statistics_eval_time,bloom_filter_eval_time,page_index_eval_time,diff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rsindex 1c9b9feb9f..2815b82f1d 100644--- a/datafusion/datasource-parquet/src/opener.rs+++ b/datafusion/datasource-parquet/src/opener.rs@@ -361,6 +361,7 @@ impl FileOpener for ParquetOpener {if let Some(range) = file_range.as_ref() {row_groups.prune_by_range(rg_metadata, range);}+// If there is a predicate that can be evaluated against the metadataif let Some(predicate) = predicate.as_ref() {if enable_row_group_stats_pruning {@@ -371,6 +372,12 @@ impl FileOpener for ParquetOpener {predicate,&file_metrics,);+ } else {+ // Update metrics: statistics unavailable, so all row groups are+ // matched (not pruned)+ file_metrics+ .row_groups_pruned_statistics+ .add_matched(row_groups.remaining_row_group_count());}if enable_bloom_filter && !row_groups.is_empty() {@@ -382,7 +389,22 @@ impl FileOpener for ParquetOpener {&file_metrics,).await;+ } else {+ // Update metrics: bloom filter unavailable, so all row groups are+ // matched (not pruned)+ file_metrics+ .row_groups_pruned_bloom_filter+ .add_matched(row_groups.remaining_row_group_count());}+ } else {+ // Update metrics: no predicate, so all row groups are matched (not pruned)+ let n_remaining_row_groups = row_groups.remaining_row_group_count();+ file_metrics+ .row_groups_pruned_statistics+ .add_matched(n_remaining_row_groups);+ file_metrics+ .row_groups_pruned_bloom_filter+ .add_matched(n_remaining_row_groups);}let mut access_plan = row_groups.build();diff --git a/datafusion/datasource-parquet/src/page_filter.rs b/datafusion/datasource-parquet/src/page_filter.rsindex 82deedd406..2698b6c5fb 100644--- a/datafusion/datasource-parquet/src/page_filter.rs+++ b/datafusion/datasource-parquet/src/page_filter.rs@@ -269,8 +269,10 @@ impl PagePruningAccessPlanFilter {}}- file_metrics.page_index_rows_pruned.add(total_skip);- file_metrics.page_index_rows_matched.add(total_select);+ file_metrics.page_index_rows_pruned.add_pruned(total_skip);+ file_metrics+ .page_index_rows_pruned+ .add_matched(total_select);access_plan}diff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rsindex 51d50d780f..2043f75070 100644--- a/datafusion/datasource-parquet/src/row_group_filter.rs+++ b/datafusion/datasource-parquet/src/row_group_filter.rs@@ -60,6 +60,11 @@ impl RowGroupAccessPlanFilter {self.access_plan.is_empty()}+ /// Return the number of row groups that are currently expected to be scanned+ pub fn remaining_row_group_count(&self) -> usize {+ self.access_plan.row_group_index_iter().count()+ }+/// Returns the inner access planpub fn build(self) -> ParquetAccessPlan {self.access_plan@@ -134,9 +139,9 @@ impl RowGroupAccessPlanFilter {for (idx, &value) in row_group_indexes.iter().zip(values.iter()) {if !value {self.access_plan.skip(*idx);- metrics.row_groups_pruned_statistics.add(1);+ metrics.row_groups_pruned_statistics.add_pruned(1);} else {- metrics.row_groups_matched_statistics.add(1);+ metrics.row_groups_pruned_statistics.add_matched(1);}}}@@ -215,10 +220,10 @@ impl RowGroupAccessPlanFilter {};if prune_group {- metrics.row_groups_pruned_bloom_filter.add(1);+ metrics.row_groups_pruned_bloom_filter.add_pruned(1);self.access_plan.skip(idx)- } else if !stats.column_sbbf.is_empty() {- metrics.row_groups_matched_bloom_filter.add(1);+ } else {+ metrics.row_groups_pruned_bloom_filter.add_matched(1);}}}@@ -494,6 +499,18 @@ mod tests {}}+ #[test]+ fn remaining_row_group_count_reports_non_skipped_groups() {+ let mut filter = RowGroupAccessPlanFilter::new(ParquetAccessPlan::new_all(4));+ assert_eq!(filter.remaining_row_group_count(), 4);++ filter.access_plan.skip(1);+ assert_eq!(filter.remaining_row_group_count(), 3);++ filter.access_plan.skip(3);+ assert_eq!(filter.remaining_row_group_count(), 2);+ }+#[test]fn row_group_pruning_predicate_simple_expr() {use datafusion_expr::{col, lit};diff --git a/datafusion/physical-plan/src/metrics/mod.rs b/datafusion/physical-plan/src/metrics/mod.rsindex e66db8f0c9..c9ddbe8f89 100644--- a/datafusion/physical-plan/src/metrics/mod.rs+++ b/datafusion/physical-plan/src/metrics/mod.rs@@ -304,7 +304,7 @@ impl MetricsSet {MetricValue::Gauge { name, .. } => name == metric_name,MetricValue::StartTimestamp(_) => false,MetricValue::EndTimestamp(_) => false,- MetricValue::PruningMetrics { .. } => false,+ MetricValue::PruningMetrics { name, .. } => name == metric_name,MetricValue::Custom { .. } => false,})}diff --git a/docs/source/user-guide/explain-usage.md b/docs/source/user-guide/explain-usage.mdindex 2288cae85d..5a1184539c 100644--- a/docs/source/user-guide/explain-usage.md+++ b/docs/source/user-guide/explain-usage.md@@ -225,14 +225,11 @@ Again, reading from bottom up:When predicate pushdown is enabled, `DataSourceExec` with `ParquetSource` gains the following metrics:-- `page_index_rows_matched`: number of rows in pages that were tested by a page index filter, and passed-- `page_index_rows_pruned`: number of rows in pages that were tested by a page index filter, and did not pass-- `row_groups_matched_bloom_filter`: number of row groups that were tested by a Bloom Filter, and passed-- `row_groups_pruned_bloom_filter`: number of row groups that were tested by a Bloom Filter, and did not pass-- `row_groups_matched_statistics`: number of row groups that were tested by row group statistics (min and max value), and passed-- `row_groups_pruned_statistics`: number of row groups that were tested by row group statistics (min and max value), and did not pass-- `pushdown_rows_matched`: rows that were tested by any of the above filtered, and passed all of them (this should be minimum of `page_index_rows_matched`, `row_groups_pruned_bloom_filter`, and `row_groups_pruned_statistics`)-- `pushdown_rows_pruned`: rows that were tested by any of the above filtered, and did not pass one of them (this should be sum of `page_index_rows_matched`, `row_groups_pruned_bloom_filter`, and `row_groups_pruned_statistics`)+- `page_index_rows_pruned`: number of rows evaluated by page index filters. The metric reports both how many rows were considered in total and how many matched (were not pruned).+- `row_groups_pruned_bloom_filter`: number of row groups evaluated by Bloom Filters, reporting both total checked groups and groups that matched.+- `row_groups_pruned_statistics`: number of row groups evaluated by row-group statistics (min/max), reporting both total checked groups and groups that matched.+- `pushdown_rows_matched`: rows that were tested by any of the above filters, and passed all of them.+- `pushdown_rows_pruned`: rows that were tested by any of the above filters, and did not pass at least one of them.- `predicate_evaluation_errors`: number of times evaluating the filter expression failed (expected to be zero in normal operation)- `num_predicate_creation_errors`: number of errors creating predicates (expected to be zero in normal operation)- `bloom_filter_eval_time`: time spent parsing and evaluating Bloom Filters