Skip to content

Apache DataFusion SQL Query Engine

feat: Better parquet row-group/page pruning metrics display (#18321)

Yongting You9e9eb944c062
diff --git a/datafusion-examples/examples/json_shredding.rs b/datafusion-examples/examples/json_shredding.rs
index a2e83bc951..5ef8b59b64 100644
--- a/datafusion-examples/examples/json_shredding.rs
+++ b/datafusion-examples/examples/json_shredding.rs
@@ -142,7 +142,7 @@ async fn main() -> Result<()> {
.await?;
let plan = format!("{}", arrow::util::pretty::pretty_format_batches(&batches)?);
println!("{plan}");
- assert_contains!(&plan, "row_groups_pruned_statistics=1");
+ assert_contains!(&plan, "row_groups_pruned_statistics=2 total → 1 matched");
assert_contains!(&plan, "pushdown_rows_pruned=1");
Ok(())
diff --git a/datafusion/core/src/datasource/physical_plan/parquet.rs b/datafusion/core/src/datasource/physical_plan/parquet.rs
index 18b855cec5..0ffb252a66 100644
--- a/datafusion/core/src/datasource/physical_plan/parquet.rs
+++ b/datafusion/core/src/datasource/physical_plan/parquet.rs
@@ -65,7 +65,7 @@ mod tests {
use datafusion_physical_plan::analyze::AnalyzeExec;
use datafusion_physical_plan::collect;
use datafusion_physical_plan::metrics::{
- ExecutionPlanMetricsSet, MetricType, MetricsSet,
+ ExecutionPlanMetricsSet, MetricType, MetricValue, MetricsSet,
};
use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties};
@@ -1175,8 +1175,10 @@ mod tests {
// There are 4 rows pruned in each of batch2, batch3, and
// batch4 for a total of 12. batch1 had no pruning as c2 was
// filled in as null
- assert_eq!(get_value(&metrics, "page_index_rows_pruned"), 12);
- assert_eq!(get_value(&metrics, "page_index_rows_matched"), 6);
+ let (page_index_pruned, page_index_matched) =
+ get_pruning_metric(&metrics, "page_index_rows_pruned");
+ assert_eq!(page_index_pruned, 12);
+ assert_eq!(page_index_matched, 6);
}
#[tokio::test]
@@ -1776,8 +1778,10 @@ mod tests {
| 5 |
+-----+
"###);
- assert_eq!(get_value(&metrics, "page_index_rows_pruned"), 4);
- assert_eq!(get_value(&metrics, "page_index_rows_matched"), 2);
+ let (page_index_pruned, page_index_matched) =
+ get_pruning_metric(&metrics, "page_index_rows_pruned");
+ assert_eq!(page_index_pruned, 4);
+ assert_eq!(page_index_matched, 2);
assert!(
get_value(&metrics, "page_index_eval_time") > 0,
"no eval time in metrics: {metrics:#?}"
@@ -1866,8 +1870,10 @@ mod tests {
assert_contains!(&explain, "predicate=c1@0 != bar");
// there's a single row group, but we can check that it matched
- // if no pruning was done this would be 0 instead of 1
- assert_contains!(&explain, "row_groups_matched_statistics=1");
+ assert_contains!(
+ &explain,
+ "row_groups_pruned_statistics=1 total \u{2192} 1 matched"
+ );
// check the projection
assert_contains!(&explain, "projection=[c1]");
@@ -1898,8 +1904,10 @@ mod tests {
// When both matched and pruned are 0, it means that the pruning predicate
// was not used at all.
- assert_contains!(&explain, "row_groups_matched_statistics=0");
- assert_contains!(&explain, "row_groups_pruned_statistics=0");
+ assert_contains!(
+ &explain,
+ "row_groups_pruned_statistics=1 total \u{2192} 1 matched"
+ );
// But pushdown predicate should be present
assert_contains!(
@@ -1952,7 +1960,12 @@ mod tests {
/// Panics if no such metric.
fn get_value(metrics: &MetricsSet, metric_name: &str) -> usize {
match metrics.sum_by_name(metric_name) {
- Some(v) => v.as_usize(),
+ Some(v) => match v {
+ MetricValue::PruningMetrics {
+ pruning_metrics, ..
+ } => pruning_metrics.pruned(),
+ _ => v.as_usize(),
+ },
_ => {
panic!(
"Expected metric not found. Looking for '{metric_name}' in\n\n{metrics:#?}"
@@ -1961,6 +1974,20 @@ mod tests {
}
}
+ fn get_pruning_metric(metrics: &MetricsSet, metric_name: &str) -> (usize, usize) {
+ match metrics.sum_by_name(metric_name) {
+ Some(MetricValue::PruningMetrics {
+ pruning_metrics, ..
+ }) => (pruning_metrics.pruned(), pruning_metrics.matched()),
+ Some(_) => panic!(
+ "Metric '{metric_name}' is not a pruning metric in\n\n{metrics:#?}"
+ ),
+ None => panic!(
+ "Expected metric not found. Looking for '{metric_name}' in\n\n{metrics:#?}"
+ ),
+ }
+ }
+
fn populate_csv_partitions(
tmp_dir: &TempDir,
partition_count: usize,
diff --git a/datafusion/core/tests/parquet/external_access_plan.rs b/datafusion/core/tests/parquet/external_access_plan.rs
index a5397c5a39..5135f95685 100644
--- a/datafusion/core/tests/parquet/external_access_plan.rs
+++ b/datafusion/core/tests/parquet/external_access_plan.rs
@@ -33,7 +33,7 @@ use datafusion_common::{assert_contains, DFSchema};
use datafusion_datasource_parquet::{ParquetAccessPlan, RowGroupAccess};
use datafusion_execution::object_store::ObjectStoreUrl;
use datafusion_expr::{col, lit, Expr};
-use datafusion_physical_plan::metrics::MetricsSet;
+use datafusion_physical_plan::metrics::{MetricValue, MetricsSet};
use datafusion_physical_plan::ExecutionPlan;
use datafusion_datasource::file_scan_config::FileScanConfigBuilder;
@@ -178,12 +178,21 @@ async fn plan_and_filter() {
.unwrap();
// Verify that row group pruning still happens for just that group
- let row_groups_pruned_statistics =
- metric_value(&parquet_metrics, "row_groups_pruned_statistics").unwrap();
- assert_eq!(
- row_groups_pruned_statistics, 1,
- "metrics : {parquet_metrics:#?}",
- );
+ let row_groups_pruned_statistics = parquet_metrics
+ .sum_by_name("row_groups_pruned_statistics")
+ .unwrap();
+ if let MetricValue::PruningMetrics {
+ pruning_metrics, ..
+ } = row_groups_pruned_statistics
+ {
+ assert_eq!(
+ pruning_metrics.pruned(),
+ 1,
+ "metrics : {parquet_metrics:#?}",
+ );
+ } else {
+ unreachable!("metrics `row_groups_pruned_statistics` should exist")
+ }
}
#[tokio::test]
diff --git a/datafusion/core/tests/parquet/filter_pushdown.rs b/datafusion/core/tests/parquet/filter_pushdown.rs
index 226497fe58..966f251613 100644
--- a/datafusion/core/tests/parquet/filter_pushdown.rs
+++ b/datafusion/core/tests/parquet/filter_pushdown.rs
@@ -29,7 +29,7 @@
use arrow::compute::concat_batches;
use arrow::record_batch::RecordBatch;
use datafusion::physical_plan::collect;
-use datafusion::physical_plan::metrics::MetricsSet;
+use datafusion::physical_plan::metrics::{MetricValue, MetricsSet};
use datafusion::prelude::{
col, lit, lit_timestamp_nano, Expr, ParquetReadOptions, SessionContext,
};
@@ -563,9 +563,9 @@ impl<'a> TestCase<'a> {
}
};
- let page_index_rows_pruned = get_value(&metrics, "page_index_rows_pruned");
+ let (page_index_rows_pruned, page_index_rows_matched) =
+ get_pruning_metrics(&metrics, "page_index_rows_pruned");
println!(" page_index_rows_pruned: {page_index_rows_pruned}");
- let page_index_rows_matched = get_value(&metrics, "page_index_rows_matched");
println!(" page_index_rows_matched: {page_index_rows_matched}");
let page_index_filtering_expected = if scan_options.enable_page_index {
@@ -592,14 +592,29 @@ impl<'a> TestCase<'a> {
}
}
+fn get_pruning_metrics(metrics: &MetricsSet, metric_name: &str) -> (usize, usize) {
+ match metrics.sum_by_name(metric_name) {
+ Some(MetricValue::PruningMetrics {
+ pruning_metrics, ..
+ }) => (pruning_metrics.pruned(), pruning_metrics.matched()),
+ Some(_) => {
+ panic!("Metric '{metric_name}' is not a pruning metric in\n\n{metrics:#?}")
+ }
+ None => panic!(
+ "Expected metric not found. Looking for '{metric_name}' in\n\n{metrics:#?}"
+ ),
+ }
+}
+
fn get_value(metrics: &MetricsSet, metric_name: &str) -> usize {
match metrics.sum_by_name(metric_name) {
+ Some(MetricValue::PruningMetrics {
+ pruning_metrics, ..
+ }) => pruning_metrics.pruned(),
Some(v) => v.as_usize(),
- _ => {
- panic!(
- "Expected metric not found. Looking for '{metric_name}' in\n\n{metrics:#?}"
- );
- }
+ None => panic!(
+ "Expected metric not found. Looking for '{metric_name}' in\n\n{metrics:#?}"
+ ),
}
}
diff --git a/datafusion/core/tests/parquet/mod.rs b/datafusion/core/tests/parquet/mod.rs
index 34a48cdae3..097600e45e 100644
--- a/datafusion/core/tests/parquet/mod.rs
+++ b/datafusion/core/tests/parquet/mod.rs
@@ -126,79 +126,97 @@ struct TestOutput {
impl TestOutput {
/// retrieve the value of the named metric, if any
fn metric_value(&self, metric_name: &str) -> Option<usize> {
+ if let Some((pruned, _matched)) = self.pruning_metric(metric_name) {
+ return Some(pruned);
+ }
+
self.parquet_metrics
.sum(|metric| metric.value().name() == metric_name)
- .map(|v| v.as_usize())
- }
-
- /// The number of times the pruning predicate evaluation errors
- fn predicate_evaluation_errors(&self) -> Option<usize> {
- self.metric_value("predicate_evaluation_errors")
- }
-
- /// The number of row_groups matched by bloom filter
- fn row_groups_matched_bloom_filter(&self) -> Option<usize> {
- self.metric_value("row_groups_matched_bloom_filter")
- }
-
- /// The number of row_groups pruned by bloom filter
- fn row_groups_pruned_bloom_filter(&self) -> Option<usize> {
- self.metric_value("row_groups_pruned_bloom_filter")
- }
-
- /// The number of row_groups matched by statistics
- fn row_groups_matched_statistics(&self) -> Option<usize> {
- self.metric_value("row_groups_matched_statistics")
- }
-
- /// The number of row_groups pruned by statistics
- fn row_groups_pruned_statistics(&self) -> Option<usize> {
- self.metric_value("row_groups_pruned_statistics")
+ .map(|v| match v {
+ MetricValue::PruningMetrics {
+ pruning_metrics, ..
+ } => pruning_metrics.pruned(),
+ _ => v.as_usize(),
+ })
}
- /// Metric `files_ranges_pruned_statistics` tracks both pruned and matched count,
- /// for testing purpose, here it only aggregate the `pruned` count.
- fn files_ranges_pruned_statistics(&self) -> Option<usize> {
+ fn pruning_metric(&self, metric_name: &str) -> Option<(usize, usize)> {
let mut total_pruned = 0;
+ let mut total_matched = 0;
let mut found = false;
for metric in self.parquet_metrics.iter() {
let metric = metric.as_ref();
- if metric.value().name() == "files_ranges_pruned_statistics" {
+ if metric.value().name() == metric_name {
if let MetricValue::PruningMetrics {
pruning_metrics, ..
} = metric.value()
{
total_pruned += pruning_metrics.pruned();
+ total_matched += pruning_metrics.matched();
found = true;
}
}
}
if found {
- Some(total_pruned)
+ Some((total_pruned, total_matched))
} else {
None
}
}
+ /// The number of times the pruning predicate evaluation errors
+ fn predicate_evaluation_errors(&self) -> Option<usize> {
+ self.metric_value("predicate_evaluation_errors")
+ }
+
+ /// The number of row_groups pruned / matched by bloom filter
+ fn row_groups_bloom_filter(&self) -> Option<(usize, usize)> {
+ self.pruning_metric("row_groups_pruned_bloom_filter")
+ }
+
+ /// The number of row_groups matched by statistics
+ fn row_groups_matched_statistics(&self) -> Option<usize> {
+ self.pruning_metric("row_groups_pruned_statistics")
+ .map(|(_pruned, matched)| matched)
+ }
+
+ /// The number of row_groups pruned by statistics
+ fn row_groups_pruned_statistics(&self) -> Option<usize> {
+ self.pruning_metric("row_groups_pruned_statistics")
+ .map(|(pruned, _matched)| pruned)
+ }
+
+ /// Metric `files_ranges_pruned_statistics` tracks both pruned and matched count,
+ /// for testing purpose, here it only aggregate the `pruned` count.
+ fn files_ranges_pruned_statistics(&self) -> Option<usize> {
+ self.pruning_metric("files_ranges_pruned_statistics")
+ .map(|(pruned, _matched)| pruned)
+ }
+
/// The number of row_groups matched by bloom filter or statistics
+ ///
+ /// E.g. starting with 10 row groups, statistics: 10 total -> 7 matched, bloom
+ /// filter: 7 total -> 3 matched, this function returns 3 for the final matched
+ /// count.
fn row_groups_matched(&self) -> Option<usize> {
- self.row_groups_matched_bloom_filter()
- .zip(self.row_groups_matched_statistics())
- .map(|(a, b)| a + b)
+ self.row_groups_bloom_filter()
+ .map(|(_pruned, matched)| matched)
}
/// The number of row_groups pruned
fn row_groups_pruned(&self) -> Option<usize> {
- self.row_groups_pruned_bloom_filter()
+ self.row_groups_bloom_filter()
+ .map(|(pruned, _matched)| pruned)
.zip(self.row_groups_pruned_statistics())
.map(|(a, b)| a + b)
}
/// The number of row pages pruned
fn row_pages_pruned(&self) -> Option<usize> {
- self.metric_value("page_index_rows_pruned")
+ self.pruning_metric("page_index_rows_pruned")
+ .map(|(pruned, _matched)| pruned)
}
fn description(&self) -> String {
diff --git a/datafusion/core/tests/parquet/row_group_pruning.rs b/datafusion/core/tests/parquet/row_group_pruning.rs
index 44409166d3..0411298055 100644
--- a/datafusion/core/tests/parquet/row_group_pruning.rs
+++ b/datafusion/core/tests/parquet/row_group_pruning.rs
@@ -133,13 +133,14 @@ impl RowGroupPruningTest {
self.expected_files_pruned_by_statistics,
"mismatched files_ranges_pruned_statistics",
);
+ let bloom_filter_metrics = output.row_groups_bloom_filter();
assert_eq!(
- output.row_groups_matched_bloom_filter(),
+ bloom_filter_metrics.map(|(_pruned, matched)| matched),
self.expected_row_group_matched_by_bloom_filter,
"mismatched row_groups_matched_bloom_filter",
);
assert_eq!(
- output.row_groups_pruned_bloom_filter(),
+ bloom_filter_metrics.map(|(pruned, _matched)| pruned),
self.expected_row_group_pruned_by_bloom_filter,
"mismatched row_groups_pruned_bloom_filter",
);
@@ -163,7 +164,7 @@ async fn prune_timestamps_nanos() {
.with_matched_by_stats(Some(3))
.with_pruned_by_stats(Some(1))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(3))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(10)
.test_row_group_prune()
@@ -181,7 +182,7 @@ async fn prune_timestamps_micros() {
.with_matched_by_stats(Some(3))
.with_pruned_by_stats(Some(1))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(3))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(10)
.test_row_group_prune()
@@ -199,7 +200,7 @@ async fn prune_timestamps_millis() {
.with_matched_by_stats(Some(3))
.with_pruned_by_stats(Some(1))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(3))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(10)
.test_row_group_prune()
@@ -217,7 +218,7 @@ async fn prune_timestamps_seconds() {
.with_matched_by_stats(Some(3))
.with_pruned_by_stats(Some(1))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(3))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(10)
.test_row_group_prune()
@@ -233,7 +234,7 @@ async fn prune_date32() {
.with_matched_by_stats(Some(1))
.with_pruned_by_stats(Some(3))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(1))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(1)
.test_row_group_prune()
@@ -262,8 +263,9 @@ async fn prune_date64() {
println!("{}", output.description());
// This should prune out groups without error
assert_eq!(output.predicate_evaluation_errors(), Some(0));
- assert_eq!(output.row_groups_matched(), Some(1));
- assert_eq!(output.row_groups_pruned(), Some(3));
+ // 'dates' table has 4 row groups, and only the first one is matched by the predicate
+ assert_eq!(output.row_groups_matched_statistics(), Some(1));
+ assert_eq!(output.row_groups_pruned_statistics(), Some(3));
assert_eq!(output.result_rows, 1, "{}", output.description());
}
@@ -276,7 +278,7 @@ async fn prune_disabled() {
.with_matched_by_stats(Some(3))
.with_pruned_by_stats(Some(1))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(3))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(10)
.test_row_group_prune()
@@ -296,7 +298,7 @@ async fn prune_disabled() {
// This should not prune any
assert_eq!(output.predicate_evaluation_errors(), Some(0));
- assert_eq!(output.row_groups_matched(), Some(0));
+ assert_eq!(output.row_groups_matched(), Some(4));
assert_eq!(output.row_groups_pruned(), Some(0));
assert_eq!(
output.result_rows,
@@ -322,7 +324,7 @@ macro_rules! int_tests {
.with_matched_by_stats(Some(3))
.with_pruned_by_stats(Some(1))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(3))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(11)
.test_row_group_prune()
@@ -337,7 +339,7 @@ macro_rules! int_tests {
.with_matched_by_stats(Some(3))
.with_pruned_by_stats(Some(1))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(3))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(11)
.test_row_group_prune()
@@ -381,10 +383,10 @@ macro_rules! int_tests {
.with_scenario(Scenario::Int)
.with_query(&format!("SELECT * FROM t where abs(i{}) = 1", $bits))
.with_expected_errors(Some(0))
- .with_matched_by_stats(Some(0))
+ .with_matched_by_stats(Some(4))
.with_pruned_by_stats(Some(0))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(4))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(3)
.test_row_group_prune()
@@ -397,10 +399,10 @@ macro_rules! int_tests {
.with_scenario(Scenario::Int)
.with_query(&format!("SELECT * FROM t where i{}+1 = 1", $bits))
.with_expected_errors(Some(0))
- .with_matched_by_stats(Some(0))
+ .with_matched_by_stats(Some(4))
.with_pruned_by_stats(Some(0))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(4))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(2)
.test_row_group_prune()
@@ -413,10 +415,10 @@ macro_rules! int_tests {
.with_scenario(Scenario::Int)
.with_query(&format!("SELECT * FROM t where 1-i{} > 1", $bits))
.with_expected_errors(Some(0))
- .with_matched_by_stats(Some(0))
+ .with_matched_by_stats(Some(4))
.with_pruned_by_stats(Some(0))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(4))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(9)
.test_row_group_prune()
@@ -498,7 +500,7 @@ macro_rules! uint_tests {
.with_matched_by_stats(Some(3))
.with_pruned_by_stats(Some(1))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(3))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(11)
.test_row_group_prune()
@@ -542,10 +544,10 @@ macro_rules! uint_tests {
.with_scenario(Scenario::UInt)
.with_query(&format!("SELECT * FROM t where power(u{}, 2) = 25", $bits))
.with_expected_errors(Some(0))
- .with_matched_by_stats(Some(0))
+ .with_matched_by_stats(Some(4))
.with_pruned_by_stats(Some(0))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(4))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(2)
.test_row_group_prune()
@@ -558,10 +560,10 @@ macro_rules! uint_tests {
.with_scenario(Scenario::UInt)
.with_query(&format!("SELECT * FROM t where u{}+1 = 6", $bits))
.with_expected_errors(Some(0))
- .with_matched_by_stats(Some(0))
+ .with_matched_by_stats(Some(4))
.with_pruned_by_stats(Some(0))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(4))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(2)
.test_row_group_prune()
@@ -682,7 +684,7 @@ async fn prune_f64_lt() {
.with_matched_by_stats(Some(3))
.with_pruned_by_stats(Some(1))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(3))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(11)
.test_row_group_prune()
@@ -694,7 +696,7 @@ async fn prune_f64_lt() {
.with_matched_by_stats(Some(3))
.with_pruned_by_stats(Some(1))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(3))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(11)
.test_row_group_prune()
@@ -712,7 +714,7 @@ async fn prune_f64_scalar_fun_and_gt() {
.with_matched_by_stats(Some(2))
.with_pruned_by_stats(Some(2))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(2))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(1)
.test_row_group_prune()
@@ -726,10 +728,10 @@ async fn prune_f64_scalar_fun() {
.with_scenario(Scenario::Float64)
.with_query("SELECT * FROM t where abs(f-1) <= 0.000001")
.with_expected_errors(Some(0))
- .with_matched_by_stats(Some(0))
+ .with_matched_by_stats(Some(4))
.with_pruned_by_stats(Some(0))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(4))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(1)
.test_row_group_prune()
@@ -743,10 +745,10 @@ async fn prune_f64_complex_expr() {
.with_scenario(Scenario::Float64)
.with_query("SELECT * FROM t where f+1 > 1.1")
.with_expected_errors(Some(0))
- .with_matched_by_stats(Some(0))
+ .with_matched_by_stats(Some(4))
.with_pruned_by_stats(Some(0))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(4))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(9)
.test_row_group_prune()
@@ -760,10 +762,10 @@ async fn prune_f64_complex_expr_subtract() {
.with_scenario(Scenario::Float64)
.with_query("SELECT * FROM t where 1-f > 1")
.with_expected_errors(Some(0))
- .with_matched_by_stats(Some(0))
+ .with_matched_by_stats(Some(4))
.with_pruned_by_stats(Some(0))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(4))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(9)
.test_row_group_prune()
@@ -782,7 +784,7 @@ async fn prune_decimal_lt() {
.with_matched_by_stats(Some(2))
.with_pruned_by_stats(Some(1))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(2))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(6)
.test_row_group_prune()
@@ -794,7 +796,7 @@ async fn prune_decimal_lt() {
.with_matched_by_stats(Some(2))
.with_pruned_by_stats(Some(1))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(2))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(8)
.test_row_group_prune()
@@ -806,7 +808,7 @@ async fn prune_decimal_lt() {
.with_matched_by_stats(Some(2))
.with_pruned_by_stats(Some(1))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(2))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(6)
.test_row_group_prune()
@@ -818,7 +820,7 @@ async fn prune_decimal_lt() {
.with_matched_by_stats(Some(2))
.with_pruned_by_stats(Some(1))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(2))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(8)
.test_row_group_prune()
@@ -894,7 +896,7 @@ async fn prune_decimal_in_list() {
.with_matched_by_stats(Some(2))
.with_pruned_by_stats(Some(1))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(2))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(5)
.test_row_group_prune()
@@ -906,7 +908,7 @@ async fn prune_decimal_in_list() {
.with_matched_by_stats(Some(2))
.with_pruned_by_stats(Some(1))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(2))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(6)
.test_row_group_prune()
@@ -918,7 +920,7 @@ async fn prune_decimal_in_list() {
.with_matched_by_stats(Some(2))
.with_pruned_by_stats(Some(1))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(2))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(5)
.test_row_group_prune()
@@ -930,7 +932,7 @@ async fn prune_decimal_in_list() {
.with_matched_by_stats(Some(2))
.with_pruned_by_stats(Some(1))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(2))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(6)
.test_row_group_prune()
@@ -1064,7 +1066,7 @@ async fn prune_string_lt() {
.with_matched_by_stats(Some(1))
.with_pruned_by_stats(Some(2))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(1))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(3)
.test_row_group_prune()
@@ -1079,7 +1081,7 @@ async fn prune_string_lt() {
.with_matched_by_stats(Some(2))
.with_pruned_by_stats(Some(1))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(2))
.with_pruned_by_bloom_filter(Some(0))
// all backends from 'mixed' and 'all backends'
.with_expected_rows(8)
@@ -1172,7 +1174,7 @@ async fn prune_binary_lt() {
.with_matched_by_stats(Some(1))
.with_pruned_by_stats(Some(2))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(1))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(3)
.test_row_group_prune()
@@ -1187,7 +1189,7 @@ async fn prune_binary_lt() {
.with_matched_by_stats(Some(2))
.with_pruned_by_stats(Some(1))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(2))
.with_pruned_by_bloom_filter(Some(0))
// all backends from 'mixed' and 'all backends'
.with_expected_rows(8)
@@ -1279,7 +1281,7 @@ async fn prune_fixedsizebinary_lt() {
.with_matched_by_stats(Some(1))
.with_pruned_by_stats(Some(2))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(1))
.with_pruned_by_bloom_filter(Some(0))
.with_expected_rows(2)
.test_row_group_prune()
@@ -1294,7 +1296,7 @@ async fn prune_fixedsizebinary_lt() {
.with_matched_by_stats(Some(2))
.with_pruned_by_stats(Some(1))
.with_pruned_files(Some(0))
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(2))
.with_pruned_by_bloom_filter(Some(0))
// all backends from 'mixed' and 'all backends'
.with_expected_rows(8)
@@ -1362,7 +1364,7 @@ async fn test_row_group_with_null_values() {
.with_pruned_files(Some(0))
.with_pruned_by_stats(Some(2))
.with_expected_rows(5)
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(1))
.with_pruned_by_bloom_filter(Some(0))
.test_row_group_prune()
.await;
@@ -1376,7 +1378,7 @@ async fn test_row_group_with_null_values() {
.with_pruned_files(Some(0))
.with_pruned_by_stats(Some(1))
.with_expected_rows(10)
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(2))
.with_pruned_by_bloom_filter(Some(0))
.test_row_group_prune()
.await;
@@ -1390,7 +1392,7 @@ async fn test_row_group_with_null_values() {
.with_pruned_files(Some(0))
.with_pruned_by_stats(Some(2))
.with_expected_rows(5)
- .with_matched_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(Some(1))
.with_pruned_by_bloom_filter(Some(0))
.test_row_group_prune()
.await;
diff --git a/datafusion/core/tests/sql/explain_analyze.rs b/datafusion/core/tests/sql/explain_analyze.rs
index a7cc30a948..b3e8dac111 100644
--- a/datafusion/core/tests/sql/explain_analyze.rs
+++ b/datafusion/core/tests/sql/explain_analyze.rs
@@ -852,10 +852,14 @@ async fn parquet_explain_analyze() {
// should contain aggregated stats
assert_contains!(&formatted, "output_rows=8");
- assert_contains!(&formatted, "row_groups_matched_bloom_filter=0");
- assert_contains!(&formatted, "row_groups_pruned_bloom_filter=0");
- assert_contains!(&formatted, "row_groups_matched_statistics=1");
- assert_contains!(&formatted, "row_groups_pruned_statistics=0");
+ assert_contains!(
+ &formatted,
+ "row_groups_pruned_bloom_filter=1 total \u{2192} 1 matched"
+ );
+ assert_contains!(
+ &formatted,
+ "row_groups_pruned_statistics=1 total \u{2192} 1 matched"
+ );
}
// This test reproduces the behavior described in
@@ -995,9 +999,7 @@ async fn parquet_explain_analyze_verbose() {
.to_string();
// should contain the raw per file stats (with the label)
- assert_contains!(&formatted, "row_groups_matched_bloom_filter{partition=0");
assert_contains!(&formatted, "row_groups_pruned_bloom_filter{partition=0");
- assert_contains!(&formatted, "row_groups_matched_statistics{partition=0");
assert_contains!(&formatted, "row_groups_pruned_statistics{partition=0");
}
diff --git a/datafusion/datasource-parquet/src/metrics.rs b/datafusion/datasource-parquet/src/metrics.rs
index 9d86a3ae9f..306bc9e6b0 100644
--- a/datafusion/datasource-parquet/src/metrics.rs
+++ b/datafusion/datasource-parquet/src/metrics.rs
@@ -44,14 +44,10 @@ pub struct ParquetFileMetrics {
pub files_ranges_pruned_statistics: PruningMetrics,
/// Number of times the predicate could not be evaluated
pub predicate_evaluation_errors: Count,
- /// Number of row groups whose bloom filters were checked and matched (not pruned)
- pub row_groups_matched_bloom_filter: Count,
- /// Number of row groups pruned by bloom filters
- pub row_groups_pruned_bloom_filter: Count,
- /// Number of row groups whose statistics were checked and matched (not pruned)
- pub row_groups_matched_statistics: Count,
- /// Number of row groups pruned by statistics
- pub row_groups_pruned_statistics: Count,
+ /// Number of row groups whose bloom filters were checked, tracked with matched/pruned counts
+ pub row_groups_pruned_bloom_filter: PruningMetrics,
+ /// Number of row groups whose statistics were checked, tracked with matched/pruned counts
+ pub row_groups_pruned_statistics: PruningMetrics,
/// Total number of bytes scanned
pub bytes_scanned: Count,
/// Total rows filtered out by predicates pushed into parquet scan
@@ -64,10 +60,8 @@ pub struct ParquetFileMetrics {
pub statistics_eval_time: Time,
/// Total time spent evaluating row group Bloom Filters
pub bloom_filter_eval_time: Time,
- /// Total rows filtered out by parquet page index
- pub page_index_rows_pruned: Count,
- /// Total rows passed through the parquet page index
- pub page_index_rows_matched: Count,
+ /// Total rows filtered or matched by parquet page index
+ pub page_index_rows_pruned: PruningMetrics,
/// Total time spent evaluating parquet page index filters
pub page_index_eval_time: Time,
/// Total time spent reading and parsing metadata from the footer
@@ -91,34 +85,20 @@ impl ParquetFileMetrics {
// -----------------------
// 'summary' level metrics
// -----------------------
- let row_groups_matched_bloom_filter = MetricBuilder::new(metrics)
- .with_new_label("filename", filename.to_string())
- .with_type(MetricType::SUMMARY)
- .counter("row_groups_matched_bloom_filter", partition);
-
let row_groups_pruned_bloom_filter = MetricBuilder::new(metrics)
.with_new_label("filename", filename.to_string())
.with_type(MetricType::SUMMARY)
- .counter("row_groups_pruned_bloom_filter", partition);
-
- let row_groups_matched_statistics = MetricBuilder::new(metrics)
- .with_new_label("filename", filename.to_string())
- .with_type(MetricType::SUMMARY)
- .counter("row_groups_matched_statistics", partition);
+ .pruning_metrics("row_groups_pruned_bloom_filter", partition);
let row_groups_pruned_statistics = MetricBuilder::new(metrics)
.with_new_label("filename", filename.to_string())
.with_type(MetricType::SUMMARY)
- .counter("row_groups_pruned_statistics", partition);
+ .pruning_metrics("row_groups_pruned_statistics", partition);
let page_index_rows_pruned = MetricBuilder::new(metrics)
.with_new_label("filename", filename.to_string())
.with_type(MetricType::SUMMARY)
- .counter("page_index_rows_pruned", partition);
- let page_index_rows_matched = MetricBuilder::new(metrics)
- .with_new_label("filename", filename.to_string())
- .with_type(MetricType::SUMMARY)
- .counter("page_index_rows_matched", partition);
+ .pruning_metrics("page_index_rows_pruned", partition);
let bytes_scanned = MetricBuilder::new(metrics)
.with_new_label("filename", filename.to_string())
@@ -173,16 +153,13 @@ impl ParquetFileMetrics {
Self {
files_ranges_pruned_statistics,
predicate_evaluation_errors,
- row_groups_matched_bloom_filter,
row_groups_pruned_bloom_filter,
- row_groups_matched_statistics,
row_groups_pruned_statistics,
bytes_scanned,
pushdown_rows_pruned,
pushdown_rows_matched,
row_pushdown_eval_time,
page_index_rows_pruned,
- page_index_rows_matched,
statistics_eval_time,
bloom_filter_eval_time,
page_index_eval_time,
diff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rs
index 1c9b9feb9f..2815b82f1d 100644
--- a/datafusion/datasource-parquet/src/opener.rs
+++ b/datafusion/datasource-parquet/src/opener.rs
@@ -361,6 +361,7 @@ impl FileOpener for ParquetOpener {
if let Some(range) = file_range.as_ref() {
row_groups.prune_by_range(rg_metadata, range);
}
+
// If there is a predicate that can be evaluated against the metadata
if let Some(predicate) = predicate.as_ref() {
if enable_row_group_stats_pruning {
@@ -371,6 +372,12 @@ impl FileOpener for ParquetOpener {
predicate,
&file_metrics,
);
+ } else {
+ // Update metrics: statistics unavailable, so all row groups are
+ // matched (not pruned)
+ file_metrics
+ .row_groups_pruned_statistics
+ .add_matched(row_groups.remaining_row_group_count());
}
if enable_bloom_filter && !row_groups.is_empty() {
@@ -382,7 +389,22 @@ impl FileOpener for ParquetOpener {
&file_metrics,
)
.await;
+ } else {
+ // Update metrics: bloom filter unavailable, so all row groups are
+ // matched (not pruned)
+ file_metrics
+ .row_groups_pruned_bloom_filter
+ .add_matched(row_groups.remaining_row_group_count());
}
+ } else {
+ // Update metrics: no predicate, so all row groups are matched (not pruned)
+ let n_remaining_row_groups = row_groups.remaining_row_group_count();
+ file_metrics
+ .row_groups_pruned_statistics
+ .add_matched(n_remaining_row_groups);
+ file_metrics
+ .row_groups_pruned_bloom_filter
+ .add_matched(n_remaining_row_groups);
}
let mut access_plan = row_groups.build();
diff --git a/datafusion/datasource-parquet/src/page_filter.rs b/datafusion/datasource-parquet/src/page_filter.rs
index 82deedd406..2698b6c5fb 100644
--- a/datafusion/datasource-parquet/src/page_filter.rs
+++ b/datafusion/datasource-parquet/src/page_filter.rs
@@ -269,8 +269,10 @@ impl PagePruningAccessPlanFilter {
}
}
- file_metrics.page_index_rows_pruned.add(total_skip);
- file_metrics.page_index_rows_matched.add(total_select);
+ file_metrics.page_index_rows_pruned.add_pruned(total_skip);
+ file_metrics
+ .page_index_rows_pruned
+ .add_matched(total_select);
access_plan
}
diff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rs
index 51d50d780f..2043f75070 100644
--- a/datafusion/datasource-parquet/src/row_group_filter.rs
+++ b/datafusion/datasource-parquet/src/row_group_filter.rs
@@ -60,6 +60,11 @@ impl RowGroupAccessPlanFilter {
self.access_plan.is_empty()
}
+ /// Return the number of row groups that are currently expected to be scanned
+ pub fn remaining_row_group_count(&self) -> usize {
+ self.access_plan.row_group_index_iter().count()
+ }
+
/// Returns the inner access plan
pub fn build(self) -> ParquetAccessPlan {
self.access_plan
@@ -134,9 +139,9 @@ impl RowGroupAccessPlanFilter {
for (idx, &value) in row_group_indexes.iter().zip(values.iter()) {
if !value {
self.access_plan.skip(*idx);
- metrics.row_groups_pruned_statistics.add(1);
+ metrics.row_groups_pruned_statistics.add_pruned(1);
} else {
- metrics.row_groups_matched_statistics.add(1);
+ metrics.row_groups_pruned_statistics.add_matched(1);
}
}
}
@@ -215,10 +220,10 @@ impl RowGroupAccessPlanFilter {
};
if prune_group {
- metrics.row_groups_pruned_bloom_filter.add(1);
+ metrics.row_groups_pruned_bloom_filter.add_pruned(1);
self.access_plan.skip(idx)
- } else if !stats.column_sbbf.is_empty() {
- metrics.row_groups_matched_bloom_filter.add(1);
+ } else {
+ metrics.row_groups_pruned_bloom_filter.add_matched(1);
}
}
}
@@ -494,6 +499,18 @@ mod tests {
}
}
+ #[test]
+ fn remaining_row_group_count_reports_non_skipped_groups() {
+ let mut filter = RowGroupAccessPlanFilter::new(ParquetAccessPlan::new_all(4));
+ assert_eq!(filter.remaining_row_group_count(), 4);
+
+ filter.access_plan.skip(1);
+ assert_eq!(filter.remaining_row_group_count(), 3);
+
+ filter.access_plan.skip(3);
+ assert_eq!(filter.remaining_row_group_count(), 2);
+ }
+
#[test]
fn row_group_pruning_predicate_simple_expr() {
use datafusion_expr::{col, lit};
diff --git a/datafusion/physical-plan/src/metrics/mod.rs b/datafusion/physical-plan/src/metrics/mod.rs
index e66db8f0c9..c9ddbe8f89 100644
--- a/datafusion/physical-plan/src/metrics/mod.rs
+++ b/datafusion/physical-plan/src/metrics/mod.rs
@@ -304,7 +304,7 @@ impl MetricsSet {
MetricValue::Gauge { name, .. } => name == metric_name,
MetricValue::StartTimestamp(_) => false,
MetricValue::EndTimestamp(_) => false,
- MetricValue::PruningMetrics { .. } => false,
+ MetricValue::PruningMetrics { name, .. } => name == metric_name,
MetricValue::Custom { .. } => false,
})
}
diff --git a/docs/source/user-guide/explain-usage.md b/docs/source/user-guide/explain-usage.md
index 2288cae85d..5a1184539c 100644
--- a/docs/source/user-guide/explain-usage.md
+++ b/docs/source/user-guide/explain-usage.md
@@ -225,14 +225,11 @@ Again, reading from bottom up:
When predicate pushdown is enabled, `DataSourceExec` with `ParquetSource` gains the following metrics:
-- `page_index_rows_matched`: number of rows in pages that were tested by a page index filter, and passed
-- `page_index_rows_pruned`: number of rows in pages that were tested by a page index filter, and did not pass
-- `row_groups_matched_bloom_filter`: number of row groups that were tested by a Bloom Filter, and passed
-- `row_groups_pruned_bloom_filter`: number of row groups that were tested by a Bloom Filter, and did not pass
-- `row_groups_matched_statistics`: number of row groups that were tested by row group statistics (min and max value), and passed
-- `row_groups_pruned_statistics`: number of row groups that were tested by row group statistics (min and max value), and did not pass
-- `pushdown_rows_matched`: rows that were tested by any of the above filtered, and passed all of them (this should be minimum of `page_index_rows_matched`, `row_groups_pruned_bloom_filter`, and `row_groups_pruned_statistics`)
-- `pushdown_rows_pruned`: rows that were tested by any of the above filtered, and did not pass one of them (this should be sum of `page_index_rows_matched`, `row_groups_pruned_bloom_filter`, and `row_groups_pruned_statistics`)
+- `page_index_rows_pruned`: number of rows evaluated by page index filters. The metric reports both how many rows were considered in total and how many matched (were not pruned).
+- `row_groups_pruned_bloom_filter`: number of row groups evaluated by Bloom Filters, reporting both total checked groups and groups that matched.
+- `row_groups_pruned_statistics`: number of row groups evaluated by row-group statistics (min/max), reporting both total checked groups and groups that matched.
+- `pushdown_rows_matched`: rows that were tested by any of the above filters, and passed all of them.
+- `pushdown_rows_pruned`: rows that were tested by any of the above filters, and did not pass at least one of them.
- `predicate_evaluation_errors`: number of times evaluating the filter expression failed (expected to be zero in normal operation)
- `num_predicate_creation_errors`: number of errors creating predicates (expected to be zero in normal operation)
- `bloom_filter_eval_time`: time spent parsing and evaluating Bloom Filters