Apache DataFusion SQL Query Engine
fix: predicate cache stats calculation (#19561)
feniljain5e893ab92193
diff --git a/datafusion/core/tests/parquet/filter_pushdown.rs b/datafusion/core/tests/parquet/filter_pushdown.rsindex e3a191ee9a..f84f91f879 100644--- a/datafusion/core/tests/parquet/filter_pushdown.rs+++ b/datafusion/core/tests/parquet/filter_pushdown.rs@@ -644,6 +644,22 @@ async fn predicate_cache_pushdown_default() -> datafusion_common::Result<()> {.await}+#[tokio::test]+async fn predicate_cache_stats_issue_19561() -> datafusion_common::Result<()> {+ let mut config = SessionConfig::new();+ config.options_mut().execution.parquet.pushdown_filters = true;+ // force to get multiple batches to trigger repeated metric compound bug+ config.options_mut().execution.batch_size = 1;+ let ctx = SessionContext::new_with_config(config);+ // The cache is on by default, and used when filter pushdown is enabled+ PredicateCacheTest {+ expected_inner_records: 8,+ expected_records: 4,+ }+ .run(&ctx)+ .await+}+#[tokio::test]async fn predicate_cache_pushdown_default_selections_only()-> datafusion_common::Result<()> {diff --git a/datafusion/datasource-parquet/src/metrics.rs b/datafusion/datasource-parquet/src/metrics.rsindex fbb14d9a6d..1e34b324a5 100644--- a/datafusion/datasource-parquet/src/metrics.rs+++ b/datafusion/datasource-parquet/src/metrics.rs@@ -16,7 +16,7 @@// under the License.use datafusion_physical_plan::metrics::{- Count, ExecutionPlanMetricsSet, MetricBuilder, MetricType, PruningMetrics,+ Count, ExecutionPlanMetricsSet, Gauge, MetricBuilder, MetricType, PruningMetrics,RatioMergeStrategy, RatioMetrics, Time,};@@ -79,11 +79,16 @@ pub struct ParquetFileMetrics {/// Parquet.////// This is the expensive path (IO + Decompression + Decoding).- pub predicate_cache_inner_records: Count,+ ///+ /// We use a Gauge here as arrow-rs reports absolute numbers rather+ /// than incremental readings, we want a `set` operation here rather+ /// than `add`. Earlier it was `Count`, which led to this issue:+ /// github.com/apache/datafusion/issues/19334+ pub predicate_cache_inner_records: Gauge,/// Predicate Cache: number of records read from the cache. This is the/// number of rows that were stored in the cache after evaluating predicates/// reused for the output.- pub predicate_cache_records: Count,+ pub predicate_cache_records: Gauge,}impl ParquetFileMetrics {@@ -169,11 +174,11 @@ impl ParquetFileMetrics {let predicate_cache_inner_records = MetricBuilder::new(metrics).with_new_label("filename", filename.to_string())- .counter("predicate_cache_inner_records", partition);+ .gauge("predicate_cache_inner_records", partition);let predicate_cache_records = MetricBuilder::new(metrics).with_new_label("filename", filename.to_string())- .counter("predicate_cache_records", partition);+ .gauge("predicate_cache_records", partition);Self {files_ranges_pruned_statistics,diff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rsindex 8f31d2df24..e79483ce79 100644--- a/datafusion/datasource-parquet/src/opener.rs+++ b/datafusion/datasource-parquet/src/opener.rs@@ -47,7 +47,7 @@ use datafusion_physical_expr_common::physical_expr::{PhysicalExpr, is_dynamic_physical_expr,};use datafusion_physical_plan::metrics::{- Count, ExecutionPlanMetricsSet, MetricBuilder, PruningMetrics,+ Count, ExecutionPlanMetricsSet, Gauge, MetricBuilder, PruningMetrics,};use datafusion_pruning::{FilePruner, PruningPredicate, build_pruning_predicate};@@ -682,15 +682,15 @@ impl FileOpener for ParquetOpener {/// arrow-rs parquet reader) to the parquet file metrics for DataFusionfn copy_arrow_reader_metrics(arrow_reader_metrics: &ArrowReaderMetrics,- predicate_cache_inner_records: &Count,- predicate_cache_records: &Count,+ predicate_cache_inner_records: &Gauge,+ predicate_cache_records: &Gauge,) {if let Some(v) = arrow_reader_metrics.records_read_from_inner() {- predicate_cache_inner_records.add(v);+ predicate_cache_inner_records.set(v);}if let Some(v) = arrow_reader_metrics.records_read_from_cache() {- predicate_cache_records.add(v);+ predicate_cache_records.set(v);}}