Apache DataFusion SQL Query Engine
feat(small): Set 'summary' level metrics for `DataSourceExec` with parquet source (#18196)
Yongting Youf4a49b538c5d
diff --git a/datafusion/core/tests/sql/explain_analyze.rs b/datafusion/core/tests/sql/explain_analyze.rsindex 54a57ed901..6d386cc456 100644--- a/datafusion/core/tests/sql/explain_analyze.rs+++ b/datafusion/core/tests/sql/explain_analyze.rs@@ -161,22 +161,35 @@ fn nanos_from_timestamp(ts: &Timestamp) -> i64 {}// Test different detail level for config `datafusion.explain.analyze_level`++async fn collect_plan_with_context(+ sql_str: &str,+ ctx: &SessionContext,+ level: ExplainAnalyzeLevel,+) -> String {+ {+ let state = ctx.state_ref();+ let mut state = state.write();+ state.config_mut().options_mut().explain.analyze_level = level;+ }+ let dataframe = ctx.sql(sql_str).await.unwrap();+ let batches = dataframe.collect().await.unwrap();+ arrow::util::pretty::pretty_format_batches(&batches)+ .unwrap()+ .to_string()+}++async fn collect_plan(sql_str: &str, level: ExplainAnalyzeLevel) -> String {+ let ctx = SessionContext::new();+ collect_plan_with_context(sql_str, &ctx, level).await+}+#[tokio::test]async fn explain_analyze_level() {- async fn collect_plan(level: ExplainAnalyzeLevel) -> String {- let mut config = SessionConfig::new();- config.options_mut().explain.analyze_level = level;- let ctx = SessionContext::new_with_config(config);- let sql = "EXPLAIN ANALYZE \+ let sql = "EXPLAIN ANALYZE \SELECT * \FROM generate_series(10) as t1(v1) \ORDER BY v1 DESC";- let dataframe = ctx.sql(sql).await.unwrap();- let batches = dataframe.collect().await.unwrap();- arrow::util::pretty::pretty_format_batches(&batches)- .unwrap()- .to_string()- }for (level, needle, should_contain) in [(ExplainAnalyzeLevel::Summary, "spill_count", false),@@ -184,7 +197,35 @@ async fn explain_analyze_level() {(ExplainAnalyzeLevel::Dev, "spill_count", true),(ExplainAnalyzeLevel::Dev, "output_rows", true),] {- let plan = collect_plan(level).await;+ let plan = collect_plan(sql, level).await;+ assert_eq!(+ plan.contains(needle),+ should_contain,+ "plan for level {level:?} unexpected content: {plan}"+ );+ }+}++#[tokio::test]+async fn explain_analyze_level_datasource_parquet() {+ let table_name = "tpch_lineitem_small";+ let parquet_path = "tests/data/tpch_lineitem_small.parquet";+ let sql = format!("EXPLAIN ANALYZE SELECT * FROM {table_name}");++ // Register test parquet file into context+ let ctx = SessionContext::new();+ ctx.register_parquet(table_name, parquet_path, ParquetReadOptions::default())+ .await+ .expect("register parquet table for explain analyze test");++ for (level, needle, should_contain) in [+ (ExplainAnalyzeLevel::Summary, "metadata_load_time", true),+ (ExplainAnalyzeLevel::Summary, "page_index_eval_time", false),+ (ExplainAnalyzeLevel::Dev, "metadata_load_time", true),+ (ExplainAnalyzeLevel::Dev, "page_index_eval_time", true),+ ] {+ let plan = collect_plan_with_context(&sql, &ctx, level).await;+assert_eq!(plan.contains(needle),should_contain,diff --git a/datafusion/datasource-parquet/src/metrics.rs b/datafusion/datasource-parquet/src/metrics.rsindex d75a979d4c..5f17fbb4b9 100644--- a/datafusion/datasource-parquet/src/metrics.rs+++ b/datafusion/datasource-parquet/src/metrics.rs@@ -16,7 +16,7 @@// under the License.use datafusion_physical_plan::metrics::{- Count, ExecutionPlanMetricsSet, MetricBuilder, Time,+ Count, ExecutionPlanMetricsSet, MetricBuilder, MetricType, Time,};/// Stores metrics about the parquet execution for a particular parquet file.@@ -88,30 +88,59 @@ impl ParquetFileMetrics {filename: &str,metrics: &ExecutionPlanMetricsSet,) -> Self {- let predicate_evaluation_errors = MetricBuilder::new(metrics)- .with_new_label("filename", filename.to_string())- .counter("predicate_evaluation_errors", partition);-+ // -----------------------+ // 'summary' level metrics+ // -----------------------let row_groups_matched_bloom_filter = MetricBuilder::new(metrics).with_new_label("filename", filename.to_string())+ .with_type(MetricType::SUMMARY).counter("row_groups_matched_bloom_filter", partition);let row_groups_pruned_bloom_filter = MetricBuilder::new(metrics).with_new_label("filename", filename.to_string())+ .with_type(MetricType::SUMMARY).counter("row_groups_pruned_bloom_filter", partition);let row_groups_matched_statistics = MetricBuilder::new(metrics).with_new_label("filename", filename.to_string())+ .with_type(MetricType::SUMMARY).counter("row_groups_matched_statistics", partition);let row_groups_pruned_statistics = MetricBuilder::new(metrics).with_new_label("filename", filename.to_string())+ .with_type(MetricType::SUMMARY).counter("row_groups_pruned_statistics", partition);+ let page_index_rows_pruned = MetricBuilder::new(metrics)+ .with_new_label("filename", filename.to_string())+ .with_type(MetricType::SUMMARY)+ .counter("page_index_rows_pruned", partition);+ let page_index_rows_matched = MetricBuilder::new(metrics)+ .with_new_label("filename", filename.to_string())+ .with_type(MetricType::SUMMARY)+ .counter("page_index_rows_matched", partition);+let bytes_scanned = MetricBuilder::new(metrics).with_new_label("filename", filename.to_string())+ .with_type(MetricType::SUMMARY).counter("bytes_scanned", partition);+ let metadata_load_time = MetricBuilder::new(metrics)+ .with_new_label("filename", filename.to_string())+ .with_type(MetricType::SUMMARY)+ .subset_time("metadata_load_time", partition);++ let files_ranges_pruned_statistics = MetricBuilder::new(metrics)+ .with_type(MetricType::SUMMARY)+ .counter("files_ranges_pruned_statistics", partition);++ // -----------------------+ // 'dev' level metrics+ // -----------------------+ let predicate_evaluation_errors = MetricBuilder::new(metrics)+ .with_new_label("filename", filename.to_string())+ .counter("predicate_evaluation_errors", partition);+let pushdown_rows_pruned = MetricBuilder::new(metrics).with_new_label("filename", filename.to_string()).counter("pushdown_rows_pruned", partition);@@ -129,24 +158,10 @@ impl ParquetFileMetrics {.with_new_label("filename", filename.to_string()).subset_time("bloom_filter_eval_time", partition);- let page_index_rows_pruned = MetricBuilder::new(metrics)- .with_new_label("filename", filename.to_string())- .counter("page_index_rows_pruned", partition);- let page_index_rows_matched = MetricBuilder::new(metrics)- .with_new_label("filename", filename.to_string())- .counter("page_index_rows_matched", partition);-let page_index_eval_time = MetricBuilder::new(metrics).with_new_label("filename", filename.to_string()).subset_time("page_index_eval_time", partition);- let metadata_load_time = MetricBuilder::new(metrics)- .with_new_label("filename", filename.to_string())- .subset_time("metadata_load_time", partition);-- let files_ranges_pruned_statistics = MetricBuilder::new(metrics)- .counter("files_ranges_pruned_statistics", partition);-let predicate_cache_inner_records = MetricBuilder::new(metrics).with_new_label("filename", filename.to_string()).counter("predicate_cache_inner_records", partition);