Apache DataFusion SQL Query Engine
feat: Add per-expression evaluation timing metrics to ProjectionExec (#19447)
Yongting You36df145e9761
diff --git a/Cargo.lock b/Cargo.lockindex 22ec582536..d3213a576b 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2527,10 +2527,12 @@ version = "51.0.0"dependencies = ["ahash 0.8.12","arrow",+ "chrono","datafusion-common","datafusion-expr-common","hashbrown 0.14.5","itertools 0.14.0",+ "parking_lot",][[package]]diff --git a/datafusion/execution/src/lib.rs b/datafusion/execution/src/lib.rsindex 2ea8182823..aced2f46d7 100644--- a/datafusion/execution/src/lib.rs+++ b/datafusion/execution/src/lib.rs@@ -32,7 +32,6 @@ pub mod cache;pub mod config;pub mod disk_manager;pub mod memory_pool;-pub mod metrics;pub mod object_store;#[cfg(feature = "parquet_encryption")]pub mod parquet_encryption;diff --git a/datafusion/physical-expr-common/Cargo.toml b/datafusion/physical-expr-common/Cargo.tomlindex 4602e59c42..d292da212e 100644--- a/datafusion/physical-expr-common/Cargo.toml+++ b/datafusion/physical-expr-common/Cargo.toml@@ -43,7 +43,9 @@ name = "datafusion_physical_expr_common"[dependencies]ahash = { workspace = true }arrow = { workspace = true }+chrono = { workspace = true }datafusion-common = { workspace = true }datafusion-expr-common = { workspace = true }hashbrown = { workspace = true }itertools = { workspace = true }+parking_lot = { workspace = true }diff --git a/datafusion/physical-expr-common/src/lib.rs b/datafusion/physical-expr-common/src/lib.rsindex 7665e45613..84378a3d26 100644--- a/datafusion/physical-expr-common/src/lib.rs+++ b/datafusion/physical-expr-common/src/lib.rs@@ -35,6 +35,7 @@pub mod binary_map;pub mod binary_view_map;pub mod datum;+pub mod metrics;pub mod physical_expr;pub mod sort_expr;pub mod tree_node;diff --git a/datafusion/execution/src/metrics/baseline.rs b/datafusion/physical-expr-common/src/metrics/baseline.rssimilarity index 98%rename from datafusion/execution/src/metrics/baseline.rsrename to datafusion/physical-expr-common/src/metrics/baseline.rsindex 91eff2d209..0de8e26494 100644--- a/datafusion/execution/src/metrics/baseline.rs+++ b/datafusion/physical-expr-common/src/metrics/baseline.rs@@ -29,7 +29,9 @@ use super::{Count, ExecutionPlanMetricsSet, MetricBuilder, Time, Timestamp};////// Example:/// ```-/// use datafusion_execution::metrics::{BaselineMetrics, ExecutionPlanMetricsSet};+/// use datafusion_physical_expr_common::metrics::{+/// BaselineMetrics, ExecutionPlanMetricsSet,+/// };/// let metrics = ExecutionPlanMetricsSet::new();////// let partition = 2;diff --git a/datafusion/execution/src/metrics/builder.rs b/datafusion/physical-expr-common/src/metrics/builder.rssimilarity index 99%rename from datafusion/execution/src/metrics/builder.rsrename to datafusion/physical-expr-common/src/metrics/builder.rsindex 63c18d7253..4fa938f69e 100644--- a/datafusion/execution/src/metrics/builder.rs+++ b/datafusion/physical-expr-common/src/metrics/builder.rs@@ -34,7 +34,7 @@ use super::{/// case of constant strings////// ```rust-/// use datafusion_execution::metrics::*;+/// use datafusion_physical_expr_common::metrics::*;////// let metrics = ExecutionPlanMetricsSet::new();/// let partition = 1;diff --git a/datafusion/execution/src/metrics/custom.rs b/datafusion/physical-expr-common/src/metrics/custom.rssimilarity index 98%rename from datafusion/execution/src/metrics/custom.rsrename to datafusion/physical-expr-common/src/metrics/custom.rsindex 8062bbbf38..0bd7ba1b10 100644--- a/datafusion/execution/src/metrics/custom.rs+++ b/datafusion/physical-expr-common/src/metrics/custom.rs@@ -44,7 +44,7 @@ use std::{any::Any, fmt::Debug, fmt::Display, sync::Arc};/// # use std::any::Any;/// # use std::sync::atomic::{AtomicUsize, Ordering};///-/// # use datafusion_execution::metrics::CustomMetricValue;+/// # use datafusion_physical_expr_common::metrics::CustomMetricValue;////// #[derive(Debug, Default)]/// struct MyCounter {diff --git a/datafusion/physical-expr-common/src/metrics/expression.rs b/datafusion/physical-expr-common/src/metrics/expression.rsnew file mode 100644index 0000000000..4a092b0d1b--- /dev/null+++ b/datafusion/physical-expr-common/src/metrics/expression.rs@@ -0,0 +1,88 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Metrics helpers for expression evaluation.++use super::{ExecutionPlanMetricsSet, MetricBuilder, MetricType, ScopedTimerGuard, Time};++/// Tracks evaluation time for a sequence of expressions.+///+/// # Example+/// Given SQL query:+/// EXPLAIN ANALYZE+/// SELECT a+1, pow(a,2)+/// FROM generate_series(1, 1000000) as t1(a)+///+/// This struct holds two time metrics for the projection expressions+/// `a+1` and `pow(a,2)`, respectively.+///+/// The output reads:+/// `ProjectionExec: expr=[a@0 + 1 as t1.a + Int64(1), power(CAST(a@0 AS Float64), 2) as pow(t1.a,Int64(2))], metrics=[... expr_0_eval_time=9.23ms, expr_1_eval_time=32.35ms...]`+#[derive(Debug, Clone)]+pub struct ExpressionEvaluatorMetrics {+ expression_times: Vec<Time>,+}++impl ExpressionEvaluatorMetrics {+ /// Create metrics for a collection of expressions.+ ///+ /// # Args+ /// - metrics: see `MetricBuilder` for details.+ /// - partition: see `MetricBuilder` for details.+ /// - expression_labels: unique identifier for each metric, so that the metric+ /// can get aggregated across multiple partitions. It is not the name showed+ /// in the `EXPLAIN ANALYZE`, the metric name will be `expr_{idx}_eval_time`+ /// according to the expression order.+ pub fn new<T>(+ metrics: &ExecutionPlanMetricsSet,+ partition: usize,+ expression_labels: impl IntoIterator<Item = T>,+ ) -> Self+ where+ T: Into<String>,+ {+ let expression_times = expression_labels+ .into_iter()+ .enumerate()+ .map(|(idx, label)| {+ MetricBuilder::new(metrics)+ .with_new_label("expr", label.into())+ .with_type(MetricType::DEV)+ // Existing PhysicalExpr formatter is a bit verbose, so use simple name+ .subset_time(format!("expr_{idx}_eval_time"), partition)+ })+ .collect();++ Self { expression_times }+ }++ /// Returns a timer guard for the expression at `index`, if present.+ #[inline]+ pub fn scoped_timer(&self, index: usize) -> Option<ScopedTimerGuard<'_>> {+ self.expression_times.get(index).map(Time::timer)+ }++ /// The number of tracked expressions.+ pub fn len(&self) -> usize {+ self.expression_times.len()+ }++ /// True when no expressions are tracked.+ pub fn is_empty(&self) -> bool {+ self.expression_times.is_empty()+ }+}diff --git a/datafusion/execution/src/metrics/mod.rs b/datafusion/physical-expr-common/src/metrics/mod.rssimilarity index 99%rename from datafusion/execution/src/metrics/mod.rsrename to datafusion/physical-expr-common/src/metrics/mod.rsindex e76ba1de38..18dafa4127 100644--- a/datafusion/execution/src/metrics/mod.rs+++ b/datafusion/physical-expr-common/src/metrics/mod.rs@@ -20,6 +20,7 @@mod baseline;mod builder;mod custom;+mod expression;mod value;use datafusion_common::HashMap;@@ -35,6 +36,7 @@ use std::{pub use baseline::{BaselineMetrics, RecordOutput, SpillMetrics, SplitMetrics};pub use builder::MetricBuilder;pub use custom::CustomMetricValue;+pub use expression::ExpressionEvaluatorMetrics;pub use value::{Count, Gauge, MetricValue, PruningMetrics, RatioMergeStrategy, RatioMetrics,ScopedTimerGuard, Time, Timestamp,@@ -47,7 +49,7 @@ pub use value::{/// [`ExecutionPlanMetricsSet`].////// ```-/// use datafusion_execution::metrics::*;+/// use datafusion_physical_expr_common::metrics::*;////// let metrics = ExecutionPlanMetricsSet::new();/// assert!(metrics.clone_inner().output_rows().is_none());diff --git a/datafusion/execution/src/metrics/value.rs b/datafusion/physical-expr-common/src/metrics/value.rssimilarity index 100%rename from datafusion/execution/src/metrics/value.rsrename to datafusion/physical-expr-common/src/metrics/value.rsdiff --git a/datafusion/physical-expr-common/src/utils.rs b/datafusion/physical-expr-common/src/utils.rsindex 8992fb2092..278294a9bf 100644--- a/datafusion/physical-expr-common/src/utils.rs+++ b/datafusion/physical-expr-common/src/utils.rs@@ -17,6 +17,7 @@use std::sync::Arc;+use crate::metrics::ExpressionEvaluatorMetrics;use crate::physical_expr::PhysicalExpr;use crate::tree_node::ExprContext;@@ -101,11 +102,25 @@ pub fn scatter(mask: &BooleanArray, truthy: &dyn Array) -> Result<ArrayRef> {pub fn evaluate_expressions_to_arrays<'a>(exprs: impl IntoIterator<Item = &'a Arc<dyn PhysicalExpr>>,batch: &RecordBatch,+) -> Result<Vec<ArrayRef>> {+ evaluate_expressions_to_arrays_with_metrics(exprs, batch, None)+}++/// Same as [`evaluate_expressions_to_arrays`] but records optional per-expression metrics.+///+/// For metrics tracking, see [`ExpressionEvaluatorMetrics`] for details.+#[inline]+pub fn evaluate_expressions_to_arrays_with_metrics<'a>(+ exprs: impl IntoIterator<Item = &'a Arc<dyn PhysicalExpr>>,+ batch: &RecordBatch,+ metrics: Option<&ExpressionEvaluatorMetrics>,) -> Result<Vec<ArrayRef>> {let num_rows = batch.num_rows();exprs.into_iter()- .map(|e| {+ .enumerate()+ .map(|(idx, e)| {+ let _timer = metrics.and_then(|m| m.scoped_timer(idx));e.evaluate(batch).and_then(|col| col.into_array_of_size(num_rows))})diff --git a/datafusion/physical-expr/src/projection.rs b/datafusion/physical-expr/src/projection.rsindex c46df87fd8..8d4afb5d19 100644--- a/datafusion/physical-expr/src/projection.rs+++ b/datafusion/physical-expr/src/projection.rs@@ -32,8 +32,11 @@ use datafusion_common::{Result, assert_or_internal_err, internal_datafusion_err, plan_err,};+use datafusion_physical_expr_common::metrics::ExecutionPlanMetricsSet;+use datafusion_physical_expr_common::metrics::ExpressionEvaluatorMetrics;+use datafusion_physical_expr_common::physical_expr::fmt_sql;use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};-use datafusion_physical_expr_common::utils::evaluate_expressions_to_arrays;+use datafusion_physical_expr_common::utils::evaluate_expressions_to_arrays_with_metrics;use indexmap::IndexMap;use itertools::Itertools;@@ -481,9 +484,30 @@ impl ProjectionExprs {Ok(Projector {projection: self.clone(),output_schema,+ expression_metrics: None,})}+ pub fn create_expression_metrics(+ &self,+ metrics: &ExecutionPlanMetricsSet,+ partition: usize,+ ) -> ExpressionEvaluatorMetrics {+ let labels: Vec<String> = self+ .exprs+ .iter()+ .map(|proj_expr| {+ let expr_sql = fmt_sql(proj_expr.expr.as_ref()).to_string();+ if proj_expr.expr.to_string() == proj_expr.alias {+ expr_sql+ } else {+ format!("{expr_sql} AS {}", proj_expr.alias)+ }+ })+ .collect();+ ExpressionEvaluatorMetrics::new(metrics, partition, labels)+ }+/// Project statistics according to this projection./// For example, for a projection `SELECT a AS x, b + 1 AS y`, where `a` is at index 0 and `b` is at index 1,/// if the input statistics has column statistics for columns `a`, `b`, and `c`, the output statistics would have column statistics for columns `x` and `y`.@@ -621,9 +645,30 @@ impl<'a> IntoIterator for &'a ProjectionExprs {pub struct Projector {projection: ProjectionExprs,output_schema: SchemaRef,+ /// If `Some`, metrics will be tracked for projection evaluation.+ expression_metrics: Option<ExpressionEvaluatorMetrics>,}impl Projector {+ /// Construct the projector with metrics. After execution, related metrics will+ /// be tracked inside `ExecutionPlanMetricsSet`+ ///+ /// See [`ExpressionEvaluatorMetrics`] for details.+ pub fn with_metrics(+ &self,+ metrics: &ExecutionPlanMetricsSet,+ partition: usize,+ ) -> Self {+ let expr_metrics = self+ .projection+ .create_expression_metrics(metrics, partition);+ Self {+ expression_metrics: Some(expr_metrics),+ projection: self.projection.clone(),+ output_schema: Arc::clone(&self.output_schema),+ }+ }+/// Project a record batch according to this projector's expressions.////// # Errors@@ -631,9 +676,10 @@ impl Projector {/// or if the output schema of the resulting record batch does not match/// the pre-computed output schema of the projector.pub fn project_batch(&self, batch: &RecordBatch) -> Result<RecordBatch> {- let arrays = evaluate_expressions_to_arrays(+ let arrays = evaluate_expressions_to_arrays_with_metrics(self.projection.exprs.iter().map(|p| &p.expr),batch,+ self.expression_metrics.as_ref(),)?;if arrays.is_empty() {diff --git a/datafusion/physical-plan/src/metrics.rs b/datafusion/physical-plan/src/metrics.rsindex 267a3e995d..fe17cbdd4a 100644--- a/datafusion/physical-plan/src/metrics.rs+++ b/datafusion/physical-plan/src/metrics.rs@@ -15,7 +15,7 @@// specific language governing permissions and limitations// under the License.-//! Metrics live in `datafusion-execution`; this module re-exports them to keep-//! the public APIs stable.+//! Metrics live in `datafusion-physical-expr-common`; this module re-exports+//! them to keep the public APIs stable.-pub use datafusion_execution::metrics::*;+pub use datafusion_physical_expr_common::metrics::*;diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rsindex 135f78677f..e8608f17a1 100644--- a/datafusion/physical-plan/src/projection.rs+++ b/datafusion/physical-plan/src/projection.rs@@ -295,8 +295,10 @@ impl ExecutionPlan for ProjectionExec {context.session_id(),context.task_id());++ let projector = self.projector.with_metrics(&self.metrics, partition);Ok(Box::pin(ProjectionStream::new(- self.projector.clone(),+ projector,self.input.execute(partition, context)?,BaselineMetrics::new(&self.metrics, partition),)?))@@ -1070,6 +1072,7 @@ mod tests {use std::sync::Arc;use crate::common::collect;+use crate::test;use crate::test::exec::StatisticsExec;diff --git a/datafusion/sqllogictest/test_files/explain_analyze.slt b/datafusion/sqllogictest/test_files/explain_analyze.sltindex b213cd9565..e109b32a95 100644--- a/datafusion/sqllogictest/test_files/explain_analyze.slt+++ b/datafusion/sqllogictest/test_files/explain_analyze.slt@@ -23,5 +23,46 @@ EXPLAIN ANALYZE SELECT * FROM generate_series(100);----Plan with Metrics LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=0, end=100, batch_size=8192], metrics=[output_rows=101, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>]+# --------------------------------------------+# Test ProjectionExec's per-expression metrics+# --------------------------------------------++statement ok+set datafusion.explain.analyze_level = dev;++# 1 expr+# Expect metric `expr_0_eval_time` exists in ProjectionExec+query TT+EXPLAIN ANALYZE+SELECT a+FROM generate_series(1, 100) as t1(a);+----+Plan with Metrics+01)ProjectionExec: expr=[value@0 as a], metrics=[output_rows=100, elapsed_compute=<slt:ignore>, output_bytes=64.0 KB, output_batches=1, expr_0_eval_time=<slt:ignore>]+<slt:ignore>++# 2 exprs+# Expect metrics `expr_0_eval_time` and `expr_1_eval_time` exist in ProjectionExec+query TT+EXPLAIN ANALYZE+SELECT a+1, pow(a,2)+FROM generate_series(1, 100) as t1(a);+----+Plan with Metrics+01)ProjectionExec: expr=[a@0 + 1 as t1.a + Int64(1), power(CAST(a@0 AS Float64), 2) as pow(t1.a,Int64(2))], metrics=[output_rows=100, elapsed_compute=<slt:ignore>, output_bytes=1632.0 B, output_batches=1, expr_0_eval_time=<slt:ignore>, expr_1_eval_time=<slt:ignore>]+<slt:ignore>++# common expressions+# Expect metrics `expr_0_eval_time` and `expr_1_eval_time` exist in ProjectionExec+query TT+EXPLAIN ANALYZE+SELECT a+1, a+1 as another_a_plus_one+FROM generate_series(1, 100) as t1(a);+----+Plan with Metrics+01)ProjectionExec: expr=[__common_expr_1@0 as t1.a + Int64(1), __common_expr_1@0 as another_a_plus_one], metrics=[output_rows=100, elapsed_compute=<slt:ignore>, output_bytes=800.0 B, output_batches=1, expr_0_eval_time=<slt:ignore>, expr_1_eval_time=<slt:ignore>]+02)--ProjectionExec: expr=[a@0 + 1 as __common_expr_1], metrics=[output_rows=100, elapsed_compute=<slt:ignore>, output_bytes=800.0 B, output_batches=1, expr_0_eval_time=<slt:ignore>]+<slt:ignore>+statement okreset datafusion.explain.analyze_level;