Apache DataFusion SQL Query Engine
feat: add partition-aware metrics snapshots (#25583)
Liang-Chi Hsieh146348b8137e
diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.tomlindex 222c0ec688..7275c4b674 100644--- a/datafusion/core/Cargo.toml+++ b/datafusion/core/Cargo.toml@@ -340,3 +340,7 @@ required-features = ["parquet"][[bench]]harness = falsename = "reset_plan_states"++[[bench]]+harness = false+name = "partition_metrics"diff --git a/datafusion/core/benches/partition_metrics.rs b/datafusion/core/benches/partition_metrics.rsnew file mode 100644index 0000000000..ceef897cd1--- /dev/null+++ b/datafusion/core/benches/partition_metrics.rs@@ -0,0 +1,190 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Execute a SQL query over a partitioned table, then report operator metrics.+//! Run with `cargo bench -p datafusion --bench partition_metrics`.++use std::hint::black_box;+use std::sync::Arc;+use std::time::Duration;++use arrow::array::Int32Array;+use arrow::datatypes::{DataType, Field, Schema};+use arrow::record_batch::RecordBatch;+use criterion::{BatchSize, BenchmarkId, Criterion, criterion_group, criterion_main};+use datafusion::datasource::MemTable;+use datafusion::prelude::{SessionConfig, SessionContext};+use datafusion_common::Result;+use datafusion_physical_plan::ExecutionPlan;+use datafusion_physical_plan::metrics::MetricsSet;+use futures::TryStreamExt;++async fn query(partitions: usize) -> Result<(SessionContext, Arc<dyn ExecutionPlan>)> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(Int32Array::from_iter_values(0..32))],+ )?;+ let table = MemTable::try_new(schema, vec![vec![batch]; partitions])?;+ let ctx = SessionContext::new_with_config(+ SessionConfig::new()+ .with_target_partitions(partitions)+ .with_batch_size(16),+ );+ ctx.register_table("t", Arc::new(table))?;+ let plan = ctx+ .sql("SELECT a + 1 AS b FROM t WHERE a < 16")+ .await?+ .create_physical_plan()+ .await?;+ assert_eq!(plan.properties().partitioning.partition_count(), partitions);+ Ok((ctx, plan))+}++fn nodes(plan: &Arc<dyn ExecutionPlan>) -> Vec<Arc<dyn ExecutionPlan>> {+ let mut result = vec![Arc::clone(plan)];+ for child in plan.children() {+ result.extend(nodes(child));+ }+ result+}++fn select(metrics: &MetricsSet, partition: usize, indexed: bool) -> usize {+ if indexed {+ metrics+ .for_partition(partition)+ .iter()+ .map(|m| {+ black_box(m.value().as_usize());+ 1+ })+ .sum()+ } else {+ metrics+ .iter()+ .filter(|m| m.partition() == Some(partition))+ .map(|m| {+ black_box(m.value().as_usize());+ 1+ })+ .sum()+ }+}++async fn execute(ctx: &SessionContext, plan: &Arc<dyn ExecutionPlan>, partition: usize) {+ let batches: Vec<_> = plan+ .execute(partition, ctx.task_ctx())+ .unwrap()+ .try_collect()+ .await+ .unwrap();+ assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 16);+}++fn benchmarks(c: &mut Criterion) {+ let runtime = tokio::runtime::Runtime::new().unwrap();+ let mut retrieval = c.benchmark_group("partition_metrics/sql_retrieval");+ retrieval+ .sample_size(20)+ .warm_up_time(Duration::from_secs(1))+ .measurement_time(Duration::from_secs(3));+ let mut expected_count = None;+ for partitions in [1, 64, 1024, 8192] {+ let (ctx, plan) = runtime.block_on(query(partitions)).unwrap();+ runtime.block_on(async {+ for p in 0..partitions {+ execute(&ctx, &plan, p).await;+ }+ });+ let nodes = nodes(&plan);+ let count: usize = nodes+ .iter()+ .filter_map(|n| n.metrics())+ .map(|m| m.for_partition(0).iter().count())+ .sum();+ assert!(count > 0);+ assert_eq!(*expected_count.get_or_insert(count), count);+ for (label, indexed) in [("full_then_filter", false), ("partition", true)] {+ retrieval.bench_with_input(+ BenchmarkId::new(label, partitions),+ &indexed,+ |b, &indexed| {+ b.iter(|| {+ for node in &nodes {+ if let Some(metrics) = node.metrics() {+ black_box(select(&metrics, 0, indexed));+ }+ }+ });+ },+ );+ }+ }+ retrieval.finish();++ let mut execution = c.benchmark_group("partition_metrics/sql_execution");+ execution+ .sample_size(10)+ .warm_up_time(Duration::from_secs(1))+ .measurement_time(Duration::from_secs(3));+ for partitions in [64, 1024] {+ for (label, report) in [+ ("execute_only", None),+ ("full_then_filter", Some(false)),+ ("partition", Some(true)),+ ] {+ execution.bench_with_input(+ BenchmarkId::new(label, partitions),+ &report,+ |b, &report| {+ b.iter_batched(+ || runtime.block_on(query(partitions)).unwrap(),+ |(ctx, plan)| {+ let nodes = nodes(&plan);+ runtime.block_on(async {+ let mut previous_snapshots = Vec::new();+ for partition in 0..partitions {+ // Keep the previous snapshots alive while executing and+ // registering the next partition's metrics.+ execute(&ctx, &plan, partition).await;+ if let Some(indexed) = report {+ let snapshots: Vec<_> = nodes+ .iter()+ .filter_map(|n| n.metrics())+ .collect();+ for metrics in &snapshots {+ black_box(select(+ metrics, partition, indexed,+ ));+ }+ black_box(&previous_snapshots);+ previous_snapshots = snapshots;+ }+ }+ });+ },+ BatchSize::PerIteration,+ );+ },+ );+ }+ }+ execution.finish();+}++criterion_group!(benches, benchmarks);+criterion_main!(benches);diff --git a/datafusion/core/src/dataframe/parquet.rs b/datafusion/core/src/dataframe/parquet.rsindex dd751eb977..175e877876 100644--- a/datafusion/core/src/dataframe/parquet.rs+++ b/datafusion/core/src/dataframe/parquet.rs@@ -490,6 +490,21 @@ mod tests {let metrics = plan.metrics().expect("DataSinkExec should return metrics from ParquetSink");+ let selected = plan.metrics().unwrap().for_partition(0);+ let expected: Vec<_> = metrics+ .iter()+ .filter(|metric| metric.partition() == Some(0))+ .collect();+ assert_eq!(selected.iter().count(), expected.len());+ for (actual, expected) in selected.iter().zip(expected) {+ assert!(Arc::ptr_eq(actual, expected));+ }+ // Sink-wide row counts are intentionally absent from a partition snapshot.+ assert!(+ selected+ .iter()+ .all(|metric| metric.value().name() != "rows_written")+ );let aggregated = metrics.aggregate_by_name();// rows_written should be 100diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rsindex 42e7116dff..a5c165abb1 100644--- a/datafusion/datasource-parquet/src/source.rs+++ b/datafusion/datasource-parquet/src/source.rs@@ -1376,6 +1376,42 @@ mod tests {use arrow::datatypes::Schema;use datafusion_physical_expr::expressions::lit;+ #[test]+ fn partition_metrics_exclude_derived_plan_metrics() {+ use datafusion_datasource::file_scan_config::FileScanConfigBuilder;+ use datafusion_datasource::source::DataSourceExec;+ use datafusion_execution::object_store::ObjectStoreUrl;+ use datafusion_physical_plan::ExecutionPlan;+ use datafusion_physical_plan::metrics::MetricBuilder;++ let source = Arc::new(ParquetSource::new(Arc::new(Schema::empty())));+ let metrics = source.metrics().clone();+ let config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), source)+ .build();+ let plan: Arc<dyn ExecutionPlan> = DataSourceExec::from_data_source(config);+ assert_eq!(plan.metrics().unwrap().for_partition(0).iter().count(), 0);+ MetricBuilder::new(&metrics).output_rows(0).add(10);+ MetricBuilder::new(&metrics).output_rows(1).add(20);+ MetricBuilder::new(&metrics).global_counter("global").add(1);+ let selected = plan.metrics().unwrap().for_partition(0);+ assert_eq!(selected.output_rows(), Some(10));+ assert!(selected.iter().all(|m| m.partition() == Some(0)));+ let full = plan.metrics().unwrap();+ assert_eq!(full.output_rows(), Some(30));+ assert!(+ full.iter().any(+ |m| m.value().name() == "output_rows_skew" && m.partition().is_none()+ )+ );+ MetricBuilder::new(&metrics).output_rows(0).add(5);+ assert_eq!(selected.output_rows(), Some(10));+ assert_eq!(+ plan.metrics().unwrap().for_partition(0).output_rows(),+ Some(15)+ );+ }+#[test]fn test_reverse_scan_default_value() {use arrow::datatypes::Schema;diff --git a/datafusion/ffi/src/execution_plan.rs b/datafusion/ffi/src/execution_plan.rsindex b242d1b275..de175e7686 100644--- a/datafusion/ffi/src/execution_plan.rs+++ b/datafusion/ffi/src/execution_plan.rs@@ -872,6 +872,9 @@ pub mod tests {let observed = metric_foreign.metrics().expect("metrics should be present");assert_eq!(observed.output_rows(), Some(42));+ assert_eq!(observed.for_partition(0).output_rows(), Some(11));+ assert_eq!(observed.for_partition(1).output_rows(), Some(31));+ assert_eq!(observed.for_partition(2).iter().count(), 0);Ok(())}diff --git a/datafusion/ffi/tests/ffi_execution_plan.rs b/datafusion/ffi/tests/ffi_execution_plan.rsindex 43a50f1286..6b2cbc380f 100644--- a/datafusion/ffi/tests/ffi_execution_plan.rs+++ b/datafusion/ffi/tests/ffi_execution_plan.rs@@ -83,6 +83,17 @@ mod tests {// across that boundary - not just the in-process From conversions// covered by physical_expr::metrics's roundtrip tests.let metrics = plan.metrics().expect("plan should report metrics");+ let selected = metrics.for_partition(0);+ let expected: Vec<_> = metrics+ .iter()+ .filter(|m| m.partition() == Some(0))+ .collect();+ assert!(!expected.is_empty());+ assert_eq!(selected.iter().count(), expected.len());+ for (actual, expected) in selected.iter().zip(expected) {+ assert!(Arc::ptr_eq(actual, expected));+ }+ assert_eq!(metrics.for_partition(usize::MAX).iter().count(), 0);// Assert the transported Bytes category, the generic variant/name/// value, and (below) the byte-formatted display output - MetricValuediff --git a/datafusion/physical-expr-common/src/metrics/mod.rs b/datafusion/physical-expr-common/src/metrics/mod.rsindex c00fcff705..7b75dfdd88 100644--- a/datafusion/physical-expr-common/src/metrics/mod.rs+++ b/datafusion/physical-expr-common/src/metrics/mod.rs@@ -22,12 +22,14 @@ mod builder;mod custom;mod elapsed_compute;mod expression;+mod snapshot;mod value;use datafusion_common::HashMap;pub use datafusion_common::format::{MetricCategory, MetricType};use datafusion_common::human_readable_size;use parking_lot::Mutex;+use snapshot::{Registry, Snapshot};use std::{borrow::Cow,fmt::{self, Debug, Display},@@ -228,9 +230,17 @@ impl Metric {}/// A snapshot of the metrics for a particular execution plan.+///+/// The set's members remain fixed as execution registers more metrics, but their+/// values continue to reflect execution progress. Use [`Self::for_partition`] to+/// select the metrics belonging to one partition.+///+/// Snapshots from [`ExecutionPlanMetricsSet::clone_inner`] retain the source+/// registry, including later registrations, until dropped. A partition selection+/// retains only the selected metrics.#[derive(Default, Debug, Clone)]pub struct MetricsSet {- metrics: Vec<Arc<Metric>>,+ metrics: Snapshot,}impl MetricsSet {@@ -239,11 +249,23 @@ impl MetricsSet {Default::default()}- /// Add the specified metric+ /// Add the specified metric without changing other snapshots.pub fn push(&mut self, metric: Arc<Metric>) {self.metrics.push(metric)}+ /// Return the metrics whose partition ID equals `partition`.+ ///+ /// Unpartitioned metrics are excluded; an unknown partition yields an empty+ /// set. Registration order and duplicates are preserved. Selected metrics+ /// share their current values with this set, but changes to either set's+ /// membership do not affect the other.+ pub fn for_partition(&self, partition: usize) -> Self {+ Self {+ metrics: self.metrics.for_partition(partition),+ }+ }+/// Returns an iterator across all metricspub fn iter(&self) -> impl Iterator<Item = &Arc<Metric>> {self.metrics.iter()@@ -360,10 +382,7 @@ impl MetricsSet {});}- let new_metrics = map- .into_iter()- .map(|(_k, v)| Arc::new(v))- .collect::<Vec<_>>();+ let new_metrics = map.into_iter().map(|(_k, v)| Arc::new(v)).collect();Self {metrics: new_metrics,@@ -371,14 +390,15 @@ impl MetricsSet {}/// Sort the order of metrics so the "most useful" show up first- pub fn sorted_for_display(mut self) -> Self {- self.metrics.sort_unstable_by_key(|metric| {+ pub fn sorted_for_display(self) -> Self {+ let mut metrics: Vec<_> = self.into_iter().collect();+ metrics.sort_unstable_by_key(|metric| {(metric.value().display_sort_key(),metric.value().name().to_owned(),)});- self+ metrics.into_iter().collect()}/// Remove all timestamp metrics (for more compact display)@@ -388,7 +408,7 @@ impl MetricsSet {let metrics = metrics.into_iter().filter(|m| !m.value.is_timestamp())- .collect::<Vec<_>>();+ .collect();Self { metrics }}@@ -397,14 +417,14 @@ impl MetricsSet {/// [`MetricType`] appears in `allowed`.pub fn filter_by_metric_types(self, allowed: &[MetricType]) -> Self {if allowed.is_empty() {- return Self { metrics: vec![] };+ return Self::new();}let metrics = self.metrics.into_iter().filter(|metric| allowed.contains(&metric.metric_type()))- .collect::<Vec<_>>();+ .collect();Self { metrics }}@@ -418,7 +438,7 @@ impl MetricsSet {/// removed.pub fn filter_by_categories(self, allowed: &[MetricCategory]) -> Self {if allowed.is_empty() {- return Self { metrics: vec![] };+ return Self::new();}let metrics = self@@ -430,7 +450,7 @@ impl MetricsSet {.unwrap_or(MetricCategory::Uncategorized);allowed.contains(&cat)})- .collect::<Vec<_>>();+ .collect();Self { metrics }}@@ -438,14 +458,14 @@ impl MetricsSet {/// Only metrics with the names appearing the list will be kept.pub fn filter_by_names(self, names: &[String]) -> Self {if names.is_empty() {- return Self { metrics: vec![] };+ return Self::new();}let metrics = self.metrics.into_iter().filter(|metric| names.iter().any(|name| name == metric.value().name()))- .collect::<Vec<_>>();+ .collect();Self { metrics }}}@@ -509,33 +529,35 @@ impl FromIterator<Arc<Metric>> for MetricsSet {/// underlying metrics set#[derive(Default, Debug, Clone)]pub struct ExecutionPlanMetricsSet {- inner: Arc<Mutex<MetricsSet>>,+ inner: Arc<Mutex<Registry>>,}impl ExecutionPlanMetricsSet {/// Create a new empty shared metrics setpub fn new() -> Self {- Self {- inner: Arc::new(Mutex::new(MetricsSet::new())),- }+ Self::default()}/// Add the specified metric to the underlying metric setpub fn register(&self, metric: Arc<Metric>) {- self.inner.lock().push(metric)+ self.inner.lock().register(metric)}- /// Return a clone of the inner [`MetricsSet`]+ /// Return a snapshot of the currently registered metrics.+ ///+ /// Later registrations do not appear in this snapshot, but changes to the+ /// values of existing metrics remain visible.pub fn clone_inner(&self) -> MetricsSet {- let guard = self.inner.lock();- (*guard).clone()+ MetricsSet {+ metrics: Snapshot::new(Arc::clone(&self.inner)),+ }}}impl From<MetricsSet> for ExecutionPlanMetricsSet {fn from(metrics: MetricsSet) -> Self {Self {- inner: Arc::new(Mutex::new(metrics)),+ inner: Arc::new(Mutex::new(Registry::new(metrics.into_iter().collect()))),}}}@@ -737,6 +759,134 @@ mod tests {assert_eq!(borrowed.to_string(), shared.to_string());}+ #[test]+ fn selected_snapshot_mutations_are_independent() {+ let registry = ExecutionPlanMetricsSet::new();+ MetricBuilder::new(®istry).output_rows(0).add(3);+ MetricBuilder::new(®istry).output_rows(1).add(7);+ MetricBuilder::new(®istry)+ .global_counter("global")+ .add(11);+ let full = registry.clone_inner();+ let selected = full.for_partition(0);+ assert_eq!(selected.for_partition(0).output_rows(), Some(3));+ assert_eq!(selected.for_partition(1).iter().count(), 0);+ let mut modified = selected.clone();+ let count = Count::new();+ count.add(13);+ modified.push(Arc::new(Metric::new(+ MetricValue::OutputRows(count),+ Some(1),+ )));+ assert_eq!(modified.output_rows(), Some(16));+ assert_eq!(modified.for_partition(1).output_rows(), Some(13));+ assert_eq!(selected.output_rows(), Some(3));+ assert_eq!(full.output_rows(), Some(10));+ assert_eq!(registry.clone_inner().iter().count(), 3);+ assert_eq!(modified.clone().into_iter().count(), 2);+ assert_eq!(+ modified.sorted_for_display().for_partition(0).output_rows(),+ Some(3)+ );+ }++ #[test]+ fn partition_snapshots_preserve_registration_and_shared_values() {+ let metrics = ExecutionPlanMetricsSet::new();+ assert_eq!(metrics.clone_inner().for_partition(0).iter().count(), 0);+ let first = MetricBuilder::new(&metrics).output_rows(0);+ first.add(11);+ MetricBuilder::new(&metrics).global_counter("global").add(7);+ MetricBuilder::new(&metrics).output_rows(usize::MAX).add(99);+ // Same name and partition must not overwrite the earlier metric.+ MetricBuilder::new(&metrics).output_rows(0).add(13);+ let snapshot = metrics.clone_inner().for_partition(0);+ assert_eq!(snapshot.output_rows(), Some(24));+ assert_eq!(snapshot.iter().count(), 2);+ assert_eq!(snapshot.aggregate_by_name().output_rows(), Some(24));+ assert_eq!(+ metrics+ .clone_inner()+ .for_partition(usize::MAX)+ .output_rows(),+ Some(99)+ );+ assert_eq!(metrics.clone_inner().for_partition(1).iter().count(), 0);++ let shared = metrics.clone();+ first.add(1);+ MetricBuilder::new(&shared).output_rows(0).add(17);+ assert_eq!(snapshot.output_rows(), Some(25));+ assert_eq!(+ shared.clone_inner().for_partition(0).output_rows(),+ Some(42)+ );+ assert_eq!(+ metrics.clone_inner().for_partition(0).output_rows(),+ Some(42)+ );++ let full = metrics.clone_inner();+ let imported = ExecutionPlanMetricsSet::from(full.clone());+ for (original, copied) in full.iter().zip(imported.clone_inner().iter()) {+ assert!(Arc::ptr_eq(original, copied));+ }+ for partition in [0, 1, usize::MAX] {+ let expected: Vec<_> = full+ .iter()+ .filter(|metric| metric.partition() == Some(partition))+ .collect();+ let selected = imported.clone_inner().for_partition(partition);+ assert_eq!(expected.len(), selected.iter().count());+ for (original, copied) in expected.into_iter().zip(selected.iter()) {+ assert!(Arc::ptr_eq(original, copied));+ }+ }+ // From shares metric values, but creates an independent registration set.+ MetricBuilder::new(&imported).output_rows(0).add(3);+ assert_eq!(+ imported.clone_inner().for_partition(0).output_rows(),+ Some(45)+ );+ assert_eq!(+ metrics.clone_inner().for_partition(0).output_rows(),+ Some(42)+ );+ assert_eq!(full.iter().filter(|m| m.partition().is_none()).count(), 1);+ }++ #[test]+ fn partition_snapshots_during_registration() {+ let metrics = ExecutionPlanMetricsSet::new();+ let barrier = std::sync::Barrier::new(5);+ std::thread::scope(|scope| {+ for partition in 0..4 {+ let metrics = &metrics;+ let barrier = &barrier;+ scope.spawn(move || {+ barrier.wait();+ for _ in 0..1000 {+ MetricBuilder::new(metrics).output_rows(partition).add(1);+ }+ });+ }+ barrier.wait();+ for _ in 0..1000 {+ for partition in 0..4 {+ let selected = metrics.clone_inner().for_partition(partition);+ assert!(selected.iter().all(|m| m.partition() == Some(partition)));+ assert!(selected.iter().count() <= 1000);+ }+ }+ });+ for partition in 0..4 {+ let selected = metrics.clone_inner().for_partition(partition);+ assert_eq!(selected.iter().count(), 1000);+ assert_eq!(selected.output_rows(), Some(1000));+ }+ assert_eq!(metrics.clone_inner().output_rows(), Some(4000));+ }+#[test]fn test_output_rows() {let metrics = ExecutionPlanMetricsSet::new();diff --git a/datafusion/physical-expr-common/src/metrics/snapshot.rs b/datafusion/physical-expr-common/src/metrics/snapshot.rsnew file mode 100644index 0000000000..c29a924ce5--- /dev/null+++ b/datafusion/physical-expr-common/src/metrics/snapshot.rs@@ -0,0 +1,376 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Metric [`Registry`] and [`Snapshot`] implementations.++use super::Metric;+use parking_lot::Mutex;+use std::collections::HashMap;+use std::fmt;+use std::sync::{Arc, OnceLock};++#[derive(Debug, Default)]+pub(super) struct Registry {+ metrics: Vec<Arc<Metric>>,+ // Readers index each registration once, avoiding repeated full scans when+ // reporting successive partitions. Registration itself only appends to metrics.+ index: PartitionIndex,+}++#[derive(Debug, Default)]+struct PartitionIndex {+ // Number of registry entries already examined, including global metrics.+ indexed: usize,+ // Partition ID -> positions in Registry::metrics, in registration order.+ // Positions also let older snapshots exclude later registrations. Storing+ // handles here would duplicate Arc ownership and still require these positions.+ positions: HashMap<usize, Vec<usize>>,+}++impl Registry {+ pub(super) fn new(metrics: Vec<Arc<Metric>>) -> Self {+ Self {+ metrics,+ index: PartitionIndex::default(),+ }+ }++ pub(super) fn register(&mut self, metric: Arc<Metric>) {+ self.metrics.push(metric);+ }++ /// Select metrics whose partition ID equals `partition`. `end` is an exclusive+ /// position in the full registration vector, including other partitions and+ /// global metrics. It fixes the membership of the requesting snapshot.+ fn select(&mut self, partition: usize, end: usize) -> Vec<Arc<Metric>> {+ let index = &mut self.index;+ for position in index.indexed..end {+ if let Some(partition) = self.metrics[position].partition() {+ index.positions.entry(partition).or_default().push(position);+ }+ }+ index.indexed = index.indexed.max(end);+ let Some(positions) = index.positions.get(&partition) else {+ return Vec::new();+ };+ // Another snapshot may already have indexed registrations after our end.+ let len = positions.partition_point(|&position| position < end);+ positions[..len]+ .iter()+ .map(|&i| Arc::clone(&self.metrics[i]))+ .collect()+ }+}++#[derive(Clone)]+pub(super) enum Snapshot {+ /// All metrics registered before a fixed boundary; partition reads use the index.+ All(Arc<AllMetrics>),+ /// A selected partition, independent of the source registry.+ Partition {+ partition: usize,+ metrics: Vec<Arc<Metric>>,+ },+ /// An arbitrary collection created directly or detached by mutation. It has+ /// no registry/index and may contain multiple partitions and global metrics.+ Standalone(Vec<Arc<Metric>>),+}++pub(super) struct AllMetrics {+ registry: Arc<Mutex<Registry>>,+ // Exclusive position in Registry::metrics, not a per-partition count.+ end: usize,+ flat: OnceLock<Vec<Arc<Metric>>>,+}++impl Default for Snapshot {+ fn default() -> Self {+ Self::Standalone(Vec::new())+ }+}++impl fmt::Debug for Snapshot {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ f.debug_list().entries(self.iter()).finish()+ }+}++impl AllMetrics {+ fn materialize(&self) -> Vec<Arc<Metric>> {+ self.registry.lock().metrics[..self.end].to_vec()+ }+}++impl Snapshot {+ pub(super) fn new(registry: Arc<Mutex<Registry>>) -> Self {+ let end = registry.lock().metrics.len();+ Self::All(Arc::new(AllMetrics {+ registry,+ end,+ flat: OnceLock::new(),+ }))+ }++ pub(super) fn for_partition(&self, partition: usize) -> Self {+ let metrics = match self {+ Self::All(snapshot) => {+ snapshot.registry.lock().select(partition, snapshot.end)+ }+ Self::Partition {+ partition: selected,+ metrics,+ } => {+ if *selected == partition {+ metrics.clone()+ } else {+ Vec::new()+ }+ }+ // Standalone collections have no index to consult. The indexed+ // All path above is used for snapshots from execution-plan registries.+ Self::Standalone(metrics) => metrics+ .iter()+ .filter(|m| m.partition() == Some(partition))+ .cloned()+ .collect(),+ };+ Self::Partition { partition, metrics }+ }++ pub(super) fn push(&mut self, metric: Arc<Metric>) {+ match self {+ Self::Standalone(metrics) => metrics.push(metric),+ Self::Partition { partition, metrics }+ if metric.partition() == Some(*partition) =>+ {+ metrics.push(metric);+ }+ _ => {+ // Detach before mutation: the registry and its index are unchanged.+ // A selected partition can become mixed when another ID is appended.+ let mut metrics: Vec<_> = std::mem::take(self).into_iter().collect();+ metrics.push(metric);+ *self = Self::Standalone(metrics);+ }+ }+ }++ pub(super) fn iter(&self) -> std::slice::Iter<'_, Arc<Metric>> {+ match self {+ Self::Standalone(metrics) | Self::Partition { metrics, .. } => metrics.iter(),+ Self::All(snapshot) => {+ snapshot.flat.get_or_init(|| snapshot.materialize()).iter()+ }+ }+ }+}++impl IntoIterator for Snapshot {+ type Item = Arc<Metric>;+ type IntoIter = std::vec::IntoIter<Self::Item>;++ fn into_iter(self) -> Self::IntoIter {+ let metrics = match self {+ Self::Standalone(metrics) | Self::Partition { metrics, .. } => metrics,+ Self::All(snapshot) => match Arc::try_unwrap(snapshot) {+ Ok(mut snapshot) => snapshot+ .flat+ .take()+ .unwrap_or_else(|| snapshot.materialize()),+ Err(snapshot) => {+ snapshot.flat.get_or_init(|| snapshot.materialize()).clone()+ }+ },+ };+ metrics.into_iter()+ }+}++impl<'a> IntoIterator for &'a Snapshot {+ type Item = &'a Arc<Metric>;+ type IntoIter = std::slice::Iter<'a, Arc<Metric>>;++ fn into_iter(self) -> Self::IntoIter {+ self.iter()+ }+}++impl Extend<Arc<Metric>> for Snapshot {+ fn extend<I: IntoIterator<Item = Arc<Metric>>>(&mut self, iter: I) {+ for metric in iter {+ self.push(metric);+ }+ }+}++impl FromIterator<Arc<Metric>> for Snapshot {+ fn from_iter<I: IntoIterator<Item = Arc<Metric>>>(iter: I) -> Self {+ Self::Standalone(iter.into_iter().collect())+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::metrics::{Count, ExecutionPlanMetricsSet, MetricValue};++ fn metric(partition: Option<usize>) -> Arc<Metric> {+ Arc::new(Metric::new(+ MetricValue::OutputRows(Count::new()),+ partition,+ ))+ }++ #[test]+ fn out_of_order_snapshots_preserve_membership_and_registration_order() {+ let registry = ExecutionPlanMetricsSet::new();+ let mut expected = Vec::new();+ let mut snapshots = Vec::new();+ for i in 0..1024 {+ let partition = match i % 5 {+ 0 => None,+ 1 => Some(usize::MAX),+ 2 => Some(0),+ _ => Some(i * 17),+ };+ let metric = metric(partition);+ expected.push(Arc::clone(&metric));+ registry.register(metric);+ if i % 31 == 0 {+ snapshots.push((registry.clone_inner(), expected.len()));+ }+ }+ // Reading full snapshots must not build the partition index.+ assert_eq!(registry.clone_inner().iter().count(), 1024);+ assert_eq!(registry.inner.lock().index.indexed, 0);+ assert_eq!(registry.inner.lock().index.positions.capacity(), 0);+ // Advance the shared index before reading older, unmaterialized snapshots.+ registry.clone_inner().for_partition(0);+ for (snapshot, len) in snapshots.into_iter().rev() {+ for partition in [0, 17, 51, 999999, usize::MAX] {+ let selected = snapshot.for_partition(partition);+ let expected: Vec<_> = expected[..len]+ .iter()+ .filter(|m| m.partition() == Some(partition))+ .collect();+ assert_eq!(selected.iter().count(), expected.len());+ for (actual, expected) in selected.iter().zip(expected) {+ assert!(Arc::ptr_eq(actual, expected));+ }+ }+ assert_eq!(snapshot.iter().count(), len);+ for (actual, expected) in snapshot.iter().zip(&expected[..len]) {+ assert!(Arc::ptr_eq(actual, expected));+ }+ }+ }++ #[test]+ fn registration_leaves_index_and_retained_snapshots_unchanged() {+ let registry = ExecutionPlanMetricsSet::new();+ registry.register(metric(Some(0)));+ let empty_partition = registry.clone_inner();+ let old = registry.clone_inner();+ assert_eq!(old.for_partition(0).iter().count(), 1);+ registry.register(metric(Some(0)));+ registry.register(metric(Some(1)));+ {+ let registry = registry.inner.lock();+ let index = ®istry.index;+ assert_eq!(index.indexed, 1);+ assert_eq!(index.positions[&0], vec![0]);+ assert!(!index.positions.contains_key(&1));+ }+ assert_eq!(registry.clone_inner().for_partition(0).iter().count(), 2);+ assert_eq!(registry.clone_inner().for_partition(1).iter().count(), 1);+ assert_eq!(empty_partition.for_partition(1).iter().count(), 0);+ assert_eq!(old.for_partition(0).iter().count(), 1);+ assert_eq!(old.iter().count(), 1);+ let registry = registry.inner.lock();+ let index = ®istry.index;+ assert_eq!(index.indexed, 3);+ assert_eq!(index.positions[&0], vec![0, 1]);+ }++ #[test]+ fn selected_result_does_not_retain_registry() {+ let registry = ExecutionPlanMetricsSet::new();+ registry.register(metric(Some(0)));+ registry.register(metric(Some(1)));+ let weak = Arc::downgrade(®istry.inner);+ let snapshot = registry.clone_inner();+ let selected = snapshot.for_partition(0);+ drop(registry);+ assert!(weak.upgrade().is_some());+ drop(snapshot);+ assert!(weak.upgrade().is_none());+ assert_eq!(selected.iter().count(), 1);+ }++ #[test]+ fn selected_snapshot_handles_same_other_and_global_additions() {+ let registry = ExecutionPlanMetricsSet::new();+ registry.register(metric(Some(7)));+ let original = registry.clone_inner().for_partition(7);+ let mut same = original.clone();+ same.push(metric(Some(7)));+ assert_eq!(same.for_partition(7).iter().count(), 2);+ assert_eq!(same.for_partition(9).iter().count(), 0);+ assert_eq!(original.iter().count(), 1);+ same.push(metric(None));+ assert_eq!(same.iter().count(), 3);+ assert_eq!(same.for_partition(7).iter().count(), 2);+ same.push(metric(Some(9)));+ assert_eq!(same.for_partition(9).iter().count(), 1);+ assert_eq!(registry.clone_inner().iter().count(), 1);+ let empty = original.for_partition(9);+ assert_eq!(empty.for_partition(7).iter().count(), 0);+ }++ #[test]+ fn standalone_selection_preserves_duplicates_order_and_globals() {+ let first = metric(Some(usize::MAX));+ let global = metric(None);+ let other = metric(Some(3));+ let metrics: Snapshot = [Arc::clone(&first), global, other, Arc::clone(&first)]+ .into_iter()+ .collect();+ let selected = metrics.for_partition(usize::MAX);+ assert_eq!(selected.iter().count(), 2);+ assert!(selected.iter().all(|m| Arc::ptr_eq(m, &first)));+ assert_eq!(metrics.iter().count(), 4);+ assert_eq!(metrics.for_partition(3).iter().count(), 1);+ assert_eq!(metrics.for_partition(0).iter().count(), 0);+ }++ #[test]+ fn mutating_full_snapshot_detaches_from_registry() {+ let registry = ExecutionPlanMetricsSet::new();+ registry.register(metric(Some(0)));+ let original = registry.clone_inner();+ assert_eq!(original.for_partition(0).iter().count(), 1);+ let mut modified = original.clone();+ modified.push(metric(Some(1)));+ registry.register(metric(Some(2)));+ assert_eq!(original.iter().count(), 1);+ assert_eq!(modified.iter().count(), 2);+ assert_eq!(modified.for_partition(1).iter().count(), 1);+ assert_eq!(modified.for_partition(2).iter().count(), 0);+ assert_eq!(registry.clone_inner().for_partition(1).iter().count(), 0);+ assert_eq!(registry.clone_inner().for_partition(2).iter().count(), 1);+ }+}diff --git a/datafusion/physical-plan/src/execution_plan.rs b/datafusion/physical-plan/src/execution_plan.rsindex 978059b9fa..2f5680e659 100644--- a/datafusion/physical-plan/src/execution_plan.rs+++ b/datafusion/physical-plan/src/execution_plan.rs@@ -706,10 +706,9 @@ pub trait ExecutionPlan: Any + Debug + DisplayAs + Send + Sync {/// [`MetricsSet`]s may change as execution progresses, the/// specific metrics will not.///- /// Once `self.execute()` has returned (technically the future is- /// resolved) for all available partitions, the set of metrics- /// should be complete. If this function is called prior to- /// `execute()` new metrics may appear in subsequent calls.+ /// New metrics may be registered during execution, including while streams+ /// returned by [`Self::execute`] are being polled. Call again to obtain+ /// metrics registered since the previous snapshot.fn metrics(&self) -> Option<MetricsSet> {None}diff --git a/datafusion/physical-plan/tests/metrics/plan.rs b/datafusion/physical-plan/tests/metrics/plan.rsnew file mode 100644index 0000000000..d09c8d981f--- /dev/null+++ b/datafusion/physical-plan/tests/metrics/plan.rs@@ -0,0 +1,118 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Shared fixture: real streaming, filter and projection operators.++use std::sync::Arc;++use arrow::array::Int32Array;+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};+use arrow::record_batch::RecordBatch;+use datafusion_common::{Result, ScalarValue};+use datafusion_execution::{SendableRecordBatchStream, TaskContext};+use datafusion_expr::Operator;+use datafusion_physical_expr::PhysicalExpr;+use datafusion_physical_expr::expressions::{BinaryExpr, Column, Literal};+use datafusion_physical_plan::ExecutionPlan;+use datafusion_physical_plan::filter::FilterExec;+use datafusion_physical_plan::projection::ProjectionExec;+use datafusion_physical_plan::stream::RecordBatchStreamAdapter;+use datafusion_physical_plan::streaming::{PartitionStream, StreamingTableExec};+use futures::StreamExt;+use tokio::sync::Notify;++#[derive(Debug)]+struct InputPartition {+ schema: SchemaRef,+ batch: RecordBatch,+ gate: Option<Arc<Notify>>,+}++impl PartitionStream for InputPartition {+ fn schema(&self) -> &SchemaRef {+ &self.schema+ }++ fn execute(&self, _: Arc<TaskContext>) -> SendableRecordBatchStream {+ let first = self.batch.clone();+ let second = self.batch.clone();+ let gate = self.gate.clone();+ let stream = futures::stream::once(async move { Ok(first) }).chain(+ futures::stream::once(async move {+ if let Some(gate) = gate {+ gate.notified().await;+ }+ Ok(second)+ }),+ );+ Box::pin(RecordBatchStreamAdapter::new(+ Arc::clone(&self.schema),+ stream,+ ))+ }+}++pub type SharedPlan = (Vec<Arc<dyn ExecutionPlan>>, Arc<Notify>);++/// Partition 0 emits one batch, then waits while other partitions finish.+/// Returns nodes in root-to-leaf order for per-node reporting.+pub fn shared_plan(partitions: usize) -> Result<SharedPlan> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(Int32Array::from_iter_values(0..32))],+ )?;+ let gate = Arc::new(Notify::new());+ let inputs = (0..partitions)+ .map(|partition| {+ Arc::new(InputPartition {+ schema: Arc::clone(&schema),+ batch: batch.clone(),+ gate: (partition == 0).then(|| Arc::clone(&gate)),+ }) as Arc<dyn PartitionStream>+ })+ .collect();+ // The limit enables StreamingTableExec's own baseline metrics without+ // truncating this finite input.+ let source: Arc<dyn ExecutionPlan> = Arc::new(StreamingTableExec::try_new(+ schema,+ inputs,+ None,+ [],+ false,+ Some(usize::MAX),+ )?);+ let column: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));+ let predicate = Arc::new(BinaryExpr::new(+ Arc::clone(&column),+ Operator::Lt,+ Arc::new(Literal::new(ScalarValue::Int32(Some(16)))),+ ));+ let filter: Arc<dyn ExecutionPlan> = Arc::new(+ FilterExec::try_new(predicate, Arc::clone(&source))?.with_batch_size(16)?,+ );+ let expression: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(+ column,+ Operator::Plus,+ Arc::new(Literal::new(ScalarValue::Int32(Some(1)))),+ ));+ let projection: Arc<dyn ExecutionPlan> = Arc::new(ProjectionExec::try_new(+ vec![(expression, "b".to_string())],+ Arc::clone(&filter),+ )?);+ Ok((vec![projection, filter, source], gate))+}diff --git a/datafusion/physical-plan/tests/partition_metrics.rs b/datafusion/physical-plan/tests/partition_metrics.rsnew file mode 100644index 0000000000..3d4e97bce3--- /dev/null+++ b/datafusion/physical-plan/tests/partition_metrics.rs@@ -0,0 +1,101 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++#[path = "metrics/plan.rs"]+mod plan;++use std::sync::Arc;++use arrow::array::Int32Array;+use datafusion_common::Result;+use datafusion_execution::TaskContext;+use futures::{FutureExt, StreamExt, TryStreamExt};++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn shared_tree_partition_metrics() -> Result<()> {+ tokio::time::timeout(std::time::Duration::from_secs(30), run_shared_tree())+ .await+ .expect("shared tree did not complete")+}++async fn run_shared_tree() -> Result<()> {+ let (nodes, gate) = plan::shared_plan(16)?;+ let context = Arc::new(TaskContext::default());+ for node in &nodes {+ assert_eq!(node.metrics().unwrap().for_partition(0).iter().count(), 0);+ }+ let mut long_stream = nodes[0].execute(0, Arc::clone(&context))?;+ let first = long_stream.next().await.unwrap()?;+ assert_eq!(first.num_rows(), 16);+ assert!(long_stream.next().now_or_never().is_none());+ let early = nodes[0].metrics().unwrap().for_partition(0);+ assert_eq!(early.output_rows(), Some(16));++ // Concurrent execution and registration on the same Arc<dyn ExecutionPlan>.+ let mut tasks = tokio::task::JoinSet::new();+ for partition in 1..16 {+ let root = Arc::clone(&nodes[0]);+ let context = Arc::clone(&context);+ tasks.spawn(async move {+ let batches: Vec<_> = root.execute(partition, context)?.try_collect().await?;+ for batch in &batches {+ let values = batch+ .column(0)+ .as_any()+ .downcast_ref::<Int32Array>()+ .unwrap();+ assert_eq!(values.values().as_ref(), &(1..17).collect::<Vec<_>>());+ }+ assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 32);+ Ok::<_, datafusion_common::DataFusionError>(())+ });+ }+ while let Some(result) = tasks.join_next().await {+ result.unwrap()?;+ }+ // Other completed partitions do not affect the still-running partition.+ assert_eq!(early.output_rows(), Some(16));+ assert_eq!(+ nodes[0].metrics().unwrap().for_partition(0).output_rows(),+ Some(16)+ );+ gate.notify_one();+ let remaining: Vec<_> = long_stream.try_collect().await?;+ assert_eq!(remaining.iter().map(|b| b.num_rows()).sum::<usize>(), 16);+ assert_eq!(early.output_rows(), Some(32));++ for (node_index, node) in nodes.iter().enumerate() {+ let rows = if node_index == 2 { 64 } else { 32 };+ let full = node.metrics().unwrap();+ assert_eq!(full.output_rows(), Some(rows * 16));+ for partition in 0..16 {+ let selected = node.metrics().unwrap().for_partition(partition);+ assert_eq!(selected.output_rows(), Some(rows));+ assert_eq!(selected.aggregate_by_name().output_rows(), Some(rows));+ let expected: Vec<_> = full+ .iter()+ .filter(|m| m.partition() == Some(partition))+ .collect();+ assert_eq!(selected.iter().count(), expected.len());+ for (actual, expected) in selected.iter().zip(expected) {+ assert!(Arc::ptr_eq(actual, expected));+ }+ }+ assert_eq!(node.metrics().unwrap().for_partition(16).iter().count(), 0);+ }+ Ok(())+}