Skip to content

Apache DataFusion SQL Query Engine

test: consolidate partition metrics tests and clarify snapshot docs (#25848)

Liang-Chi Hsieh91086e0c3614
diff --git a/datafusion/core/tests/core_integration.rs b/datafusion/core/tests/core_integration.rs
index c1cd01eeb7..ddf14ff072 100644
--- a/datafusion/core/tests/core_integration.rs
+++ b/datafusion/core/tests/core_integration.rs
@@ -66,6 +66,9 @@ mod extension_types;
/// Run all tests that are found in the `function_docs` directory
mod function_docs;
+/// Run metrics integration tests in this shared test binary.
+mod metrics;
+
/// Helper functions for tests.
mod helper;
diff --git a/datafusion/physical-plan/tests/partition_metrics.rs b/datafusion/core/tests/metrics/mod.rs
similarity index 99%
rename from datafusion/physical-plan/tests/partition_metrics.rs
rename to datafusion/core/tests/metrics/mod.rs
index 3d4e97bce3..5d545d9dbf 100644
--- a/datafusion/physical-plan/tests/partition_metrics.rs
+++ b/datafusion/core/tests/metrics/mod.rs
@@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
-#[path = "metrics/plan.rs"]
mod plan;
use std::sync::Arc;
diff --git a/datafusion/physical-plan/tests/metrics/plan.rs b/datafusion/core/tests/metrics/plan.rs
similarity index 100%
rename from datafusion/physical-plan/tests/metrics/plan.rs
rename to datafusion/core/tests/metrics/plan.rs
diff --git a/datafusion/physical-expr-common/src/metrics/mod.rs b/datafusion/physical-expr-common/src/metrics/mod.rs
index 7b75dfdd88..9a1c17872e 100644
--- a/datafusion/physical-expr-common/src/metrics/mod.rs
+++ b/datafusion/physical-expr-common/src/metrics/mod.rs
@@ -234,10 +234,6 @@ impl Metric {
/// The set's members remain fixed as execution registers more metrics, but their
/// values continue to reflect execution progress. Use [`Self::for_partition`] to
/// select the metrics belonging to one partition.
-///
-/// Snapshots from [`ExecutionPlanMetricsSet::clone_inner`] retain the source
-/// registry, including later registrations, until dropped. A partition selection
-/// retains only the selected metrics.
#[derive(Default, Debug, Clone)]
pub struct MetricsSet {
metrics: Snapshot,
diff --git a/datafusion/physical-expr-common/src/metrics/snapshot.rs b/datafusion/physical-expr-common/src/metrics/snapshot.rs
index c29a924ce5..9d90ee427a 100644
--- a/datafusion/physical-expr-common/src/metrics/snapshot.rs
+++ b/datafusion/physical-expr-common/src/metrics/snapshot.rs
@@ -79,6 +79,7 @@ impl Registry {
#[derive(Clone)]
pub(super) enum Snapshot {
/// All metrics registered before a fixed boundary; partition reads use the index.
+ /// Retains the source registry, including later registrations, until dropped.
All(Arc<AllMetrics>),
/// A selected partition, independent of the source registry.
Partition {
@@ -94,6 +95,11 @@ pub(super) struct AllMetrics {
registry: Arc<Mutex<Registry>>,
// Exclusive position in Registry::metrics, not a per-partition count.
end: usize,
+ // Lazily copies registry.metrics[..end] for full iteration. The vector keeps
+ // the snapshot's original membership; only the metric values remain shared.
+ // Caching it gives iter(&self) stable references without holding the registry
+ // lock, and avoids copying the handles again on subsequent iterations. Keep
+ // the registry as well so partition selection can still use its shared index.
flat: OnceLock<Vec<Arc<Metric>>>,
}