Skip to content

Apache DataFusion SQL Query Engine

fix: avoid double-counting build_time in HashJoinExec elapsed_compute (#24142)

ulrichurriola-parada-ops739974b275d2
diff --git a/datafusion/physical-plan/src/joins/utils.rs b/datafusion/physical-plan/src/joins/utils.rs
index 4702493f4e..4f98b649e6 100644
--- a/datafusion/physical-plan/src/joins/utils.rs
+++ b/datafusion/physical-plan/src/joins/utils.rs
@@ -1777,10 +1777,28 @@ pub(crate) struct BuildProbeJoinMetrics {
/// Average number of build-side join-key matches per matched probe row before
/// applying any join filter
pub(crate) avg_fanout: metrics::RatioMetrics,
+ /// Ensures the `elapsed_compute` update below happens exactly once, no
+ /// matter how many clones of `BuildProbeJoinMetrics` exist (see
+ /// `ElapsedComputeFinalizer` for why this is needed).
+ _elapsed_compute_finalizer: Arc<ElapsedComputeFinalizer>,
+}
+
+// Upon being dropped, this will update the "elapsed compute" part of the metrics.
+// Wrapped in an `Arc` so that it is only dropped once (we previously had a bug
+// where it was cloned, causing over-estimated metrics).
+struct ElapsedComputeFinalizer {
+ baseline: BaselineMetrics,
+ build_time: metrics::Time,
+ join_time: metrics::Time,
+}
+
+// Define Debug impl for ElapsedComputeFinalizer directly to avoid duplicates fields
+impl Debug for ElapsedComputeFinalizer {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.debug_struct("ElapsedComputeFinalizer").finish()
+ }
}
-// This Drop implementation updates the elapsed compute part of the metrics.
-//
// Why is this in a Drop?
// - We keep track of build_time and join_time separately, but baseline metrics have
// a total elapsed_compute time. Instead of remembering to update both the metrics
@@ -1791,7 +1809,7 @@ pub(crate) struct BuildProbeJoinMetrics {
// - The elapsed_compute `Time` is represented by an `Arc<AtomicUsize>`. So even when
// this `BuildProbeJoinMetrics` is dropped, the elapsed_compute is usable through the
// Arc reference.
-impl Drop for BuildProbeJoinMetrics {
+impl Drop for ElapsedComputeFinalizer {
fn drop(&mut self) {
self.baseline.elapsed_compute().add(&self.build_time);
self.baseline.elapsed_compute().add(&self.join_time);
@@ -1833,6 +1851,12 @@ impl BuildProbeJoinMetrics {
.with_type(MetricType::Summary)
.ratio_metrics("avg_fanout", partition);
+ let elapsed_compute_finalizer = Arc::new(ElapsedComputeFinalizer {
+ baseline: baseline.clone(),
+ build_time: build_time.clone(),
+ join_time: join_time.clone(),
+ });
+
Self {
baseline,
build_time,
@@ -1844,6 +1868,7 @@ impl BuildProbeJoinMetrics {
input_rows,
probe_hit_rate,
avg_fanout,
+ _elapsed_compute_finalizer: elapsed_compute_finalizer,
}
}
}
@@ -2572,8 +2597,10 @@ pub fn compare_join_arrays(
mod tests {
use std::collections::HashMap;
use std::pin::Pin;
+ use std::time::Duration;
use super::*;
+ use crate::metrics::MetricValue;
use arrow::datatypes::{DataType, Fields};
use arrow::error::{ArrowError, Result as ArrowResult};
@@ -2752,6 +2779,48 @@ mod tests {
check_join_set_is_valid(&left, &right, on)
}
+ #[test]
+ fn build_probe_join_metrics_elapsed_compute_not_double_counted_on_clone() {
+ // `BuildProbeJoinMetrics` is cloned into the build-side future
+ // (`collect_left_input`) while the original stays with the
+ // `HashJoinStream` (see `HashJoinExec::execute`). Reproduce that
+ // shape here without spinning up a full hash join: accrue build_time
+ // before the clone is dropped (mirroring a build side that yields
+ // control at least once before completing), then accrue the rest of
+ // build_time plus join_time before the original is dropped.
+ let metrics_set = ExecutionPlanMetricsSet::new();
+ let join_metrics = BuildProbeJoinMetrics::new(0, &metrics_set);
+
+ let build_side_clone = join_metrics.clone();
+ join_metrics
+ .build_time
+ .add_duration(Duration::from_millis(100));
+ drop(build_side_clone);
+
+ join_metrics
+ .build_time
+ .add_duration(Duration::from_millis(50));
+ join_metrics
+ .join_time
+ .add_duration(Duration::from_millis(20));
+ drop(join_metrics);
+
+ let elapsed_compute = metrics_set
+ .clone_inner()
+ .iter()
+ .find_map(|m| match m.value() {
+ MetricValue::ElapsedCompute(time) => Some(time.value()),
+ _ => None,
+ })
+ .expect("elapsed_compute metric should be present");
+ let expected = Duration::from_millis(100 + 50 + 20).as_nanos() as usize;
+ assert_eq!(
+ elapsed_compute, expected,
+ "elapsed_compute should equal build_time + join_time exactly once, \
+ not once per BuildProbeJoinMetrics clone"
+ );
+ }
+
#[test]
fn check_valid() -> Result<()> {
let left = vec![Column::new("a", 0), Column::new("b1", 1)];