Apache DataFusion SQL Query Engine
Add skewness metric to nodes that execute in partitioned mode (#25977)
Lía Adrianac2baac54aff3
diff --git a/datafusion/core/tests/sql/explain_analyze.rs b/datafusion/core/tests/sql/explain_analyze.rsindex 67614cfbde..3efe084bae 100644--- a/datafusion/core/tests/sql/explain_analyze.rs+++ b/datafusion/core/tests/sql/explain_analyze.rs@@ -19,6 +19,7 @@ use super::*;use insta::assert_snapshot;use rstest::rstest;+use datafusion::catalog::MemTable;use datafusion::config::ConfigOptions;use datafusion::physical_plan::display::DisplayableExecutionPlan;use datafusion::physical_plan::metrics::Timestamp;@@ -908,6 +909,83 @@ async fn parquet_explain_analyze() {);}+/// Operators that run once per partition report `output_rows_skew`. A single+/// join/group/window key hashes every row to one partition, so each+/// hash-partitioned operator must report the maximum skew.+#[tokio::test]+#[cfg_attr(coverage, ignore)]+async fn explain_analyze_output_rows_skew_on_hash_partitioned_operators() {+ let mut config = SessionConfig::new().with_target_partitions(4);+ // Force a partitioned (hash repartitioned) join instead of `CollectLeft`+ let options = config.options_mut();+ options.optimizer.hash_join_single_partition_threshold = 0;+ options.optimizer.hash_join_single_partition_threshold_rows = 0;+ let ctx = SessionContext::new_with_config(config);+ // Spread the rows over 4 input partitions so they get hash repartitioned+ // (a single input partition already satisfies the hash distribution)+ let schema = Arc::new(Schema::new(vec![+ Field::new("k", DataType::Int64, false),+ Field::new("v", DataType::Int64, false),+ ]));+ let partitions = (0..4)+ .map(|p| {+ let v = (p * 25..(p + 1) * 25).collect::<Vec<i64>>();+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(Int64Array::from(vec![1; v.len()])),+ Arc::new(Int64Array::from(v)),+ ],+ )+ .unwrap();+ vec![batch]+ })+ .collect();+ let table = MemTable::try_new(schema, partitions).unwrap();+ ctx.register_table("t", Arc::new(table)).unwrap();++ let sql = "EXPLAIN ANALYZE \+ SELECT a.k, count(*) FROM t a JOIN t b ON a.k = b.k GROUP BY a.k";+ let formatted = explain_analyze_output(&ctx, sql).await;+ for operator in [+ "RepartitionExec: partitioning=Hash",+ "HashJoinExec: mode=Partitioned",+ "AggregateExec: mode=SinglePartitioned",+ ] {+ assert_operator_metric(&formatted, operator, "output_rows_skew=100%");+ }++ let sql = "EXPLAIN ANALYZE \+ SELECT k, \+ row_number() OVER (PARTITION BY k ORDER BY v), \+ sum(v) OVER (PARTITION BY k) \+ FROM t";+ let formatted = explain_analyze_output(&ctx, sql).await;+ for operator in ["BoundedWindowAggExec", "WindowAggExec"] {+ assert_operator_metric(&formatted, operator, "output_rows_skew=100%");+ }+}++async fn explain_analyze_output(ctx: &SessionContext, sql: &str) -> String {+ let actual = execute_to_batches(ctx, sql).await;+ arrow::util::pretty::pretty_format_batches(&actual)+ .unwrap()+ .to_string()+}++fn assert_operator_metric(formatted: &str, operator: &str, metric: &str) {+ // Leading space so `WindowAggExec` does not match `BoundedWindowAggExec`+ let needle = format!(" {operator}");+ let line = formatted+ .lines()+ .find(|line| line.contains(&needle))+ .unwrap_or_else(|| panic!("{operator} not found in plan:\n{formatted}"));+ assert!(+ line.contains(metric),+ "expected {metric} on {operator}, got:\n{line}"+ );+}+/// The Parquet scan's `elapsed_compute` must cover decoding, which dominates a/// full scan, and not only the projection of batches that are already decoded./// See <https://github.com/apache/datafusion/issues/18195>.diff --git a/datafusion/physical-expr-common/src/metrics/mod.rs b/datafusion/physical-expr-common/src/metrics/mod.rsindex 9a1c17872e..262550f594 100644--- a/datafusion/physical-expr-common/src/metrics/mod.rs+++ b/datafusion/physical-expr-common/src/metrics/mod.rs@@ -250,6 +250,22 @@ impl MetricsSet {self.metrics.push(metric)}+ /// Returns this set with the derived `output_rows_skew` metric appended,+ /// or unchanged if no partition reported `output_rows` yet.+ ///+ /// This is typically used in `ExecutionPlan::metrics` of operators that+ /// execute in multiple partitions (e.g. repartitions, partitioned joins and+ /// aggregations), where it shows how evenly their output rows are spread+ /// across partitions. With a single partition the skew is always `0%`.+ ///+ /// See [`BaselineMetrics::output_rows_skew_metric`] for how skew is computed.+ pub fn with_output_rows_skew(mut self) -> Self {+ if let Some(output_rows_skew) = BaselineMetrics::output_rows_skew_metric(&self) {+ self.push(output_rows_skew);+ }+ self+ }+/// Return the metrics whose partition ID equals `partition`.////// Unpartitioned metrics are excluded; an unknown partition yields an emptydiff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rsindex 6552bd95e0..d098272688 100644--- a/datafusion/physical-plan/src/aggregates/mod.rs+++ b/datafusion/physical-plan/src/aggregates/mod.rs@@ -2600,7 +2600,7 @@ impl ExecutionPlan for AggregateExec {}fn metrics(&self) -> Option<MetricsSet> {- Some(self.metrics.clone_inner())+ Some(self.metrics.clone_inner().with_output_rows_skew())}fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex ce8ee72493..10a9fa70a8 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -1863,7 +1863,7 @@ impl ExecutionPlan for HashJoinExec {}fn metrics(&self) -> Option<MetricsSet> {- Some(self.metrics.clone_inner())+ Some(self.metrics.clone_inner().with_output_rows_skew())}fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rsindex 3df29b577c..51ce1345fc 100644--- a/datafusion/physical-plan/src/repartition/mod.rs+++ b/datafusion/physical-plan/src/repartition/mod.rs@@ -1884,7 +1884,7 @@ impl ExecutionPlan for RepartitionExec {}fn metrics(&self) -> Option<MetricsSet> {- Some(self.metrics.clone_inner())+ Some(self.metrics.clone_inner().with_output_rows_skew())}fn child_stats_requests(&self, _partition: Option<usize>) -> Vec<ChildStats> {diff --git a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rsindex 0c6e77a1d6..724a722898 100644--- a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs+++ b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs@@ -528,7 +528,7 @@ impl ExecutionPlan for BoundedWindowAggExec {}fn metrics(&self) -> Option<MetricsSet> {- Some(self.metrics.clone_inner())+ Some(self.metrics.clone_inner().with_output_rows_skew())}fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {diff --git a/datafusion/physical-plan/src/windows/window_agg_exec.rs b/datafusion/physical-plan/src/windows/window_agg_exec.rsindex dba4c063d1..e00aaa21bb 100644--- a/datafusion/physical-plan/src/windows/window_agg_exec.rs+++ b/datafusion/physical-plan/src/windows/window_agg_exec.rs@@ -320,7 +320,7 @@ impl ExecutionPlan for WindowAggExec {}fn metrics(&self) -> Option<MetricsSet> {- Some(self.metrics.clone_inner())+ Some(self.metrics.clone_inner().with_output_rows_skew())}fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {diff --git a/datafusion/sqllogictest/test_files/dynamic_row_group_pruning.slt b/datafusion/sqllogictest/test_files/dynamic_row_group_pruning.sltindex 753bf6fdfd..10d25d7235 100644--- a/datafusion/sqllogictest/test_files/dynamic_row_group_pruning.slt+++ b/datafusion/sqllogictest/test_files/dynamic_row_group_pruning.slt@@ -338,7 +338,7 @@ query TTexplain analyze select rgsel.b from dim join rgsel on dim.column1 = rgsel.b;----Plan with Metrics-01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(column1@0, b@0)], projection=[b@1], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, output_batches=1, build_mem_used=<slt:ignore>, array_map_created_count=0, build_input_batches=1, build_input_rows=3, input_batches=2, input_rows=3, build_time=<slt:ignore>, join_time=<slt:ignore>, avg_fanout=100% (3/3), probe_hit_rate=100% (3/3)]+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(column1@0, b@0)], projection=[b@1], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, output_batches=1, build_mem_used=<slt:ignore>, array_map_created_count=0, build_input_batches=1, build_input_rows=3, input_batches=2, input_rows=3, build_time=<slt:ignore>, join_time=<slt:ignore>, avg_fanout=100% (3/3), output_rows_skew=0%, probe_hit_rate=100% (3/3)]02)--DataSourceExec: partitions=1, partition_sizes=[1], metrics=[]03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_row_group_pruning/rgsel.parquet]]}, projection=[b], file_type=parquet, predicate=DynamicFilter [ b@1 >= 1000 AND b@1 <= 3050 AND hash_lookup ], dynamic_rg_pruning=eligible, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 >= 1000 AND b_null_count@1 != row_count@2 AND b_min@3 <= 3050 AND b_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(b_min@3, b_max@0, 3/4096 buckets), required_guarantees=[], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, output_batches=2, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 2 matched, row_groups_pruned_bloom_filter=2 total → 2 matched, page_index_pages_pruned=20 total → 3 matched, page_index_rows_pruned=200 total → 30 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, bytes_processed=<slt:ignore>, bytes_scanned=<slt:ignore>, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=27, row_groups_pruned_dynamic_filter=0, predicate_cache_inner_records=200, predicate_cache_records=13, bloom_filter_eval_time=<slt:ignore>, metadata_load_time=<slt:ignore>, page_index_eval_time=<slt:ignore>, row_pushdown_eval_time=<slt:ignore>, statistics_eval_time=<slt:ignore>, time_elapsed_opening=<slt:ignore>, time_elapsed_processing=<slt:ignore>, time_elapsed_scanning_total=<slt:ignore>, time_elapsed_scanning_until_data=<slt:ignore>, output_rows_skew=<slt:ignore>, scan_efficiency_ratio=<slt:ignore>]diff --git a/datafusion/sqllogictest/test_files/joins.slt b/datafusion/sqllogictest/test_files/joins.sltindex f4dbd212eb..e922233fd8 100644--- a/datafusion/sqllogictest/test_files/joins.slt+++ b/datafusion/sqllogictest/test_files/joins.slt@@ -5263,7 +5263,7 @@ LEFT ANTI JOIN () t2 ON t1.k = t2.k;----Plan with Metrics-01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(k@0, k@0)], metrics=[output_rows=2, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, output_batches=1, build_mem_used=<slt:ignore>, array_map_created_count=0, build_input_batches=0, build_input_rows=0, input_batches=1, input_rows=2, build_time=<slt:ignore>, join_time=<slt:ignore>, avg_fanout=N/A (0/0), probe_hit_rate=0% (0/2)]+01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(k@0, k@0)], metrics=[output_rows=2, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, output_batches=1, build_mem_used=<slt:ignore>, array_map_created_count=0, build_input_batches=0, build_input_rows=0, input_batches=1, input_rows=2, build_time=<slt:ignore>, join_time=<slt:ignore>, avg_fanout=N/A (0/0), output_rows_skew=0%, probe_hit_rate=0% (0/2)]02)--ProjectionExec: expr=[column1@0 as k], metrics=[output_rows=0, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, output_batches=0, expr_0_eval_time=<slt:ignore>]03)----FilterExec: column1@0 != 1, metrics=[output_rows=0, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, output_batches=0, selectivity=0% (0/1)]04)------DataSourceExec: partitions=1, partition_sizes=[1], metrics=[]