Apache DataFusion SQL Query Engine
feat(metric): Add output skewness metric to detect skewed plans easier (#21211)
Yongting Youe57198ab926a
diff --git a/datafusion/core/tests/sql/explain_analyze.rs b/datafusion/core/tests/sql/explain_analyze.rsindex 570a63c411..b8dc6ab59c 100644--- a/datafusion/core/tests/sql/explain_analyze.rs+++ b/datafusion/core/tests/sql/explain_analyze.rs@@ -69,7 +69,7 @@ async fn explain_analyze_baseline_metrics() {assert_metrics!(&formatted,"AggregateExec: mode=Partial, gby=[c1@0 as c1]",- "reduction_factor=5.1% (5/99)"+ "reduction_factor=5.05% (5/99)");{@@ -887,7 +887,8 @@ async fn parquet_explain_analyze() {&formatted,"row_groups_pruned_statistics=1 total \u{2192} 1 matched");- assert_contains!(&formatted, "scan_efficiency_ratio=14%");+ assert_contains!(&formatted, "output_rows_skew=0%");+ assert_contains!(&formatted, "scan_efficiency_ratio=13.99%");// The order of metrics is expected to be the same as the actual pruning order// (file-> row-group -> page)diff --git a/datafusion/datasource/src/source.rs b/datafusion/datasource/src/source.rsindex bffd94af21..73fc46f634 100644--- a/datafusion/datasource/src/source.rs+++ b/datafusion/datasource/src/source.rs@@ -27,7 +27,9 @@ use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType, SchedulingType,};use datafusion_physical_plan::metrics::SplitMetrics;-use datafusion_physical_plan::metrics::{ExecutionPlanMetricsSet, MetricsSet};+use datafusion_physical_plan::metrics::{+ BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet,+};use datafusion_physical_plan::projection::ProjectionExec;use datafusion_physical_plan::stream::BatchSplitStream;use datafusion_physical_plan::{@@ -356,7 +358,20 @@ impl ExecutionPlan for DataSourceExec {}fn metrics(&self) -> Option<MetricsSet> {- Some(self.data_source.metrics().clone_inner())+ let mut metrics = self.data_source.metrics().clone_inner();++ // Add `output_rows_skew` metric to the metrics set.+ // Done here because it's a derived metric from output_rows metric.+ if let Some(file_scan_config) =+ self.data_source.as_any().downcast_ref::<FileScanConfig>()+ && file_scan_config.file_source().file_type() == "parquet"+ && let Some(output_rows_skew) =+ BaselineMetrics::output_rows_skew_metric(&metrics)+ {+ metrics.push(output_rows_skew);+ }++ Some(metrics)}fn partition_statistics(&self, partition: Option<usize>) -> Result<Arc<Statistics>> {diff --git a/datafusion/physical-expr-common/src/metrics/baseline.rs b/datafusion/physical-expr-common/src/metrics/baseline.rsindex 74e30f0dc1..cba7b6b6d8 100644--- a/datafusion/physical-expr-common/src/metrics/baseline.rs+++ b/datafusion/physical-expr-common/src/metrics/baseline.rs@@ -17,12 +17,16 @@//! Metrics common for almost all operators-use std::task::Poll;+use std::{borrow::Cow, collections::BTreeMap, sync::Arc, task::Poll};use arrow::record_batch::RecordBatch;use datafusion_common::{Result, utils::memory::get_record_batch_memory_size};-use super::{Count, ExecutionPlanMetricsSet, MetricBuilder, Time, Timestamp};+use super::{+ Count, ExecutionPlanMetricsSet, Metric, MetricBuilder, MetricsSet, Time, Timestamp,+};++const OUTPUT_ROWS_SKEW_METRIC_NAME: &str = "output_rows_skew";/// Helper for creating and tracking common "baseline" metrics for/// each operator@@ -125,6 +129,61 @@ impl BaselineMetrics {&self.output_batches}+ /// Returns a derived metric that summarizes how unevenly `output_rows`+ /// are distributed across partitions.+ ///+ /// The score is normalized to the range `[0%, 100%]`, where `0%`+ /// indicates a perfectly balanced distribution and `100%` indicates the+ /// most skewed distribution.+ ///+ /// The calculation is:+ /// `effective_parallelism = square(sum(r_i)) / sum(square(r_i))`+ /// `output_rows_skew = (1 - ((effective_parallelism - 1) / (partition_count - 1))) * 100%`+ ///+ /// Example: for 4 partitions with output rows `[10, 10, 10, 10]`,+ /// `effective_parallelism = 40^2 / (10^2 + 10^2 + 10^2 + 10^2) = 4`,+ /// so `output_rows_skew = 0%`. For `[40, 0, 0, 0]`, the score is `100%`.+ pub fn output_rows_skew_metric(metrics: &MetricsSet) -> Option<Arc<Metric>> {+ let output_rows = metrics+ .iter()+ .filter_map(|metric| match (metric.partition(), metric.value()) {+ (Some(partition), super::MetricValue::OutputRows(count)) => {+ Some((partition, count.value() as u128))+ }+ _ => None,+ })+ .fold(+ BTreeMap::<usize, u128>::new(),+ |mut output_rows, (partition, rows)| {+ *output_rows.entry(partition).or_default() += rows;+ output_rows+ },+ )+ .into_values()+ .collect::<Vec<_>>();++ if output_rows.is_empty() {+ return None;+ }++ let ratio_metrics = super::RatioMetrics::new().with_display_raw_values(false);+ if let Some(score) = output_rows_skew_score(&output_rows) {+ ratio_metrics.set_part((score * 10_000.0).round() as usize);+ ratio_metrics.set_total(10_000);+ }++ Some(Arc::new(+ Metric::new(+ super::MetricValue::Ratio {+ name: Cow::Borrowed(OUTPUT_ROWS_SKEW_METRIC_NAME),+ ratio_metrics,+ },+ None,+ )+ .with_type(super::MetricType::DEV),+ ))+ }+/// Records the fact that this operator's execution is complete/// (recording the `end_time` metric).///@@ -178,6 +237,38 @@ impl Drop for BaselineMetrics {}}+/// See [`BaselineMetrics::output_rows_skew_metric`] for the algorithm.+fn output_rows_skew_score(output_rows: &[u128]) -> Option<f64> {+ if output_rows.is_empty() {+ return None;+ }++ let partition_count = output_rows.len();+ if partition_count == 1 {+ return Some(0.0);+ }++ let (total_rows, sum_of_squares) =+ output_rows+ .iter()+ .fold((0.0, 0.0), |(total_rows, sum_of_squares), rows| {+ let rows = *rows as f64;+ (total_rows + rows, sum_of_squares + rows.powi(2))+ });+ if total_rows == 0.0 {+ return None;+ }++ if sum_of_squares == 0.0 {+ return None;+ }++ let effective_parallelism = total_rows.powi(2) / sum_of_squares;+ let balanced_score = (effective_parallelism - 1.0) / (partition_count as f64 - 1.0);++ Some((1.0 - balanced_score).clamp(0.0, 1.0))+}+/// Helper for creating and tracking spill-related metrics for/// each operator#[derive(Debug, Clone)]diff --git a/datafusion/physical-expr-common/src/metrics/value.rs b/datafusion/physical-expr-common/src/metrics/value.rsindex d9e93aa361..d744dcde42 100644--- a/datafusion/physical-expr-common/src/metrics/value.rs+++ b/datafusion/physical-expr-common/src/metrics/value.rs@@ -468,6 +468,8 @@ pub struct RatioMetrics {part: Arc<AtomicUsize>,total: Arc<AtomicUsize>,merge_strategy: RatioMergeStrategy,+ /// Ratios are displayed as `1% (1/100)`; this controls the latter part.+ display_raw_values: bool,}#[derive(Debug, Clone, Default)]@@ -485,6 +487,7 @@ impl RatioMetrics {part: Arc::new(AtomicUsize::new(0)),total: Arc::new(AtomicUsize::new(0)),merge_strategy: RatioMergeStrategy::AddPartAddTotal,+ display_raw_values: true,}}@@ -493,6 +496,11 @@ impl RatioMetrics {self}+ pub fn with_display_raw_values(mut self, display_raw_values: bool) -> Self {+ self.display_raw_values = display_raw_values;+ self+ }+/// Add `n` to the numerator (`part`) valuepub fn add_part(&self, n: usize) {self.part.fetch_add(n, Ordering::Relaxed);@@ -544,44 +552,53 @@ impl RatioMetrics {impl PartialEq for RatioMetrics {fn eq(&self, other: &Self) -> bool {- self.part() == other.part() && self.total() == other.total()+ self.part() == other.part()+ && self.total() == other.total()+ && self.display_raw_values == other.display_raw_values}}-/// Format a float number with `digits` most significant numbers.-///-/// fmt_significant(12.5) -> "12"-/// fmt_significant(0.0543) -> "0.054"-/// fmt_significant(0.000123) -> "0.00012"-fn fmt_significant(mut x: f64, digits: usize) -> String {- if x == 0.0 {- return "0".to_string();- }-- let exp = x.abs().log10().floor(); // exponent of first significant digit- let scale = 10f64.powf(-(exp - (digits as f64 - 1.0)));- x = (x * scale).round() / scale; // round to N significant digits- format!("{x}")-}-impl Display for RatioMetrics {+ /// Format the ratio to a format like '18.26% (220/1150)'fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {let part = self.part();let total = self.total();+ // Format the ratio first (for example, `6667/10000` -> `66.67%`),+ // then optionally append the raw values as ` (6.67 K/10.00 K)`.+ if total == 0 {+ write!(f, "N/A")?;+ } else {+ // Use basis points so we can round with integer math:+ // e.g. 18.26% has basis point 1826+ let basis_points = (((part as u128 * 10_000) + (total as u128 / 2))+ / total as u128) as usize;+ let whole = basis_points / 100;+ let fractional = basis_points % 100;++ if fractional == 0 {+ write!(f, "{whole}%")?;+ } else if fractional.is_multiple_of(10) {+ write!(f, "{whole}.{}%", fractional / 10)?;+ } else {+ write!(f, "{whole}.{fractional:02}%")?;+ }+ }++ if !self.display_raw_values {+ return Ok(());+ }+if total == 0 {if part == 0 {- write!(f, "N/A (0/0)")+ write!(f, " (0/0)")} else {- write!(f, "N/A ({}/0)", human_readable_count(part))+ write!(f, " ({}/0)", human_readable_count(part))}} else {- let percentage = (part as f64 / total as f64) * 100.0;-write!(f,- "{}% ({}/{})",- fmt_significant(percentage, 2),+ " ({}/{})",human_readable_count(part),human_readable_count(total))@@ -865,7 +882,8 @@ impl MetricValue {Self::Ratio {name: name.clone(),ratio_metrics: RatioMetrics::new()- .with_merge_strategy(merge_strategy),+ .with_merge_strategy(merge_strategy)+ .with_display_raw_values(ratio_metrics.display_raw_values),}}Self::Custom { name, value } => Self::Custom {@@ -1232,7 +1250,21 @@ mod tests {};tiny_ratio_metrics.add_part(1);tiny_ratio_metrics.add_total(3000);- assert_eq!("0.033% (1/3.00 K)", tiny_ratio.to_string());+ assert_eq!("0.03% (1/3.00 K)", tiny_ratio.to_string());++ ratio_metrics.set_part(6667);+ ratio_metrics.set_total(10_000);+ assert_eq!("66.67% (6.67 K/10.00 K)", ratio.to_string());++ let percentage_only = RatioMetrics::new().with_display_raw_values(false);+ let ratio = MetricValue::Ratio {+ name: Cow::Borrowed("percentage_only"),+ ratio_metrics: percentage_only.clone(),+ };+ assert_eq!("N/A", ratio.to_string());+ percentage_only.set_part(6667);+ percentage_only.set_total(10_000);+ assert_eq!("66.67%", ratio.to_string());}#[test]diff --git a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.sltindex b0d9fceeff..3d0b50b6a6 100644--- a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt+++ b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt@@ -104,7 +104,7 @@ Plan with Metrics03)----ProjectionExec: expr=[id@0 as id, value@1 as v, value@1 + id@0 as name], metrics=[output_rows=10, <slt:ignore>]04)------FilterExec: value@1 > 3, metrics=[output_rows=10, <slt:ignore>, selectivity=100% (10/10)]05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, metrics=[output_rows=10, <slt:ignore>]-06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/test_data.parquet]]}, projection=[id, value], file_type=parquet, predicate=value@1 > 3 AND DynamicFilter [ value@1 IS NULL OR value@1 > 800 ], pruning_predicate=value_null_count@1 != row_count@2 AND value_max@0 > 3 AND (value_null_count@1 > 0 OR value_null_count@1 != row_count@2 AND value_max@0 > 800), required_guarantees=[], metrics=[output_rows=10, elapsed_compute=<slt:ignore>, output_bytes=80.0 B, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched -> 1 fully matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=1 total → 1 matched, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=210, metadata_load_time=<slt:ignore>, scan_efficiency_ratio=18% (210/1.15 K)]+06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/test_data.parquet]]}, projection=[id, value], file_type=parquet, predicate=value@1 > 3 AND DynamicFilter [ value@1 IS NULL OR value@1 > 800 ], pruning_predicate=value_null_count@1 != row_count@2 AND value_max@0 > 3 AND (value_null_count@1 > 0 OR value_null_count@1 != row_count@2 AND value_max@0 > 800), required_guarantees=[], metrics=[output_rows=10, elapsed_compute=<slt:ignore>, output_bytes=80.0 B, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched -> 1 fully matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=1 total → 1 matched, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=210, metadata_load_time=<slt:ignore>, scan_efficiency_ratio=18.31% (210/1.15 K)]statement okset datafusion.explain.analyze_level = dev;diff --git a/datafusion/sqllogictest/test_files/explain_analyze.slt b/datafusion/sqllogictest/test_files/explain_analyze.sltindex b35e9c8588..84216f4083 100644--- a/datafusion/sqllogictest/test_files/explain_analyze.slt+++ b/datafusion/sqllogictest/test_files/explain_analyze.slt@@ -23,6 +23,109 @@ EXPLAIN ANALYZE SELECT * FROM generate_series(100);----Plan with Metrics LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=0, end=100, batch_size=8192], metrics=[output_rows=101, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>]+# --------------------------------------------+# Test parquet-only output_rows_skew metric+# --------------------------------------------++statement ok+set datafusion.explain.analyze_level = dev;++statement ok+set datafusion.execution.target_partitions = 4;++query I+COPY (+ SELECT 1 AS x+ UNION ALL SELECT 2+ UNION ALL SELECT 3+ UNION ALL SELECT 4+)+TO 'test_files/scratch/explain_analyze/output_rows_skew/f1.parquet'+STORED AS PARQUET;+----+4++statement ok+CREATE EXTERNAL TABLE skew_parquet+STORED AS PARQUET+LOCATION 'test_files/scratch/explain_analyze/output_rows_skew';++# All partition's output_rows: [4]+query TT+EXPLAIN ANALYZE SELECT * FROM skew_parquet;+----+Plan with Metrics DataSourceExec: <slt:ignore>output_rows_skew=0%<slt:ignore>++statement ok+DROP TABLE skew_parquet;++query I+COPY (SELECT * FROM (SELECT 1 AS x) WHERE false)+TO 'test_files/scratch/explain_analyze/output_rows_skew/f2.parquet'+STORED AS PARQUET;+----+0++# All partition's output_rows: [4, 0]+statement ok+CREATE EXTERNAL TABLE skew_parquet+STORED AS PARQUET+LOCATION 'test_files/scratch/explain_analyze/output_rows_skew';++query TT+EXPLAIN ANALYZE SELECT * FROM skew_parquet;+----+Plan with Metrics DataSourceExec: <slt:ignore>output_rows_skew=100%<slt:ignore>++statement ok+DROP TABLE skew_parquet;++query I+COPY (SELECT * FROM (SELECT 1 AS x))+TO 'test_files/scratch/explain_analyze/output_rows_skew/f3.parquet'+STORED AS PARQUET;+----+1++query I+COPY (SELECT * FROM (SELECT 1 AS x) WHERE false)+TO 'test_files/scratch/explain_analyze/output_rows_skew/f4.parquet'+STORED AS PARQUET;+----+0++# All partition's output_rows: [4, 0, 1, 0]+statement ok+CREATE EXTERNAL TABLE skew_parquet+STORED AS PARQUET+LOCATION 'test_files/scratch/explain_analyze/output_rows_skew';++query TT+EXPLAIN ANALYZE SELECT * FROM skew_parquet;+----+Plan with Metrics DataSourceExec: <slt:ignore>output_rows_skew=84.31%<slt:ignore>++# All partition's output_rows: [0]+statement ok+CREATE EXTERNAL TABLE skew_parquet_single+STORED AS PARQUET+LOCATION 'test_files/scratch/explain_analyze/output_rows_skew/f4.parquet';++query TT+EXPLAIN ANALYZE SELECT * FROM skew_parquet_single;+----+Plan with Metrics DataSourceExec: <slt:ignore>output_rows_skew=0%<slt:ignore>++statement ok+DROP TABLE skew_parquet_single;++statement ok+DROP TABLE skew_parquet;++# The SLT runner sets `target_partitions` to 4 for deterministic plans.+statement ok+set datafusion.execution.target_partitions = 4;+# --------------------------------------------# Test ProjectionExec's per-expression metrics# --------------------------------------------diff --git a/docs/source/user-guide/explain-usage.md b/docs/source/user-guide/explain-usage.mdindex c047659e99..5ea7a250b4 100644--- a/docs/source/user-guide/explain-usage.md+++ b/docs/source/user-guide/explain-usage.md@@ -225,6 +225,7 @@ Again, reading from bottom up:When predicate pushdown is enabled, `DataSourceExec` with `ParquetSource` gains the following metrics:+- `output_rows_skew`: output skew score derived from per-partition `output_rows`. `0%` is perfectly balanced, `100%` is maximally skewed, and `N/A` means no output rows were produced.- `page_index_rows_pruned`: number of rows evaluated by page index filters. The metric reports both how many rows were considered in total and how many matched (were not pruned).- `page_index_pages_pruned`: number of pages evaluated by page index filters. The metric reports both how many pages were considered in total and how many matched (were not pruned).- `row_groups_pruned_bloom_filter`: number of row groups evaluated by Bloom Filters, reporting both total checked groups and groups that matched.