Skip to content

Apache DataFusion SQL Query Engine

fix: Correct computation of selectivity for multi-key joins (#22725)

Neil Conway533ef35a65a8
diff --git a/datafusion/physical-plan/src/joins/utils.rs b/datafusion/physical-plan/src/joins/utils.rs
index 8108b7f2db..5918097194 100644
--- a/datafusion/physical-plan/src/joins/utils.rs
+++ b/datafusion/physical-plan/src/joins/utils.rs
@@ -640,8 +640,8 @@ fn estimate_inner_join_cardinality(
..
} = right_stats;
- // The algorithm here is partly based on the non-histogram selectivity estimation
- // from Spark's Catalyst optimizer.
+ // Follow Spark Catalyst's conservative NDV join estimate: for multi-key
+ // joins, use the most selective key instead of multiplying all key denominators.
let mut join_selectivity = Precision::Absent;
for (left_stat, right_stat) in left_column_statistics
.iter()
@@ -654,7 +654,11 @@ fn estimate_inner_join_cardinality(
// Seems like there are a few implementations of this algorithm that implement
// exponential decay for the selectivity (like Hive's Optiq Optimizer). Needs
// further exploration.
- join_selectivity = max_distinct;
+ join_selectivity = if join_selectivity.get_value().is_some() {
+ join_selectivity.max(&max_distinct)
+ } else {
+ max_distinct
+ };
}
}
@@ -2730,6 +2734,68 @@ mod tests {
Ok(())
}
+ #[test]
+ fn test_join_cardinality_key_order() -> Result<()> {
+ // Reversing join key order should not change estimated cardinality
+ let left_col_stats = vec![
+ create_column_stats(Inexact(0), Inexact(100), Inexact(100), Absent),
+ create_column_stats(Inexact(0), Inexact(500), Inexact(500), Absent),
+ create_column_stats(Inexact(1000), Inexact(10000), Absent, Absent),
+ ];
+
+ let right_col_stats = vec![
+ create_column_stats(Inexact(0), Inexact(100), Inexact(50), Absent),
+ create_column_stats(Inexact(0), Inexact(2000), Inexact(2500), Absent),
+ create_column_stats(Inexact(0), Inexact(100), Absent, Absent),
+ ];
+
+ let join_on_ab = vec![
+ (
+ Arc::new(Column::new("a", 0)) as _,
+ Arc::new(Column::new("c", 0)) as _,
+ ),
+ (
+ Arc::new(Column::new("b", 1)) as _,
+ Arc::new(Column::new("d", 1)) as _,
+ ),
+ ];
+ let join_on_ba = vec![
+ (
+ Arc::new(Column::new("b", 1)) as _,
+ Arc::new(Column::new("d", 1)) as _,
+ ),
+ (
+ Arc::new(Column::new("a", 0)) as _,
+ Arc::new(Column::new("c", 0)) as _,
+ ),
+ ];
+
+ let stats_ab = estimate_join_cardinality(
+ &JoinType::Inner,
+ create_stats(Some(1000), left_col_stats.clone(), false),
+ create_stats(Some(2000), right_col_stats.clone(), false),
+ &join_on_ab,
+ )
+ .unwrap();
+ let stats_ba = estimate_join_cardinality(
+ &JoinType::Inner,
+ create_stats(Some(1000), left_col_stats.clone(), false),
+ create_stats(Some(2000), right_col_stats.clone(), false),
+ &join_on_ba,
+ )
+ .unwrap();
+
+ assert_eq!(stats_ab.num_rows, 1000);
+ assert_eq!(stats_ba.num_rows, stats_ab.num_rows);
+ assert_eq!(stats_ba.column_statistics, stats_ab.column_statistics);
+ assert_eq!(
+ stats_ab.column_statistics,
+ [left_col_stats, right_col_stats].concat()
+ );
+
+ Ok(())
+ }
+
#[test]
fn test_join_cardinality_when_one_column_is_disjoint() -> Result<()> {
// Left table (rows=1000)