Apache DataFusion SQL Query Engine
fix: Correct join cardinality estimation for semi and anti joins with disjoint column ranges (#22674)
Neil Conwayd1674505624a
diff --git a/datafusion/physical-plan/src/joins/utils.rs b/datafusion/physical-plan/src/joins/utils.rsindex 9a6d1e5545..8108b7f2db 100644--- a/datafusion/physical-plan/src/joins/utils.rs+++ b/datafusion/physical-plan/src/joins/utils.rs@@ -475,7 +475,7 @@ fn estimate_join_cardinality(right_stats: Statistics,on: &JoinOn,) -> Option<PartialJoinStatistics> {- let (left_col_stats, right_col_stats) = on+ let (left_key_stats, right_key_stats) = on.iter().map(|(left, right)| {match (@@ -500,12 +500,12 @@ fn estimate_join_cardinality(Statistics {num_rows: left_stats.num_rows,total_byte_size: Precision::Absent,- column_statistics: left_col_stats,+ column_statistics: left_key_stats,},Statistics {num_rows: right_stats.num_rows,total_byte_size: Precision::Absent,- column_statistics: right_col_stats,+ column_statistics: right_key_stats,},)?;@@ -545,38 +545,49 @@ fn estimate_join_cardinality(let is_left = matches!(join_type, JoinType::LeftSemi | JoinType::LeftAnti);let is_anti = matches!(join_type, JoinType::LeftAnti | JoinType::RightAnti);- let ((outer_stats, inner_stats), (outer_col_stats, inner_col_stats)) =- if is_left {- (- (&left_stats, &right_stats),- (&left_col_stats, &right_col_stats),- )- } else {- (- (&right_stats, &left_stats),- (&right_col_stats, &left_col_stats),- )- };+ let (outer_stats, inner_stats, outer_key_stats, inner_key_stats) = if is_left+ {+ (&left_stats, &right_stats, &left_key_stats, &right_key_stats)+ } else {+ (&right_stats, &left_stats, &right_key_stats, &left_key_stats)+ };let outer_rows = *outer_stats.num_rows.get_value()?;- let cardinality =- if estimate_disjoint_inputs(outer_stats, inner_stats).is_some() {- // Disjoint inputs: semi produces 0, anti keeps all rows.- if is_anti { outer_rows } else { 0 }+ let outer_join_key_stats = Statistics {+ num_rows: outer_stats.num_rows,+ total_byte_size: Precision::Absent,+ column_statistics: outer_key_stats.clone(),+ };+ let inner_join_key_stats = Statistics {+ num_rows: inner_stats.num_rows,+ total_byte_size: Precision::Absent,+ column_statistics: inner_key_stats.clone(),+ };++ let semi_cardinality =+ if estimate_disjoint_inputs(&outer_join_key_stats, &inner_join_key_stats)+ .is_some()+ {+ // If join keys are disjoint, no rows will match+ Some(0)} else {- match estimate_semi_join_cardinality(+ estimate_semi_join_cardinality(&outer_stats.num_rows,&inner_stats.num_rows,- outer_col_stats,- inner_col_stats,- ) {- Some(semi) if is_anti => outer_rows.saturating_sub(semi),- Some(semi) => semi,- None => outer_rows,- }+ outer_key_stats,+ inner_key_stats,+ )};+ // Semi joins keep the matching rows; anti joins keep the rest. When no+ // estimate is available, conservatively assume all outer rows pass.+ let cardinality = match (semi_cardinality, is_anti) {+ (Some(semi), true) => outer_rows.saturating_sub(semi),+ (Some(semi), false) => semi,+ (None, _) => outer_rows,+ };+let outer_stats = if is_left { left_stats } else { right_stats };Some(PartialJoinStatistics {num_rows: cardinality,@@ -759,8 +770,8 @@ fn estimate_disjoint_inputs(fn estimate_semi_join_cardinality(outer_num_rows: &Precision<usize>,inner_num_rows: &Precision<usize>,- outer_col_stats: &[ColumnStatistics],- inner_col_stats: &[ColumnStatistics],+ outer_key_stats: &[ColumnStatistics],+ inner_key_stats: &[ColumnStatistics],) -> Option<usize> {let outer_rows = *outer_num_rows.get_value()?;if outer_rows == 0 {@@ -774,7 +785,7 @@ fn estimate_semi_join_cardinality(let mut selectivity = 1.0_f64;let mut has_selectivity_estimate = false;- for (outer_stat, inner_stat) in outer_col_stats.iter().zip(inner_col_stats.iter()) {+ for (outer_stat, inner_stat) in outer_key_stats.iter().zip(inner_key_stats.iter()) {let outer_has_stats = outer_stat.distinct_count.get_value().is_some()|| (outer_stat.min_value.get_value().is_some()&& outer_stat.max_value.get_value().is_some());@@ -3246,6 +3257,50 @@ mod tests {Ok(())}+ #[test]+ fn test_semi_anti_join_disjoint_check_uses_only_join_keys() {+ let join_on = vec![(+ Arc::new(Column::new("l_key", 0)) as _,+ Arc::new(Column::new("r_key", 0)) as _,+ )];++ // Ranges for the join key overlap; ranges for the other column are disjoint+ let left_stats = Statistics {+ num_rows: Inexact(50),+ total_byte_size: Absent,+ column_statistics: vec![+ create_column_stats(Inexact(1), Inexact(10), Absent, Absent),+ create_column_stats(Inexact(100), Inexact(200), Absent, Absent),+ ],+ };+ let right_stats = Statistics {+ num_rows: Inexact(10),+ total_byte_size: Absent,+ column_statistics: vec![+ create_column_stats(Inexact(1), Inexact(10), Absent, Absent),+ create_column_stats(Inexact(1000), Inexact(2000), Absent, Absent),+ ],+ };++ let left_semi = estimate_join_cardinality(+ &JoinType::LeftSemi,+ left_stats.clone(),+ right_stats.clone(),+ &join_on,+ )+ .map(|c| c.num_rows);+ assert_eq!(left_semi, Some(50));++ let left_anti = estimate_join_cardinality(+ &JoinType::LeftAnti,+ left_stats,+ right_stats,+ &join_on,+ )+ .map(|c| c.num_rows);+ assert_eq!(left_anti, Some(0));+ }+#[test]fn test_calculate_join_output_ordering() -> Result<()> {let left_ordering = LexOrdering::new(vec![