Skip to content

Apache DataFusion SQL Query Engine

feat: estimate cardinality for semi and anti-joins using distinct counts (#20904)

Burak Şena737c27b775f
diff --git a/datafusion/physical-plan/src/joins/utils.rs b/datafusion/physical-plan/src/joins/utils.rs
index a5ac890163..90cab7246d 100644
--- a/datafusion/physical-plan/src/joins/utils.rs
+++ b/datafusion/physical-plan/src/joins/utils.rs
@@ -531,35 +531,48 @@ fn estimate_join_cardinality(
})
}
- // For SemiJoins estimation result is either zero, in cases when inputs
- // are non-overlapping according to statistics, or equal to number of rows
- // for outer input
- JoinType::LeftSemi | JoinType::RightSemi => {
- let (outer_stats, inner_stats) = match join_type {
- JoinType::LeftSemi => (left_stats, right_stats),
- _ => (right_stats, left_stats),
- };
- let cardinality = match estimate_disjoint_inputs(&outer_stats, &inner_stats) {
- Some(estimation) => *estimation.get_value()?,
- None => *outer_stats.num_rows.get_value()?,
- };
+ JoinType::LeftSemi
+ | JoinType::RightSemi
+ | JoinType::LeftAnti
+ | JoinType::RightAnti => {
+ let is_left = matches!(join_type, JoinType::LeftSemi | JoinType::LeftAnti);
+ let is_anti = matches!(join_type, JoinType::LeftAnti | JoinType::RightAnti);
+
+ let ((outer_stats, inner_stats), (outer_col_stats, inner_col_stats)) =
+ if is_left {
+ (
+ (&left_stats, &right_stats),
+ (&left_col_stats, &right_col_stats),
+ )
+ } else {
+ (
+ (&right_stats, &left_stats),
+ (&right_col_stats, &left_col_stats),
+ )
+ };
- Some(PartialJoinStatistics {
- num_rows: cardinality,
- column_statistics: outer_stats.column_statistics,
- })
- }
+ let outer_rows = *outer_stats.num_rows.get_value()?;
- // For AntiJoins estimation always equals to outer statistics, as
- // non-overlapping inputs won't affect estimation
- JoinType::LeftAnti | JoinType::RightAnti => {
- let outer_stats = match join_type {
- JoinType::LeftAnti => left_stats,
- _ => right_stats,
- };
+ let cardinality =
+ if estimate_disjoint_inputs(outer_stats, inner_stats).is_some() {
+ // Disjoint inputs: semi produces 0, anti keeps all rows.
+ if is_anti { outer_rows } else { 0 }
+ } else {
+ match estimate_semi_join_cardinality(
+ &outer_stats.num_rows,
+ &inner_stats.num_rows,
+ outer_col_stats,
+ inner_col_stats,
+ ) {
+ Some(semi) if is_anti => outer_rows.saturating_sub(semi),
+ Some(semi) => semi,
+ None => outer_rows,
+ }
+ };
+ let outer_stats = if is_left { left_stats } else { right_stats };
Some(PartialJoinStatistics {
- num_rows: *outer_stats.num_rows.get_value()?,
+ num_rows: cardinality,
column_statistics: outer_stats.column_statistics,
})
}
@@ -699,6 +712,95 @@ fn estimate_disjoint_inputs(
None
}
+/// Estimates the number of outer rows that have at least one matching
+/// key on the inner side (i.e. semi join cardinality) using NDV
+/// (Number of Distinct Values) statistics.
+///
+/// Assuming the smaller domain is contained in the larger, the number
+/// of overlapping distinct values is `min(outer_ndv, inner_ndv)`.
+/// Under the uniformity assumption (each distinct value contributes
+/// equally to row counts), the surviving fraction of outer rows is:
+///
+/// Null rows cannot match, so each column's selectivity is further
+/// reduced by the outer null fraction:
+///
+/// ```text
+/// null_frac_i = outer_null_count_i / outer_rows
+/// selectivity_i = min(outer_ndv_i, inner_ndv_i) / outer_ndv_i * (1 - null_frac_i)
+/// ```
+///
+/// For multi-column join keys the overall selectivity is the product
+/// of per-column factors:
+///
+/// ```text
+/// semi_cardinality = outer_rows * product_i(selectivity_i)
+/// ```
+///
+/// Anti join cardinality is derived as the complement:
+/// `outer_rows - semi_cardinality`.
+///
+/// Boundary cases:
+/// * `inner_ndv >= outer_ndv` → selectivity = `1.0 - null_frac`
+/// * `null_frac = 1.0` → selectivity = 0.0 (no non-null rows can match)
+/// * Missing NDV statistics → returns `None` (fallback to `outer_rows`)
+///
+/// PostgreSQL uses a similar approach in `eqjoinsel_semi`
+/// (`src/backend/utils/adt/selfuncs.c`). When NDV statistics are
+/// available on both sides it computes selectivity as `nd2 / nd1`,
+/// which is equivalent to `min(outer_ndv, inner_ndv) / outer_ndv`.
+/// If either side lacks statistics it falls back to a default.
+fn estimate_semi_join_cardinality(
+ outer_num_rows: &Precision<usize>,
+ inner_num_rows: &Precision<usize>,
+ outer_col_stats: &[ColumnStatistics],
+ inner_col_stats: &[ColumnStatistics],
+) -> Option<usize> {
+ let outer_rows = *outer_num_rows.get_value()?;
+ if outer_rows == 0 {
+ return Some(0);
+ }
+ let inner_rows = *inner_num_rows.get_value()?;
+ if inner_rows == 0 {
+ return Some(0);
+ }
+
+ let mut selectivity = 1.0_f64;
+ let mut has_selectivity_estimate = false;
+
+ for (outer_stat, inner_stat) in outer_col_stats.iter().zip(inner_col_stats.iter()) {
+ let outer_has_stats = outer_stat.distinct_count.get_value().is_some()
+ || (outer_stat.min_value.get_value().is_some()
+ && outer_stat.max_value.get_value().is_some());
+ let inner_has_stats = inner_stat.distinct_count.get_value().is_some()
+ || (inner_stat.min_value.get_value().is_some()
+ && inner_stat.max_value.get_value().is_some());
+ if !outer_has_stats || !inner_has_stats {
+ continue;
+ }
+
+ let outer_ndv = max_distinct_count(outer_num_rows, outer_stat);
+ let inner_ndv = max_distinct_count(inner_num_rows, inner_stat);
+
+ if let (Some(&o), Some(&i)) = (outer_ndv.get_value(), inner_ndv.get_value())
+ && o > 0
+ {
+ let null_frac = outer_stat
+ .null_count
+ .get_value()
+ .map(|&nc| nc as f64 / outer_rows as f64)
+ .unwrap_or(0.0);
+ selectivity *= (o.min(i) as f64) / (o as f64) * (1.0 - null_frac);
+ has_selectivity_estimate = true;
+ }
+ }
+
+ if has_selectivity_estimate {
+ Some((outer_rows as f64 * selectivity).ceil() as usize)
+ } else {
+ None
+ }
+}
+
/// Estimate the number of maximum distinct values that can be present in the
/// given column from its statistics. If distinct_count is available, uses it
/// directly. Otherwise, if the column is numeric and has min/max values, it
@@ -2697,7 +2799,7 @@ mod tests {
JoinType::LeftSemi,
(50, Inexact(10), Inexact(20), Absent, Absent),
(10, Inexact(15), Inexact(25), Absent, Absent),
- Some(50),
+ Some(46),
),
(
JoinType::RightSemi,
@@ -2733,13 +2835,13 @@ mod tests {
JoinType::LeftAnti,
(50, Inexact(10), Inexact(20), Absent, Absent),
(10, Inexact(15), Inexact(25), Absent, Absent),
- Some(50),
+ Some(4),
),
(
JoinType::RightAnti,
(50, Inexact(10), Inexact(20), Absent, Absent),
(10, Inexact(15), Inexact(25), Absent, Absent),
- Some(10),
+ Some(0),
),
(
JoinType::LeftAnti,
@@ -2765,6 +2867,108 @@ mod tests {
(10, Inexact(30), Absent, Absent, Absent),
Some(50),
),
+ // NDV-based semi join: outer_ndv=20, inner_ndv=10
+ // selectivity = 10/20 = 0.5, cardinality = ceil(50 * 0.5) = 25
+ (
+ JoinType::LeftSemi,
+ (50, Inexact(1), Inexact(100), Inexact(20), Absent),
+ (10, Inexact(1), Inexact(100), Inexact(10), Absent),
+ Some(25),
+ ),
+ // inner_ndv(30) >= outer_ndv(20) -> selectivity 1.0, no reduction
+ (
+ JoinType::LeftSemi,
+ (50, Inexact(1), Inexact(100), Inexact(20), Absent),
+ (100, Inexact(1), Inexact(100), Inexact(30), Absent),
+ Some(50),
+ ),
+ // NDV-based anti join: semi=25, anti = 50 - 25 = 25
+ (
+ JoinType::LeftAnti,
+ (50, Inexact(1), Inexact(100), Inexact(20), Absent),
+ (10, Inexact(1), Inexact(100), Inexact(10), Absent),
+ Some(25),
+ ),
+ // inner covers all outer: semi=50, anti = 0
+ (
+ JoinType::LeftAnti,
+ (50, Inexact(1), Inexact(100), Inexact(20), Absent),
+ (100, Inexact(1), Inexact(100), Inexact(30), Absent),
+ Some(0),
+ ),
+ // RightSemi with explicit NDV (NDV within row count, used as-is):
+ // For RightSemi, sides are swapped: outer = right (20 rows, ndv=10),
+ // inner = left (50 rows, ndv=5). selectivity = min(10,5)/10 = 0.5,
+ // cardinality = ceil(20 * 0.5) = 10.
+ (
+ JoinType::RightSemi,
+ (50, Inexact(1), Inexact(100), Inexact(5), Absent),
+ (20, Inexact(1), Inexact(100), Inexact(10), Absent),
+ Some(10),
+ ),
+ // RightAnti with explicit NDV: anti = outer_rows - semi = 20 - 10 = 10.
+ (
+ JoinType::RightAnti,
+ (50, Inexact(1), Inexact(100), Inexact(5), Absent),
+ (20, Inexact(1), Inexact(100), Inexact(10), Absent),
+ Some(10),
+ ),
+ // RightSemi where right-side NDV (20) exceeds right-side row count (10):
+ // NDV is clamped to 10, so outer_ndv=10, inner_ndv=10,
+ // selectivity = min(10,10)/10 = 1.0, cardinality = ceil(10 * 1.0) = 10.
+ (
+ JoinType::RightSemi,
+ (50, Inexact(1), Inexact(100), Inexact(10), Absent),
+ (10, Inexact(1), Inexact(100), Inexact(20), Absent),
+ Some(10),
+ ),
+ // RightAnti with NDV clamped by row count: anti = 10 - 10 = 0.
+ (
+ JoinType::RightAnti,
+ (50, Inexact(1), Inexact(100), Inexact(10), Absent),
+ (10, Inexact(1), Inexact(100), Inexact(20), Absent),
+ Some(0),
+ ),
+ // Empty inner table: no match possible, semi → 0
+ (
+ JoinType::LeftSemi,
+ (100, Absent, Absent, Absent, Absent),
+ (0, Absent, Absent, Absent, Absent),
+ Some(0),
+ ),
+ // NDV-based semi with nulls on outer side:
+ // outer_ndv=20, inner_ndv=10, null_frac=10/100=0.1
+ // selectivity = 10/20 * (1-0.1) = 0.5 * 0.9 = 0.45
+ // semi = ceil(100 * 0.45) = 45
+ (
+ JoinType::LeftSemi,
+ (100, Absent, Absent, Inexact(20), Inexact(10)),
+ (200, Absent, Absent, Inexact(10), Absent),
+ Some(45),
+ ),
+ // Anti-join with nulls on outer side:
+ // semi=45, anti = 100 - 45 = 55
+ (
+ JoinType::LeftAnti,
+ (100, Absent, Absent, Inexact(20), Inexact(10)),
+ (200, Absent, Absent, Inexact(10), Absent),
+ Some(55),
+ ),
+ // All outer rows are null: null_frac=1.0
+ // selectivity = 10/20 * (1-1.0) = 0.0, semi = 0
+ (
+ JoinType::LeftSemi,
+ (100, Absent, Absent, Inexact(20), Inexact(100)),
+ (200, Absent, Absent, Inexact(10), Absent),
+ Some(0),
+ ),
+ // All outer rows are null (anti): anti = 100 - 0 = 100
+ (
+ JoinType::LeftAnti,
+ (100, Absent, Absent, Inexact(20), Inexact(100)),
+ (200, Absent, Absent, Inexact(10), Absent),
+ Some(100),
+ ),
];
let join_on = vec![(
@@ -2884,6 +3088,157 @@ mod tests {
Ok(())
}
+ #[test]
+ fn test_semi_join_multi_column_and_mixed_stats() -> Result<()> {
+ let join_on = vec![
+ (
+ Arc::new(Column::new("l_col0", 0)) as _,
+ Arc::new(Column::new("r_col0", 0)) as _,
+ ),
+ (
+ Arc::new(Column::new("l_col1", 1)) as _,
+ Arc::new(Column::new("r_col1", 1)) as _,
+ ),
+ ];
+
+ // Multi-column: both columns have NDV on both sides.
+ // col0: outer_ndv=20, inner_ndv=10 → selectivity = 10/20 = 0.5
+ // col1: outer_ndv=40, inner_ndv=10 → selectivity = 10/40 = 0.25
+ // total selectivity = 0.5 * 0.25 = 0.125
+ // semi = ceil(100 * 0.125) = 13
+ let result = estimate_join_cardinality(
+ &JoinType::LeftSemi,
+ Statistics {
+ num_rows: Inexact(100),
+ total_byte_size: Absent,
+ column_statistics: vec![
+ create_column_stats(Absent, Absent, Inexact(20), Absent),
+ create_column_stats(Absent, Absent, Inexact(40), Absent),
+ ],
+ },
+ Statistics {
+ num_rows: Inexact(200),
+ total_byte_size: Absent,
+ column_statistics: vec![
+ create_column_stats(Absent, Absent, Inexact(10), Absent),
+ create_column_stats(Absent, Absent, Inexact(10), Absent),
+ ],
+ },
+ &join_on,
+ )
+ .map(|c| c.num_rows);
+ assert_eq!(result, Some(13), "multi-column semi join");
+
+ // Multi-column anti: anti = 100 - 13 = 87
+ let result = estimate_join_cardinality(
+ &JoinType::LeftAnti,
+ Statistics {
+ num_rows: Inexact(100),
+ total_byte_size: Absent,
+ column_statistics: vec![
+ create_column_stats(Absent, Absent, Inexact(20), Absent),
+ create_column_stats(Absent, Absent, Inexact(40), Absent),
+ ],
+ },
+ Statistics {
+ num_rows: Inexact(200),
+ total_byte_size: Absent,
+ column_statistics: vec![
+ create_column_stats(Absent, Absent, Inexact(10), Absent),
+ create_column_stats(Absent, Absent, Inexact(10), Absent),
+ ],
+ },
+ &join_on,
+ )
+ .map(|c| c.num_rows);
+ assert_eq!(result, Some(87), "multi-column anti join");
+
+ // Mixed stats: col0 has NDV on both sides, col1 has NDV only on outer.
+ // col1 is skipped (either side missing), so selectivity comes from col0 only.
+ // col0: outer_ndv=20, inner_ndv=10 → selectivity = 0.5
+ // semi = ceil(100 * 0.5) = 50
+ let result = estimate_join_cardinality(
+ &JoinType::LeftSemi,
+ Statistics {
+ num_rows: Inexact(100),
+ total_byte_size: Absent,
+ column_statistics: vec![
+ create_column_stats(Absent, Absent, Inexact(20), Absent),
+ create_column_stats(Absent, Absent, Inexact(40), Absent),
+ ],
+ },
+ Statistics {
+ num_rows: Inexact(200),
+ total_byte_size: Absent,
+ column_statistics: vec![
+ create_column_stats(Absent, Absent, Inexact(10), Absent),
+ create_column_stats(Absent, Absent, Absent, Absent),
+ ],
+ },
+ &join_on,
+ )
+ .map(|c| c.num_rows);
+ assert_eq!(result, Some(50), "mixed stats: col1 skipped");
+
+ // Mixed stats: neither column has stats on both sides → fallback to outer_rows
+ let result = estimate_join_cardinality(
+ &JoinType::LeftSemi,
+ Statistics {
+ num_rows: Inexact(100),
+ total_byte_size: Absent,
+ column_statistics: vec![
+ create_column_stats(Absent, Absent, Inexact(20), Absent),
+ create_column_stats(Absent, Absent, Absent, Absent),
+ ],
+ },
+ Statistics {
+ num_rows: Inexact(200),
+ total_byte_size: Absent,
+ column_statistics: vec![
+ create_column_stats(Absent, Absent, Absent, Absent),
+ create_column_stats(Absent, Absent, Inexact(10), Absent),
+ ],
+ },
+ &join_on,
+ )
+ .map(|c| c.num_rows);
+ assert_eq!(result, Some(100), "no column has stats on both sides");
+
+ // Multi-column with nulls on one column:
+ // col0: outer_ndv=20, inner_ndv=10, null_frac=0.0 → 10/20 * 1.0 = 0.5
+ // col1: outer_ndv=40, inner_ndv=10, null_frac=20/100=0.2 → 10/40 * 0.8 = 0.2
+ // total selectivity = 0.5 * 0.2 = 0.1
+ // semi = ceil(100 * 0.1) = 10
+ let result = estimate_join_cardinality(
+ &JoinType::LeftSemi,
+ Statistics {
+ num_rows: Inexact(100),
+ total_byte_size: Absent,
+ column_statistics: vec![
+ create_column_stats(Absent, Absent, Inexact(20), Absent),
+ create_column_stats(Absent, Absent, Inexact(40), Inexact(20)),
+ ],
+ },
+ Statistics {
+ num_rows: Inexact(200),
+ total_byte_size: Absent,
+ column_statistics: vec![
+ create_column_stats(Absent, Absent, Inexact(10), Absent),
+ create_column_stats(Absent, Absent, Inexact(10), Absent),
+ ],
+ },
+ &join_on,
+ )
+ .map(|c| c.num_rows);
+ assert_eq!(
+ result,
+ Some(10),
+ "multi-column semi join with nulls on one column"
+ );
+
+ Ok(())
+ }
+
#[test]
fn test_calculate_join_output_ordering() -> Result<()> {
let left_ordering = LexOrdering::new(vec![