Skip to content

Apache DataFusion SQL Query Engine

fix: avoid overflow in join cardinality estimation (#23788)

xudong.w05b7e11c7e45
diff --git a/datafusion/physical-plan/src/joins/utils.rs b/datafusion/physical-plan/src/joins/utils.rs
index 90b39f7ada..654d873ae1 100644
--- a/datafusion/physical-plan/src/joins/utils.rs
+++ b/datafusion/physical-plan/src/joins/utils.rs
@@ -846,15 +846,22 @@ fn estimate_inner_join_cardinality(
// With the assumption that the smaller input's domain is generally represented in the bigger
// input's domain, we can estimate the inner join's cardinality by taking the cartesian product
// of the two inputs and normalizing it by the selectivity factor.
- let left_num_rows = left_stats.num_rows.get_value()?;
- let right_num_rows = right_stats.num_rows.get_value()?;
+ let left_num_rows = *left_stats.num_rows.get_value()?;
+ let right_num_rows = *right_stats.num_rows.get_value()?;
+ // Widen before multiplying so the intermediate Cartesian product does not
+ // overflow when the normalized cardinality is still representable as usize.
+ let cartesian_product = (left_num_rows as u128) * (right_num_rows as u128);
+ let normalized_cardinality =
+ |value: usize| usize::try_from(cartesian_product / value as u128);
match join_selectivity {
- Precision::Exact(value) if value > 0 => {
- Some(Precision::Exact((left_num_rows * right_num_rows) / value))
- }
- Precision::Inexact(value) if value > 0 => {
- Some(Precision::Inexact((left_num_rows * right_num_rows) / value))
- }
+ Precision::Exact(value) if value > 0 => Some(
+ normalized_cardinality(value)
+ .map(Precision::Exact)
+ .unwrap_or(Precision::Inexact(usize::MAX)),
+ ),
+ Precision::Inexact(value) if value > 0 => Some(Precision::Inexact(
+ normalized_cardinality(value).unwrap_or(usize::MAX),
+ )),
// Since we don't have any information about the selectivity (which is derived
// from the number of distinct rows information) we can give up here for now.
// And let other passes handle this (otherwise we would need to produce an
@@ -3071,6 +3078,46 @@ mod tests {
Ok(())
}
+ #[test]
+ fn test_inner_join_cardinality_multiplication_overflow() {
+ let statistics = |num_rows, distinct_count| Statistics {
+ num_rows,
+ total_byte_size: Absent,
+ column_statistics: vec![ColumnStatistics {
+ distinct_count,
+ ..Default::default()
+ }],
+ };
+ let large_row_count = usize::MAX / 2 + 1;
+
+ // The Cartesian product overflows usize, but applying the NDV divisor
+ // produces a representable cardinality.
+ assert_eq!(
+ estimate_inner_join_cardinality(
+ statistics(Inexact(large_row_count), Inexact(1)),
+ statistics(Inexact(3), Inexact(3)),
+ ),
+ Some(Inexact(large_row_count))
+ );
+ assert_eq!(
+ estimate_inner_join_cardinality(
+ statistics(Exact(large_row_count), Exact(1)),
+ statistics(Exact(3), Exact(3)),
+ ),
+ Some(Exact(large_row_count))
+ );
+
+ // If the normalized result itself cannot fit in usize, cap the
+ // estimate and mark it as inexact.
+ assert_eq!(
+ estimate_inner_join_cardinality(
+ statistics(Exact(usize::MAX), Exact(1)),
+ statistics(Exact(2), Exact(1)),
+ ),
+ Some(Inexact(usize::MAX))
+ );
+ }
+
#[test]
fn test_inner_join_cardinality_multiple_column() -> Result<()> {
let left_col_stats = vec![