Apache DataFusion SQL Query Engine
Fix Subtraction overflow in `max_distinct_count` when hash join has a pushed-down limit (#20799)
Kartik Guptaf734ec54dc1b
diff --git a/datafusion/physical-plan/src/joins/utils.rs b/datafusion/physical-plan/src/joins/utils.rsindex 3130134e25..55a3c5419f 100644--- a/datafusion/physical-plan/src/joins/utils.rs+++ b/datafusion/physical-plan/src/joins/utils.rs@@ -722,11 +722,12 @@ fn max_distinct_count(}}Precision::Exact(count) => {- let count = count - stats.null_count.get_value().unwrap_or(&0);+ let null_count = *stats.null_count.get_value().unwrap_or(&0);+ let non_null_count = count.checked_sub(null_count).unwrap_or(0);if stats.null_count.is_exact().unwrap_or(false) {- Precision::Exact(count)+ Precision::Exact(non_null_count)} else {- Precision::Inexact(count)+ Precision::Inexact(non_null_count)}}};@@ -2939,4 +2940,19 @@ mod tests {Ok(())}++ #[test]+ fn test_max_distinct_count_no_overflow_when_null_count_exceeds_num_rows() {+ let num_rows = Exact(2);+ let stats = ColumnStatistics {+ distinct_count: Absent,+ null_count: Exact(5),+ min_value: Absent,+ max_value: Absent,+ sum_value: Absent,+ byte_size: Absent,+ };+ let result = max_distinct_count(&num_rows, &stats);+ assert_eq!(result, Exact(0));+ }}diff --git a/datafusion/sqllogictest/test_files/joins.slt b/datafusion/sqllogictest/test_files/joins.sltindex 136a685735..c5b2ba5362 100644--- a/datafusion/sqllogictest/test_files/joins.slt+++ b/datafusion/sqllogictest/test_files/joins.slt@@ -5404,3 +5404,26 @@ DROP TABLE t1;statement count 0DROP TABLE t2;++statement ok+CREATE TABLE t1(a INT, b INT) AS VALUES+ (NULL, 1), (NULL, 2), (NULL, 3), (NULL, 4), (NULL, 5);++statement ok+CREATE TABLE t2(c INT) AS VALUES (1), (2);++# This query panicked before the fix: the ORDER BY forces a SortExec,+# the LIMIT gets pushed into SortExec.fetch, and the HashJoinExec+# calls partition_statistics() on the SortExec child during execution.+query II+SELECT sub.a, sub.b FROM (+ SELECT * FROM t1 ORDER BY b LIMIT 1+) sub+JOIN t2 ON sub.a = t2.c;+----++statement ok+DROP TABLE t1;++statement ok+DROP TABLE t2;