Apache DataFusion SQL Query Engine
fix: preserve unbounded multiplication interval endpoints (#25795)
Huaijined43e69fdca8
diff --git a/datafusion/expr-common/src/interval_arithmetic.rs b/datafusion/expr-common/src/interval_arithmetic.rsindex 9f1291353d..48bf8a391d 100644--- a/datafusion/expr-common/src/interval_arithmetic.rs+++ b/datafusion/expr-common/src/interval_arithmetic.rs@@ -663,8 +663,8 @@ impl Interval {return Ok(None);}- let lower = max_of_bounds(&lhs.lower, &rhs.lower);- let upper = min_of_bounds(&lhs.upper, &rhs.upper);+ let lower = max_lower_bound(&lhs.lower, &rhs.lower);+ let upper = min_upper_bound(&lhs.upper, &rhs.upper);// New lower and upper bounds must always construct a valid interval.debug_assert!(@@ -686,18 +686,8 @@ impl Interval {let lhs = lhs_owned.as_ref().unwrap_or(self);let rhs = rhs_owned.as_ref().unwrap_or(rhs);- let lower =- if lhs.lower.is_null() || (!rhs.lower.is_null() && lhs.lower <= rhs.lower) {- lhs.lower.clone()- } else {- rhs.lower.clone()- };- let upper =- if lhs.upper.is_null() || (!rhs.upper.is_null() && lhs.upper >= rhs.upper) {- lhs.upper.clone()- } else {- rhs.upper.clone()- };+ let lower = min_lower_bound(&lhs.lower, &rhs.lower);+ let upper = max_upper_bound(&lhs.upper, &rhs.upper);// New lower and upper bounds must always construct a valid interval.debug_assert!(@@ -1346,9 +1336,17 @@ fn next_value_helper<const INC: bool>(value: ScalarValue) -> ScalarValue {}}-/// Returns the greater of the given interval bounds. Assumes that a `NULL`-/// value represents `NEG_INF`.-fn max_of_bounds(first: &ScalarValue, second: &ScalarValue) -> ScalarValue {+/// Returns the lesser of two lower bounds, treating `NULL` as `NEG_INF`.+fn min_lower_bound(first: &ScalarValue, second: &ScalarValue) -> ScalarValue {+ if first.is_null() || (!second.is_null() && first <= second) {+ first.clone()+ } else {+ second.clone()+ }+}++/// Returns the greater of two lower bounds, treating `NULL` as `NEG_INF`.+fn max_lower_bound(first: &ScalarValue, second: &ScalarValue) -> ScalarValue {if !first.is_null() && (second.is_null() || first >= second) {first.clone()} else {@@ -1356,9 +1354,8 @@ fn max_of_bounds(first: &ScalarValue, second: &ScalarValue) -> ScalarValue {}}-/// Returns the lesser of the given interval bounds. Assumes that a `NULL`-/// value represents `INF`.-fn min_of_bounds(first: &ScalarValue, second: &ScalarValue) -> ScalarValue {+/// Returns the lesser of two upper bounds, treating `NULL` as `INF`.+fn min_upper_bound(first: &ScalarValue, second: &ScalarValue) -> ScalarValue {if !first.is_null() && (second.is_null() || first <= second) {first.clone()} else {@@ -1366,6 +1363,15 @@ fn min_of_bounds(first: &ScalarValue, second: &ScalarValue) -> ScalarValue {}}+/// Returns the greater of two upper bounds, treating `NULL` as `INF`.+fn max_upper_bound(first: &ScalarValue, second: &ScalarValue) -> ScalarValue {+ if first.is_null() || (!second.is_null() && first >= second) {+ first.clone()+ } else {+ second.clone()+ }+}+/// This function updates the given intervals by enforcing (i.e. propagating)/// the inequality `left > right` (or the `left >= right` inequality, if `strict`/// is `true`).@@ -1482,13 +1488,14 @@ fn mul_helper_multi_zero_inclusive({return Interval::make_unbounded(dt).unwrap();}- // Since unbounded cases are handled above, we can safely- // use the utility functions here to eliminate code duplication.- let lower = min_of_bounds(+ // Even finite inputs can produce unbounded endpoints through overflow.+ // When combining the products, NULL lower bounds represent NEG_INF and+ // NULL upper bounds represent INF, so either must dominate finite bounds.+ let lower = min_lower_bound(&mul_bounds::<false>(dt, &lhs.lower, &rhs.upper),&mul_bounds::<false>(dt, &rhs.lower, &lhs.upper),);- let upper = max_of_bounds(+ let upper = max_upper_bound(&mul_bounds::<true>(dt, &lhs.upper, &rhs.upper),&mul_bounds::<true>(dt, &lhs.lower, &rhs.lower),);@@ -3481,6 +3488,29 @@ mod tests {Ok(())}+ #[test]+ fn test_mul_zero_inclusive_overflow() -> Result<()> {+ // Both operands contain zero. Overflow in either candidate endpoint+ // must not be hidden by the other, finite candidate.+ let cases = [+ ((0_i8, 20_i8), (-3, 7), (Some(-60_i8), None)),+ ((-20, 0), (-7, 3), (Some(-60), None)),+ ((-20, 0), (-3, 7), (None, Some(60))),+ ((0, 20), (-7, 3), (None, Some(60))),+ ((-20, 20), (-7, 7), (None, None)),+ ((-3, 3), (-7, 7), (Some(-21), Some(21))),+ ((0, 0), (i8::MIN, i8::MAX), (Some(0), Some(0))),+ ];+ for ((lo, hi), (rlo, rhi), (expected_lo, expected_hi)) in cases {+ let lhs = Interval::make(Some(lo), Some(hi))?;+ let rhs = Interval::make(Some(rlo), Some(rhi))?;+ let expected = Interval::make(expected_lo, expected_hi)?;+ assert_eq!(lhs.mul(&rhs)?, expected, "{lhs:?} * {rhs:?}");+ assert_eq!(rhs.mul(&lhs)?, expected, "{rhs:?} * {lhs:?}");+ }+ Ok(())+ }+#[test]fn test_mul() -> Result<()> {let cases = vec![diff --git a/datafusion/sqllogictest/test_files/parquet_statistics.slt b/datafusion/sqllogictest/test_files/parquet_statistics.sltindex 2c9a29c74b..71de383370 100644--- a/datafusion/sqllogictest/test_files/parquet_statistics.slt+++ b/datafusion/sqllogictest/test_files/parquet_statistics.slt@@ -186,6 +186,42 @@ physical_planstatement okDROP TABLE typed_table;+######+# Wrapping multiplication must not produce empty filter statistics+######++statement ok+SET datafusion.execution.target_partitions = 1;++statement ok+COPY (+ SELECT CAST(a AS TINYINT) AS a, CAST(b AS TINYINT) AS b+ FROM (VALUES (0, -3), (20, 7)) AS t(a, b)+)+TO 'test_files/scratch/parquet_statistics/mul_wrap.parquet' STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE mul_wrap STORED AS PARQUET+LOCATION 'test_files/scratch/parquet_statistics/mul_wrap.parquet';++# 20 * 7 wraps to -116 in Int8. Overflow in [0,20] * [-3,7] must+# remain visible to interval analysis, rather than producing [-60,0]+# and incorrectly concluding that the filter cannot match any rows.+query III+SELECT a, b, a * b AS x FROM mul_wrap WHERE a * b < -100;+----+20 7 -116++query TT+EXPLAIN SELECT a, b FROM mul_wrap WHERE a * b < -100;+----+physical_plan+01)FilterExec: a@0 * b@1 < -100, statistics=[Rows=Inexact(2), Bytes=Inexact(4), [(Col[0]: Min=Inexact(Int8(0)) Max=Inexact(Int8(20)) Null=Inexact(0) ScanBytes=Inexact(2)),(Col[1]: Min=Inexact(Int8(-3)) Max=Inexact(Int8(7)) Null=Inexact(0) ScanBytes=Inexact(2))]]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/mul_wrap.parquet]]}, projection=[a, b], file_type=parquet, predicate=a@0 * b@1 < -100, statistics=[Rows=Inexact(2), Bytes=Inexact(4), [(Col[0]: Min=Inexact(Int8(0)) Max=Inexact(Int8(20)) Null=Inexact(0) ScanBytes=Inexact(2)),(Col[1]: Min=Inexact(Int8(-3)) Max=Inexact(Int8(7)) Null=Inexact(0) ScanBytes=Inexact(2))]]++statement ok+DROP TABLE mul_wrap;+####### Aggregate statistics through casts######