Apache DataFusion SQL Query Engine
fix: handle zero endpoints in division interval bounds (#25985)
Huaijin10a6bc5fce7d
diff --git a/datafusion/expr-common/src/interval_arithmetic.rs b/datafusion/expr-common/src/interval_arithmetic.rsindex 8df35d363a..797c48cbe0 100644--- a/datafusion/expr-common/src/interval_arithmetic.rs+++ b/datafusion/expr-common/src/interval_arithmetic.rs@@ -859,10 +859,13 @@ impl Interval {let zero = ScalarValue::new_zero(&dt)?;// We want 0 to be approachable from both negative and positive sides.let zero_point = match &dt {- DataType::Float32 | DataType::Float64 => Self::new(zero.clone(), zero),- _ => Self::new(prev_value(zero.clone()), next_value(zero)),+ DataType::Float32 | DataType::Float64 => {+ Self::new(zero.clone(), zero.clone())+ }+ _ => Self::new(prev_value(zero.clone()), next_value(zero.clone())),};+ // The helpers treat intervals with a finite upper bound <= 0 as non-positive.// Exit early with an unbounded interval if zero is strictly inside the// right hand side:if rhs_ref.contains(&zero_point)? == Self::TRUE && !dt.is_unsigned_integer() {@@ -873,19 +876,9 @@ impl Interval {else if lhs_ref.contains(&zero_point)? == Self::TRUE&& !dt.is_unsigned_integer(){- Ok(div_helper_lhs_zero_inclusive(- &dt,- lhs_ref,- rhs_ref,- &zero_point,- ))+ Ok(div_helper_lhs_zero_inclusive(&dt, lhs_ref, rhs_ref, &zero))} else {- Ok(div_helper_zero_exclusive(- &dt,- lhs_ref,- rhs_ref,- &zero_point,- ))+ Ok(div_helper_zero_exclusive(&dt, lhs_ref, rhs_ref, &zero))}}@@ -1652,10 +1645,10 @@ fn div_helper_lhs_zero_inclusive(dt: &DataType,lhs: &Interval,rhs: &Interval,- zero_point: &Interval,+ zero: &ScalarValue,) -> Interval {// With the following interval bounds, there is no possibility to create an invalid interval.- if rhs.upper <= zero_point.lower && !rhs.upper.is_null() {+ if rhs.upper <= *zero && !rhs.upper.is_null() {// <-------=====0=====------->// <--======----0------------>let lower = div_bounds::<false>(dt, &lhs.upper, &rhs.upper);@@ -1671,12 +1664,12 @@ fn div_helper_lhs_zero_inclusive(}/// Divides the left-hand side interval by the right-hand side interval when-/// neither interval contains zero.+/// neither interval straddles zero (zero endpoints are allowed).////// This function takes in two intervals (`lhs` and `rhs`) as arguments and/// returns their quotient (whose data type is known to be `dt`). It is-/// specifically designed to handle intervals that do not contain zero within-/// their ranges. Returns an error if the division of bounds fails.+/// specifically designed to handle intervals that do not straddle zero.+/// Returns an error if the division of bounds fails.////// ``` text/// Left-hand side: <--======----0------------>@@ -1705,11 +1698,11 @@ fn div_helper_zero_exclusive(dt: &DataType,lhs: &Interval,rhs: &Interval,- zero_point: &Interval,+ zero: &ScalarValue,) -> Interval {let (lower, upper) = match (- lhs.upper <= zero_point.lower && !lhs.upper.is_null(),- rhs.upper <= zero_point.lower && !rhs.upper.is_null(),+ lhs.upper <= *zero && !lhs.upper.is_null(),+ rhs.upper <= *zero && !rhs.upper.is_null(),) {// With the following interval bounds, there is no possibility to create an invalid interval.(true, true) => (@@ -3910,9 +3903,123 @@ mod tests {Ok(())}+ #[test]+ fn test_div_small_integer_intervals_contain_runtime_results() -> Result<()> {+ // Include zero at either endpoint, inside the range, and as a singleton.+ for lo in -3_i64..=3 {+ for hi in lo..=3 {+ for rlo in -3_i64..=3 {+ for rhi in rlo..=3 {+ let lhs = Interval::make(Some(lo), Some(hi))?;+ let rhs = Interval::make(Some(rlo), Some(rhi))?;+ let result = lhs.div(&rhs)?;+ for a in lo..=hi {+ for b in rlo..=rhi {+ if b != 0 {+ assert!(+ result.contains_value(ScalarValue::Int64(+ Some(a / b)+ ))?,+ "{lhs:?} / {rhs:?} = {result:?} excludes {a} / {b}",+ );+ }+ }+ }+ }+ }+ }+ }+ Ok(())+ }+#[test]fn test_div() -> Result<()> {let cases = vec![+ (+ Interval::make(Some(-3_i64), Some(0_i64))?,+ Interval::make(Some(-2_i64), Some(-1_i64))?,+ Interval::make(Some(0_i64), Some(3_i64))?,+ ),+ (+ Interval::make(Some(-3_i64), Some(0_i64))?,+ Interval::make(Some(1_i64), Some(2_i64))?,+ Interval::make(Some(-3_i64), Some(0_i64))?,+ ),+ (+ Interval::make(Some(0_i64), Some(3_i64))?,+ Interval::make(Some(-2_i64), Some(-1_i64))?,+ Interval::make(Some(-3_i64), Some(0_i64))?,+ ),+ (+ Interval::make(Some(0_i64), Some(3_i64))?,+ Interval::make(Some(1_i64), Some(2_i64))?,+ Interval::make(Some(0_i64), Some(3_i64))?,+ ),+ (+ Interval::make(Some(-3_i64), Some(-1_i64))?,+ Interval::make(Some(-3_i64), Some(0_i64))?,+ Interval::make(Some(0_i64), None)?,+ ),+ (+ Interval::make(Some(1_i64), Some(3_i64))?,+ Interval::make(Some(-3_i64), Some(0_i64))?,+ Interval::make(None, Some(0_i64))?,+ ),+ (+ Interval::make(Some(-3_i64), Some(-1_i64))?,+ Interval::make(Some(0_i64), Some(3_i64))?,+ Interval::make(None, Some(0_i64))?,+ ),+ (+ Interval::make(Some(1_i64), Some(3_i64))?,+ Interval::make(Some(0_i64), Some(3_i64))?,+ Interval::make(Some(0_i64), None)?,+ ),+ (+ Interval::make(Some(-6_i64), Some(6_i64))?,+ Interval::make(Some(-3_i64), Some(0_i64))?,+ Interval::make::<i64>(None, None)?,+ ),+ (+ Interval::make(Some(-6_i64), Some(6_i64))?,+ Interval::make(Some(0_i64), Some(3_i64))?,+ Interval::make::<i64>(None, None)?,+ ),+ (+ Interval::make(Some(0_i64), Some(0_i64))?,+ Interval::make(Some(0_i64), Some(0_i64))?,+ Interval::make::<i64>(None, None)?,+ ),+ (+ Interval::make(Some(0_u32), Some(0_u32))?,+ Interval::make(Some(0_u32), Some(5_u32))?,+ Interval::make(Some(0_u32), Some(0_u32))?,+ ),+ (+ Interval::make(Some(0_u32), Some(0_u32))?,+ Interval::make(Some(0_u32), Some(0_u32))?,+ Interval::make::<u32>(None, None)?,+ ),+ (+ Interval::make(Some(0_u32), Some(5_u32))?,+ Interval::make(Some(0_u32), Some(5_u32))?,+ Interval::make(Some(0_u32), None)?,+ ),+ (+ Interval::make(Some(1_u32), Some(5_u32))?,+ Interval::make(Some(0_u32), Some(5_u32))?,+ Interval::make(Some(0_u32), None)?,+ ),+ (+ Interval::make(Some(0_u64), Some(0_u64))?,+ Interval::make(Some(0_u64), Some(5_u64))?,+ Interval::make(Some(0_u64), Some(0_u64))?,+ ),+ (+ Interval::make(Some(0_u64), Some(0_u64))?,+ Interval::make(Some(0_u64), Some(0_u64))?,+ Interval::make::<u64>(None, None)?,+ ),(Interval::make(Some(100_i64), Some(200_i64))?,Interval::make(Some(1_i64), Some(2_i64))?,@@ -4090,7 +4197,7 @@ mod tests {),];for case in cases {- let result = case.0.div(case.1)?;+ let result = case.0.div(&case.1)?;if case.0.data_type().is_floating() {assert!(result.lower().is_null() && case.2.lower().is_null()@@ -4101,7 +4208,7 @@ mod tests {|| result.upper().ge(case.2.upper()));} else {- assert_eq!(result, case.2);+ assert_eq!(result, case.2, "{:?} / {:?}", case.0, case.1);}}diff --git a/datafusion/sqllogictest/test_files/parquet_statistics.slt b/datafusion/sqllogictest/test_files/parquet_statistics.sltindex 2a8d174751..84383413fa 100644--- a/datafusion/sqllogictest/test_files/parquet_statistics.slt+++ b/datafusion/sqllogictest/test_files/parquet_statistics.slt@@ -222,6 +222,37 @@ physical_planstatement okDROP TABLE mul_wrap;+######+# Division with a zero numerator endpoint must not infer an empty filter+######++statement ok+COPY (+ SELECT * FROM (VALUES (-3, -1), (-2, -1), (0, -2)) AS t(a, b)+)+TO 'test_files/scratch/parquet_statistics/div_zero_endpoint.parquet' STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE div_zero_endpoint STORED AS PARQUET+LOCATION 'test_files/scratch/parquet_statistics/div_zero_endpoint.parquet';++# [-3,0] / [-2,-1] must include 2 and 3, rather than being narrowed to [0,1].+query III+SELECT a, b, a / b AS q FROM div_zero_endpoint WHERE a / b > 1 ORDER BY a;+----+-3 -1 3+-2 -1 2++query TT+EXPLAIN SELECT a, b FROM div_zero_endpoint WHERE a / b > 1;+----+physical_plan+01)FilterExec: a@0 / b@1 > 1, statistics=[Rows=Inexact(3), Bytes=Inexact(48), [(Col[0]: Min=Inexact(Int64(-3)) Max=Inexact(Int64(0)) Null=Inexact(0) ScanBytes=Inexact(24)),(Col[1]: Min=Inexact(Int64(-2)) Max=Inexact(Int64(-1)) Null=Inexact(0) ScanBytes=Inexact(24))]]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/div_zero_endpoint.parquet]]}, projection=[a, b], file_type=parquet, predicate=a@0 / b@1 > 1, statistics=[Rows=Inexact(3), Bytes=Inexact(48), [(Col[0]: Min=Inexact(Int64(-3)) Max=Inexact(Int64(0)) Null=Inexact(0) ScanBytes=Inexact(24)),(Col[1]: Min=Inexact(Int64(-2)) Max=Inexact(Int64(-1)) Null=Inexact(0) ScanBytes=Inexact(24))]]++statement ok+DROP TABLE div_zero_endpoint;+####### Aggregate statistics through casts######