Skip to content

Apache DataFusion SQL Query Engine

fix: handle zero endpoints in division interval bounds (#25985)

Huaijin10a6bc5fce7d
diff --git a/datafusion/expr-common/src/interval_arithmetic.rs b/datafusion/expr-common/src/interval_arithmetic.rs
index 8df35d363a..797c48cbe0 100644
--- a/datafusion/expr-common/src/interval_arithmetic.rs
+++ b/datafusion/expr-common/src/interval_arithmetic.rs
@@ -859,10 +859,13 @@ impl Interval {
let zero = ScalarValue::new_zero(&dt)?;
// We want 0 to be approachable from both negative and positive sides.
let zero_point = match &dt {
- DataType::Float32 | DataType::Float64 => Self::new(zero.clone(), zero),
- _ => Self::new(prev_value(zero.clone()), next_value(zero)),
+ DataType::Float32 | DataType::Float64 => {
+ Self::new(zero.clone(), zero.clone())
+ }
+ _ => Self::new(prev_value(zero.clone()), next_value(zero.clone())),
};
+ // The helpers treat intervals with a finite upper bound <= 0 as non-positive.
// Exit early with an unbounded interval if zero is strictly inside the
// right hand side:
if rhs_ref.contains(&zero_point)? == Self::TRUE && !dt.is_unsigned_integer() {
@@ -873,19 +876,9 @@ impl Interval {
else if lhs_ref.contains(&zero_point)? == Self::TRUE
&& !dt.is_unsigned_integer()
{
- Ok(div_helper_lhs_zero_inclusive(
- &dt,
- lhs_ref,
- rhs_ref,
- &zero_point,
- ))
+ Ok(div_helper_lhs_zero_inclusive(&dt, lhs_ref, rhs_ref, &zero))
} else {
- Ok(div_helper_zero_exclusive(
- &dt,
- lhs_ref,
- rhs_ref,
- &zero_point,
- ))
+ Ok(div_helper_zero_exclusive(&dt, lhs_ref, rhs_ref, &zero))
}
}
@@ -1652,10 +1645,10 @@ fn div_helper_lhs_zero_inclusive(
dt: &DataType,
lhs: &Interval,
rhs: &Interval,
- zero_point: &Interval,
+ zero: &ScalarValue,
) -> Interval {
// With the following interval bounds, there is no possibility to create an invalid interval.
- if rhs.upper <= zero_point.lower && !rhs.upper.is_null() {
+ if rhs.upper <= *zero && !rhs.upper.is_null() {
// <-------=====0=====------->
// <--======----0------------>
let lower = div_bounds::<false>(dt, &lhs.upper, &rhs.upper);
@@ -1671,12 +1664,12 @@ fn div_helper_lhs_zero_inclusive(
}
/// Divides the left-hand side interval by the right-hand side interval when
-/// neither interval contains zero.
+/// neither interval straddles zero (zero endpoints are allowed).
///
/// This function takes in two intervals (`lhs` and `rhs`) as arguments and
/// returns their quotient (whose data type is known to be `dt`). It is
-/// specifically designed to handle intervals that do not contain zero within
-/// their ranges. Returns an error if the division of bounds fails.
+/// specifically designed to handle intervals that do not straddle zero.
+/// Returns an error if the division of bounds fails.
///
/// ``` text
/// Left-hand side: <--======----0------------>
@@ -1705,11 +1698,11 @@ fn div_helper_zero_exclusive(
dt: &DataType,
lhs: &Interval,
rhs: &Interval,
- zero_point: &Interval,
+ zero: &ScalarValue,
) -> Interval {
let (lower, upper) = match (
- lhs.upper <= zero_point.lower && !lhs.upper.is_null(),
- rhs.upper <= zero_point.lower && !rhs.upper.is_null(),
+ lhs.upper <= *zero && !lhs.upper.is_null(),
+ rhs.upper <= *zero && !rhs.upper.is_null(),
) {
// With the following interval bounds, there is no possibility to create an invalid interval.
(true, true) => (
@@ -3910,9 +3903,123 @@ mod tests {
Ok(())
}
+ #[test]
+ fn test_div_small_integer_intervals_contain_runtime_results() -> Result<()> {
+ // Include zero at either endpoint, inside the range, and as a singleton.
+ for lo in -3_i64..=3 {
+ for hi in lo..=3 {
+ for rlo in -3_i64..=3 {
+ for rhi in rlo..=3 {
+ let lhs = Interval::make(Some(lo), Some(hi))?;
+ let rhs = Interval::make(Some(rlo), Some(rhi))?;
+ let result = lhs.div(&rhs)?;
+ for a in lo..=hi {
+ for b in rlo..=rhi {
+ if b != 0 {
+ assert!(
+ result.contains_value(ScalarValue::Int64(
+ Some(a / b)
+ ))?,
+ "{lhs:?} / {rhs:?} = {result:?} excludes {a} / {b}",
+ );
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ Ok(())
+ }
+
#[test]
fn test_div() -> Result<()> {
let cases = vec![
+ (
+ Interval::make(Some(-3_i64), Some(0_i64))?,
+ Interval::make(Some(-2_i64), Some(-1_i64))?,
+ Interval::make(Some(0_i64), Some(3_i64))?,
+ ),
+ (
+ Interval::make(Some(-3_i64), Some(0_i64))?,
+ Interval::make(Some(1_i64), Some(2_i64))?,
+ Interval::make(Some(-3_i64), Some(0_i64))?,
+ ),
+ (
+ Interval::make(Some(0_i64), Some(3_i64))?,
+ Interval::make(Some(-2_i64), Some(-1_i64))?,
+ Interval::make(Some(-3_i64), Some(0_i64))?,
+ ),
+ (
+ Interval::make(Some(0_i64), Some(3_i64))?,
+ Interval::make(Some(1_i64), Some(2_i64))?,
+ Interval::make(Some(0_i64), Some(3_i64))?,
+ ),
+ (
+ Interval::make(Some(-3_i64), Some(-1_i64))?,
+ Interval::make(Some(-3_i64), Some(0_i64))?,
+ Interval::make(Some(0_i64), None)?,
+ ),
+ (
+ Interval::make(Some(1_i64), Some(3_i64))?,
+ Interval::make(Some(-3_i64), Some(0_i64))?,
+ Interval::make(None, Some(0_i64))?,
+ ),
+ (
+ Interval::make(Some(-3_i64), Some(-1_i64))?,
+ Interval::make(Some(0_i64), Some(3_i64))?,
+ Interval::make(None, Some(0_i64))?,
+ ),
+ (
+ Interval::make(Some(1_i64), Some(3_i64))?,
+ Interval::make(Some(0_i64), Some(3_i64))?,
+ Interval::make(Some(0_i64), None)?,
+ ),
+ (
+ Interval::make(Some(-6_i64), Some(6_i64))?,
+ Interval::make(Some(-3_i64), Some(0_i64))?,
+ Interval::make::<i64>(None, None)?,
+ ),
+ (
+ Interval::make(Some(-6_i64), Some(6_i64))?,
+ Interval::make(Some(0_i64), Some(3_i64))?,
+ Interval::make::<i64>(None, None)?,
+ ),
+ (
+ Interval::make(Some(0_i64), Some(0_i64))?,
+ Interval::make(Some(0_i64), Some(0_i64))?,
+ Interval::make::<i64>(None, None)?,
+ ),
+ (
+ Interval::make(Some(0_u32), Some(0_u32))?,
+ Interval::make(Some(0_u32), Some(5_u32))?,
+ Interval::make(Some(0_u32), Some(0_u32))?,
+ ),
+ (
+ Interval::make(Some(0_u32), Some(0_u32))?,
+ Interval::make(Some(0_u32), Some(0_u32))?,
+ Interval::make::<u32>(None, None)?,
+ ),
+ (
+ Interval::make(Some(0_u32), Some(5_u32))?,
+ Interval::make(Some(0_u32), Some(5_u32))?,
+ Interval::make(Some(0_u32), None)?,
+ ),
+ (
+ Interval::make(Some(1_u32), Some(5_u32))?,
+ Interval::make(Some(0_u32), Some(5_u32))?,
+ Interval::make(Some(0_u32), None)?,
+ ),
+ (
+ Interval::make(Some(0_u64), Some(0_u64))?,
+ Interval::make(Some(0_u64), Some(5_u64))?,
+ Interval::make(Some(0_u64), Some(0_u64))?,
+ ),
+ (
+ Interval::make(Some(0_u64), Some(0_u64))?,
+ Interval::make(Some(0_u64), Some(0_u64))?,
+ Interval::make::<u64>(None, None)?,
+ ),
(
Interval::make(Some(100_i64), Some(200_i64))?,
Interval::make(Some(1_i64), Some(2_i64))?,
@@ -4090,7 +4197,7 @@ mod tests {
),
];
for case in cases {
- let result = case.0.div(case.1)?;
+ let result = case.0.div(&case.1)?;
if case.0.data_type().is_floating() {
assert!(
result.lower().is_null() && case.2.lower().is_null()
@@ -4101,7 +4208,7 @@ mod tests {
|| result.upper().ge(case.2.upper())
);
} else {
- assert_eq!(result, case.2);
+ assert_eq!(result, case.2, "{:?} / {:?}", case.0, case.1);
}
}
diff --git a/datafusion/sqllogictest/test_files/parquet_statistics.slt b/datafusion/sqllogictest/test_files/parquet_statistics.slt
index 2a8d174751..84383413fa 100644
--- a/datafusion/sqllogictest/test_files/parquet_statistics.slt
+++ b/datafusion/sqllogictest/test_files/parquet_statistics.slt
@@ -222,6 +222,37 @@ physical_plan
statement ok
DROP TABLE mul_wrap;
+######
+# Division with a zero numerator endpoint must not infer an empty filter
+######
+
+statement ok
+COPY (
+ SELECT * FROM (VALUES (-3, -1), (-2, -1), (0, -2)) AS t(a, b)
+)
+TO 'test_files/scratch/parquet_statistics/div_zero_endpoint.parquet' STORED AS PARQUET;
+
+statement ok
+CREATE EXTERNAL TABLE div_zero_endpoint STORED AS PARQUET
+LOCATION 'test_files/scratch/parquet_statistics/div_zero_endpoint.parquet';
+
+# [-3,0] / [-2,-1] must include 2 and 3, rather than being narrowed to [0,1].
+query III
+SELECT a, b, a / b AS q FROM div_zero_endpoint WHERE a / b > 1 ORDER BY a;
+----
+-3 -1 3
+-2 -1 2
+
+query TT
+EXPLAIN SELECT a, b FROM div_zero_endpoint WHERE a / b > 1;
+----
+physical_plan
+01)FilterExec: a@0 / b@1 > 1, statistics=[Rows=Inexact(3), Bytes=Inexact(48), [(Col[0]: Min=Inexact(Int64(-3)) Max=Inexact(Int64(0)) Null=Inexact(0) ScanBytes=Inexact(24)),(Col[1]: Min=Inexact(Int64(-2)) Max=Inexact(Int64(-1)) Null=Inexact(0) ScanBytes=Inexact(24))]]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/div_zero_endpoint.parquet]]}, projection=[a, b], file_type=parquet, predicate=a@0 / b@1 > 1, statistics=[Rows=Inexact(3), Bytes=Inexact(48), [(Col[0]: Min=Inexact(Int64(-3)) Max=Inexact(Int64(0)) Null=Inexact(0) ScanBytes=Inexact(24)),(Col[1]: Min=Inexact(Int64(-2)) Max=Inexact(Int64(-1)) Null=Inexact(0) ScanBytes=Inexact(24))]]
+
+statement ok
+DROP TABLE div_zero_endpoint;
+
######
# Aggregate statistics through casts
######