Skip to content

Apache DataFusion SQL Query Engine

fix: prevent unsafe integer interval propagation (#25234)

Huaijinb9cea816dd20
diff --git a/datafusion/core/tests/fifo/mod.rs b/datafusion/core/tests/fifo/mod.rs
index edaee514d6..098aeaa537 100644
--- a/datafusion/core/tests/fifo/mod.rs
+++ b/datafusion/core/tests/fifo/mod.rs
@@ -239,9 +239,10 @@ mod unix_test {
// Create a mutex for tracking if the right input source is waiting for data.
let waiting = Arc::new(AtomicBool::new(true));
- // Create schema
+ // Use floating-point ordering so interval pruning does not depend on
+ // proving that integer arithmetic in the join filter cannot overflow.
let schema = Arc::new(Schema::new(vec![
- Field::new("a1", DataType::UInt32, false),
+ Field::new("a1", DataType::Float64, false),
Field::new("a2", DataType::UInt32, false),
]));
diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rs
index dfb1d136d0..1117d3fdd5 100644
--- a/datafusion/physical-expr/src/expressions/binary.rs
+++ b/datafusion/physical-expr/src/expressions/binary.rs
@@ -122,6 +122,23 @@ impl BinaryExpr {
&self.op
}
+ /// Mathematical intervals do not cover values produced by wrapping arithmetic.
+ fn integer_arithmetic_may_wrap(
+ &self,
+ left: &Interval,
+ right: &Interval,
+ result: &Interval,
+ ) -> bool {
+ !self.fail_on_overflow
+ && result.data_type().is_integer()
+ && matches!(
+ self.op,
+ Operator::Plus | Operator::Minus | Operator::Multiply
+ )
+ && (result.is_unbounded()
+ || unsigned_subtraction_may_underflow(self.op, left, right, result))
+ }
+
/// Wrapping on overflow breaks monotonicity (e.g. the sum of two
/// ascending `UInt8` columns can wrap back to small values), so the
/// derived ordering is kept only when overflow is impossible. `time ±
@@ -700,7 +717,12 @@ impl PhysicalExpr for BinaryExpr {
let left_interval = children[0];
let right_interval = children[1];
// Calculate current node's interval:
- apply_operator(&self.op, left_interval, right_interval)
+ let result = apply_operator(&self.op, left_interval, right_interval)?;
+ if self.integer_arithmetic_may_wrap(left_interval, right_interval, &result) {
+ Interval::make_unbounded(&result.data_type())
+ } else {
+ Ok(result)
+ }
}
fn propagate_constraints(
@@ -712,6 +734,36 @@ impl PhysicalExpr for BinaryExpr {
let left_interval = children[0];
let right_interval = children[1];
+ if left_interval.data_type().is_integer()
+ && right_interval.data_type().is_integer()
+ && matches!(
+ self.op,
+ Operator::Plus | Operator::Minus | Operator::Multiply | Operator::Divide
+ )
+ {
+ // Integer division truncates: a / 2 = 1 permits both 2 and 3.
+ // Wrapping arithmetic is likewise not invertible over mathematical
+ // intervals. Keep the input domains rather than exclude valid rows.
+ let contains_zero = |range: &Interval| -> Result<bool> {
+ range.contains_value(ScalarValue::new_zero(&range.data_type())?)
+ };
+ // If an operand can be zero, a zero product does not constrain
+ // the other operand. Dividing the parent interval loses that case.
+ let zero_product = self.op == Operator::Multiply
+ && contains_zero(interval)?
+ && (contains_zero(left_interval)? || contains_zero(right_interval)?);
+ if self.op == Operator::Divide
+ || zero_product
+ || self.integer_arithmetic_may_wrap(
+ left_interval,
+ right_interval,
+ &apply_operator(&self.op, left_interval, right_interval)?,
+ )
+ {
+ return Ok(Some(vec![]));
+ }
+ }
+
if self.op.eq(&Operator::And) {
if interval.eq(&Interval::TRUE) {
// A certainly true logical conjunction can only derive from possibly
@@ -832,7 +884,7 @@ impl PhysicalExpr for BinaryExpr {
let (r_order, r_range) = (children[1].sort_properties, &children[1].range);
match self.op() {
Operator::Plus => {
- let range = l_range.add(r_range)?;
+ let range = self.evaluate_bounds(&[l_range, r_range])?;
Ok(ExprProperties {
sort_properties: self.arithmetic_sort_properties(
l_order.add(&r_order),
@@ -846,7 +898,7 @@ impl PhysicalExpr for BinaryExpr {
})
}
Operator::Minus => {
- let range = l_range.sub(r_range)?;
+ let range = self.evaluate_bounds(&[l_range, r_range])?;
Ok(ExprProperties {
sort_properties: self.arithmetic_sort_properties(
l_order.sub(&r_order),
@@ -6353,6 +6405,246 @@ mod tests {
}
}
+ #[test]
+ fn test_integer_interval_propagation_covers_runtime_values() {
+ // Enumerate small domains and both ends of Int8, including zero divisors,
+ // truncation, signed overflow and checked arithmetic. Every successful
+ // runtime evaluation must remain possible after interval propagation.
+ let batch = RecordBatch::new_empty(Arc::new(Schema::empty()));
+ let domains = [
+ (-3i8, 3i8),
+ (0, 1),
+ (1, 3),
+ (-3, -1),
+ (-128, -127),
+ (126, 127),
+ ];
+ for checked in [false, true] {
+ for op in [
+ Operator::Plus,
+ Operator::Minus,
+ Operator::Multiply,
+ Operator::Divide,
+ ] {
+ let expr = BinaryExpr::new(lit(0i8), op, lit(0i8))
+ .with_fail_on_overflow(checked);
+ for (lo, hi) in domains {
+ for (rlo, rhi) in
+ domains.into_iter().chain([(-1, -1), (0, 0), (2, 2)])
+ {
+ let left = Interval::make(Some(lo), Some(hi)).unwrap();
+ let right = Interval::make(Some(rlo), Some(rhi)).unwrap();
+ let bounds = expr.evaluate_bounds(&[&left, &right]).unwrap();
+ if matches!(op, Operator::Plus | Operator::Minus) {
+ let children = [left.clone(), right.clone()].map(|range| {
+ ExprProperties {
+ range,
+ ..ExprProperties::new_unknown()
+ }
+ });
+ assert_eq!(
+ expr.get_properties(&children).unwrap().range,
+ bounds
+ );
+ }
+ for a in lo..=hi {
+ for b in rlo..=rhi {
+ // Use the execution kernel as the oracle rather than
+ // duplicating its checked and wrapping arithmetic.
+ let runtime = BinaryExpr::new(lit(a), op, lit(b))
+ .with_fail_on_overflow(checked);
+ let result = match runtime.evaluate(&batch) {
+ Ok(value) => value.into_array(1).unwrap(),
+ Err(error) => {
+ assert!(
+ checked || op == Operator::Divide,
+ "wrapping {a} {op} {b} failed: {error}"
+ );
+ continue;
+ }
+ };
+ let result = result.as_primitive::<Int8Type>().value(0);
+ let result =
+ Interval::make(Some(result), Some(result)).unwrap();
+ assert_eq!(
+ bounds.contains(&result).unwrap(),
+ Interval::TRUE,
+ "forward {a} {op} {b}, checked={checked}, bounds={bounds:?}"
+ );
+ let propagated = expr
+ .propagate_constraints(&result, &[&left, &right])
+ .unwrap();
+ let propagated = propagated
+ .expect("successful runtime result must be feasible");
+ if !propagated.is_empty() {
+ assert_eq!(
+ propagated[0]
+ .contains(
+ Interval::make(Some(a), Some(a)).unwrap()
+ )
+ .unwrap(),
+ Interval::TRUE,
+ "left input excluded for {a} {op} {b}, checked={checked}: {propagated:?}"
+ );
+ assert_eq!(
+ propagated[1]
+ .contains(
+ Interval::make(Some(b), Some(b)).unwrap()
+ )
+ .unwrap(),
+ Interval::TRUE,
+ "right input excluded for {a} {op} {b}, checked={checked}: {propagated:?}"
+ );
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ #[test]
+ fn test_integer_interval_error_propagation() {
+ // Compare error messages without backtraces, which differ by call path.
+ let integer = Interval::make(Some(1i32), Some(2i32)).unwrap();
+ let boolean = Interval::TRUE;
+ for op in [Operator::Plus, Operator::Minus] {
+ let expr = BinaryExpr::new(lit(1i32), op, lit(true));
+ let expected = apply_operator(&op, &integer, &boolean)
+ .unwrap_err()
+ .strip_backtrace();
+ assert_eq!(
+ expr.evaluate_bounds(&[&integer, &boolean])
+ .unwrap_err()
+ .strip_backtrace(),
+ expected
+ );
+ let children =
+ [integer.clone(), boolean.clone()].map(|range| ExprProperties {
+ range,
+ ..ExprProperties::new_unknown()
+ });
+ assert_eq!(
+ expr.get_properties(&children)
+ .unwrap_err()
+ .strip_backtrace(),
+ expected
+ );
+ }
+
+ // A nonnumeric parent cannot describe an integer product. Preserve
+ // the error from constructing zero for its unsupported type.
+ let parent = Interval::make_unbounded(&DataType::Utf8).unwrap();
+ let expr = BinaryExpr::new(lit(1i32), Operator::Multiply, lit(2i32));
+ assert_eq!(
+ expr.propagate_constraints(&parent, &[&integer, &integer])
+ .unwrap_err()
+ .strip_backtrace(),
+ ScalarValue::new_zero(&DataType::Utf8)
+ .unwrap_err()
+ .strip_backtrace()
+ );
+ }
+
+ #[test]
+ fn test_unsigned_subtraction_interval_underflow() {
+ let expr = BinaryExpr::new(lit(0u8), Operator::Minus, lit(1u8));
+ let left = Interval::make(Some(0u8), Some(2u8)).unwrap();
+ let right = Interval::make(Some(1u8), Some(1u8)).unwrap();
+ let wrapped = Interval::make(Some(255u8), Some(255u8)).unwrap();
+ assert_eq!(
+ expr.evaluate_bounds(&[&left, &right])
+ .unwrap()
+ .contains(&wrapped)
+ .unwrap(),
+ Interval::TRUE
+ );
+ assert_eq!(
+ expr.propagate_constraints(&wrapped, &[&left, &right])
+ .unwrap(),
+ Some(vec![])
+ );
+ }
+
+ #[test]
+ fn test_nested_wrapping_arithmetic_properties() {
+ let difference = Arc::new(BinaryExpr::new(lit(0u8), Operator::Minus, lit(1u8)));
+ let singleton = |value: u8| ExprProperties {
+ sort_properties: SortProperties::Singleton,
+ range: Interval::make(Some(value), Some(value)).unwrap(),
+ ..ExprProperties::new_unknown()
+ };
+ let difference_props = difference
+ .get_properties(&[singleton(0), singleton(1)])
+ .unwrap();
+ assert_eq!(difference_props.sort_properties, SortProperties::Singleton);
+ assert!(
+ difference_props
+ .range
+ .contains_value(ScalarValue::UInt8(Some(255)))
+ .unwrap()
+ );
+
+ // The inner constant wraps to 255. An incorrect range of [0, 0]
+ // would let the outer addition claim to preserve ascending order.
+ let expr =
+ BinaryExpr::new(Arc::new(Column::new("a", 0)), Operator::Plus, difference);
+ let properties = expr
+ .get_properties(&[
+ ExprProperties {
+ sort_properties: SortProperties::Ordered(SortOptions::default()),
+ range: Interval::make(Some(0u8), Some(1u8)).unwrap(),
+ ..ExprProperties::new_unknown()
+ },
+ difference_props,
+ ])
+ .unwrap();
+ let batch = RecordBatch::try_new(
+ Arc::new(Schema::new(vec![Field::new("a", DataType::UInt8, false)])),
+ vec![Arc::new(UInt8Array::from(vec![0, 1]))],
+ )
+ .unwrap();
+ let actual = expr.evaluate(&batch).unwrap().into_array(2).unwrap();
+ assert_eq!(actual.as_ref(), &UInt8Array::from(vec![255, 0]));
+ assert_eq!(properties.sort_properties, SortProperties::Unordered);
+ for value in [0u8, 255] {
+ assert!(
+ properties
+ .range
+ .contains_value(ScalarValue::UInt8(Some(value)))
+ .unwrap()
+ );
+ }
+ }
+
+ #[test]
+ fn test_integer_comparison_still_propagates() {
+ // Integer comparisons must bypass the arithmetic overflow guard and
+ // still narrow their inputs: a = 5 restricts a in [0, 10] to [5, 5].
+ let expr = BinaryExpr::new(lit(0i32), Operator::Eq, lit(5i32));
+ let left = Interval::make(Some(0i32), Some(10i32)).unwrap();
+ let right = Interval::make(Some(5i32), Some(5i32)).unwrap();
+ assert_eq!(
+ expr.propagate_constraints(&Interval::TRUE, &[&left, &right])
+ .unwrap(),
+ Some(vec![right.clone(), right])
+ );
+ }
+
+ #[test]
+ fn test_safe_integer_multiplication_still_propagates() {
+ let expr = BinaryExpr::new(lit(0i32), Operator::Multiply, lit(2i32));
+ let left = Interval::make(Some(0i32), Some(10i32)).unwrap();
+ let right = Interval::make(Some(2i32), Some(2i32)).unwrap();
+ let parent = Interval::make(Some(4i32), Some(4i32)).unwrap();
+ assert_eq!(
+ expr.propagate_constraints(&parent, &[&left, &right])
+ .unwrap(),
+ Some(vec![Interval::make(Some(2i32), Some(2i32)).unwrap(), right])
+ );
+ }
+
#[test]
fn test_evaluate_bounds_int32() {
let schema = Schema::new(vec![
diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs
index ae87168ec7..7377ec8936 100644
--- a/datafusion/physical-plan/src/filter.rs
+++ b/datafusion/physical-plan/src/filter.rs
@@ -2344,6 +2344,18 @@ mod tests {
)),
)),
));
+ // i32::MIN also satisfies this predicate because subtracting 5 wraps.
+ // A mathematical lower bound of 5 would exclude a valid input value.
+ let batch = RecordBatch::try_new(
+ input.schema(),
+ vec![Arc::new(arrow::array::Int32Array::from(vec![i32::MIN]))],
+ )
+ .unwrap();
+ let result = predicate.evaluate(&batch).unwrap().into_array(1).unwrap();
+ assert_eq!(
+ result.as_ref(),
+ &arrow::array::BooleanArray::from(vec![true])
+ );
let filter: Arc<dyn ExecutionPlan> =
Arc::new(FilterExec::try_new(predicate, input)?);
let filter_statistics =
@@ -2356,7 +2368,7 @@ mod tests {
// `a <= 10` rejects nulls, so `a` has no surviving nulls even
// though the input statistics are entirely unknown.
null_count: Precision::Exact(0),
- min_value: Precision::Inexact(ScalarValue::Int32(Some(5))),
+ min_value: Precision::Absent,
max_value: Precision::Inexact(ScalarValue::Int32(Some(10))),
sum_value: Precision::Absent,
distinct_count: Precision::Absent,
diff --git a/datafusion/sqllogictest/test_files/filter_without_sort_exec.slt b/datafusion/sqllogictest/test_files/filter_without_sort_exec.slt
index 87e994f243..df23390c64 100644
--- a/datafusion/sqllogictest/test_files/filter_without_sort_exec.slt
+++ b/datafusion/sqllogictest/test_files/filter_without_sort_exec.slt
@@ -189,3 +189,99 @@ physical_plan
02)--FilterExec: b@0 > 1
03)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true
04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/filter_without_sort_exec/cast_ordering.parquet]]}, projection=[b], output_ordering=[b@0 ASC NULLS LAST], file_type=parquet, predicate=b@0 > 1, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 1, required_guarantees=[]
+
+statement ok
+SET datafusion.execution.target_partitions = 1;
+
+# Wrapping multiplication has more than one preimage of 2. The filter
+# must not mark a as constant and remove either requested sort.
+statement ok
+CREATE TABLE wrap_test (a INT) AS VALUES (-2147483647), (1);
+
+query II
+SELECT a, a * 2::INT FROM wrap_test ORDER BY a;
+----
+-2147483647 2
+1 2
+
+query I
+SELECT a FROM wrap_test WHERE a * 2::INT = 2::INT ORDER BY a DESC;
+----
+1
+-2147483647
+
+query I
+SELECT a - 2::INT AS x FROM wrap_test WHERE a * 2::INT = 2::INT ORDER BY x;
+----
+-1
+2147483647
+
+query II
+SELECT MIN(a), MAX(a) FROM wrap_test WHERE a * 2::INT = 2::INT;
+----
+-2147483647 1
+
+# Integer division truncates: both 2 and 3 satisfy a / 2 = 1.
+statement ok
+CREATE TABLE division_test (a INT) AS VALUES (2), (3);
+
+query II
+SELECT a, a / 2::INT FROM division_test ORDER BY a;
+----
+2 1
+3 1
+
+query I
+SELECT a FROM division_test WHERE a / 2::INT = 1::INT ORDER BY a DESC;
+----
+3
+2
+
+# Widening integer casts leave unbounded endpoints unbounded.
+statement ok
+SET datafusion.explain.physical_plan_only = true;
+
+statement ok
+CREATE TABLE widening_input (a INT) AS
+VALUES (2147483647), (1), (2147483646), (2147483647);
+
+# Without source-type bounds, interval inference conservatively retains the sort.
+query TT
+EXPLAIN SELECT a FROM widening_input
+WHERE CAST(a AS BIGINT) + 1 > 2147483647 ORDER BY a;
+----
+physical_plan
+01)SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false]
+02)--FilterExec: CAST(a@0 AS Int64) + 1 > 2147483647
+03)----DataSourceExec: partitions=1, partition_sizes=[1]
+
+query I
+SELECT a FROM widening_input
+WHERE CAST(a AS BIGINT) + 1 > 2147483647 ORDER BY a;
+----
+2147483647
+2147483647
+
+# Two distinct values can pass this predicate; the sort must remain.
+query TT
+EXPLAIN SELECT a FROM widening_input
+WHERE CAST(a AS BIGINT) + 1 > 2147483646 ORDER BY a;
+----
+physical_plan
+01)SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false]
+02)--FilterExec: CAST(a@0 AS Int64) + 1 > 2147483646
+03)----DataSourceExec: partitions=1, partition_sizes=[1]
+
+query I
+SELECT a FROM widening_input
+WHERE CAST(a AS BIGINT) + 1 > 2147483646 ORDER BY a;
+----
+2147483646
+2147483647
+2147483647
+
+statement ok
+SET datafusion.execution.target_partitions = 4;
+
+statement ok
+RESET datafusion.explain.physical_plan_only;