Skip to content

Apache DataFusion SQL Query Engine

fix: preserve sorts for wrapping integer negation (#25568)

Huaijin11c8c3d90b05
diff --git a/datafusion/physical-expr/src/expressions/negative.rs b/datafusion/physical-expr/src/expressions/negative.rs
index a932c8d522..f625deab80 100644
--- a/datafusion/physical-expr/src/expressions/negative.rs
+++ b/datafusion/physical-expr/src/expressions/negative.rs
@@ -28,9 +28,9 @@ use arrow::{
datatypes::{DataType, Schema},
record_batch::RecordBatch,
};
-use datafusion_common::{Result, internal_err, plan_err};
+use datafusion_common::{Result, ScalarValue, internal_err, plan_err};
use datafusion_expr::interval_arithmetic::Interval;
-use datafusion_expr::sort_properties::ExprProperties;
+use datafusion_expr::sort_properties::{ExprProperties, SortProperties};
#[expect(deprecated)]
use datafusion_expr::statistics::Distribution::{
self, Bernoulli, Exponential, Gaussian, Generic, Uniform,
@@ -115,10 +115,9 @@ impl PhysicalExpr for NegativeExpr {
}
/// Given the child interval of a NegativeExpr, it calculates the NegativeExpr's interval.
- /// It replaces the upper and lower bounds after multiplying them with -1.
- /// Ex: `(a, b]` => `[-b, -a)`
+ /// Reflects the bounds unless signed integer negation can wrap.
fn evaluate_bounds(&self, children: &[&Interval]) -> Result<Interval> {
- children[0].arithmetic_negate()
+ negate_bounds(children[0])
}
/// Returns a new [`Interval`] of a NegativeExpr that has the existing `interval` given that
@@ -128,7 +127,7 @@ impl PhysicalExpr for NegativeExpr {
interval: &Interval,
children: &[&Interval],
) -> Result<Option<Vec<Interval>>> {
- let negated_interval = interval.arithmetic_negate()?;
+ let negated_interval = negate_bounds(interval)?;
Ok(children[0]
.intersect(negated_interval)?
@@ -160,11 +159,17 @@ impl PhysicalExpr for NegativeExpr {
}
}
- /// The ordering of a [`NegativeExpr`] is simply the reverse of its child.
+ /// Negation reverses ordering only when the input cannot cross a wrap.
fn get_properties(&self, children: &[ExprProperties]) -> Result<ExprProperties> {
Ok(ExprProperties {
- sort_properties: -children[0].sort_properties,
- range: children[0].range.clone().arithmetic_negate()?,
+ sort_properties: if children[0].sort_properties != SortProperties::Singleton
+ && negation_may_wrap(&children[0].range)
+ {
+ SortProperties::Unordered
+ } else {
+ -children[0].sort_properties
+ },
+ range: negate_bounds(&children[0].range)?,
preserves_lex_ordering: false,
// Negation is one-to-one but reverses the ordering direction.
strictly_order_preserving: false,
@@ -196,6 +201,26 @@ impl PhysicalExpr for NegativeExpr {
}
}
+fn negate_bounds(range: &Interval) -> Result<Interval> {
+ if range.data_type() == DataType::Null {
+ return Ok(range.clone());
+ }
+ if range.data_type().is_signed_integer() && negation_may_wrap(range) {
+ // Do not derive bounds across a possible integer wrap.
+ return Interval::make_unbounded(&range.data_type());
+ }
+ range.arithmetic_negate()
+}
+
+// Signed integer array negation wraps at the minimum value.
+fn negation_may_wrap(range: &Interval) -> bool {
+ let data_type = range.data_type();
+ data_type == DataType::Null
+ || (data_type.is_signed_integer()
+ && (range.lower().is_null()
+ || ScalarValue::min(&data_type).as_ref() == Some(range.lower())))
+}
+
#[cfg(feature = "proto")]
impl NegativeExpr {
/// Reconstruct a [`NegativeExpr`] from its protobuf representation.
@@ -290,6 +315,135 @@ mod tests {
Ok(())
}
+ #[test]
+ fn test_wrapping_negation_properties() {
+ let expr = NegativeExpr::new(Arc::new(Column::new("a", 0)));
+ let ordered = SortProperties::Ordered(Default::default());
+ for data_type in [Int8, Int16, Int32, Int64] {
+ let minimum = ScalarValue::min(&data_type).unwrap();
+ let full = Interval::make_unbounded(&data_type).unwrap();
+ let singleton = Interval::try_new(minimum.clone(), minimum.clone()).unwrap();
+ for range in [
+ full.clone(),
+ singleton.clone(),
+ Interval::try_new(ScalarValue::try_from(&data_type).unwrap(), minimum)
+ .unwrap(),
+ ] {
+ let child = ExprProperties::new_unknown()
+ .with_range(range.clone())
+ .with_order(ordered);
+ let result = expr.get_properties(&[child]).unwrap();
+ assert_eq!(result.sort_properties, SortProperties::Unordered);
+ assert_eq!(result.range, full);
+ assert_eq!(expr.evaluate_bounds(&[&range]).unwrap(), full);
+ assert_eq!(
+ expr.propagate_constraints(&range, &[&full]).unwrap(),
+ Some(vec![full.clone()])
+ );
+ }
+ let child = ExprProperties::new_unknown()
+ .with_range(singleton)
+ .with_order(SortProperties::Singleton);
+ assert_eq!(
+ expr.get_properties(&[child]).unwrap().sort_properties,
+ SortProperties::Singleton
+ );
+ }
+ }
+
+ #[test]
+ fn test_non_wrapping_negation_properties() {
+ let expr = NegativeExpr::new(Arc::new(Column::new("a", 0)));
+ let ranges = [
+ Interval::make(Some(i8::MIN + 1), Some(1_i8)).unwrap(),
+ Interval::make(Some(i16::MIN + 1), Some(1_i16)).unwrap(),
+ Interval::make(Some(i32::MIN + 1), Some(1_i32)).unwrap(),
+ Interval::make(Some(i64::MIN + 1), Some(1_i64)).unwrap(),
+ // An unbounded upper bound cannot introduce MIN when the lower bound excludes it.
+ Interval::make(Some(i8::MIN + 1), None).unwrap(),
+ Interval::make(Some(i16::MIN + 1), None).unwrap(),
+ Interval::make(Some(i32::MIN + 1), None).unwrap(),
+ Interval::make(Some(i64::MIN + 1), None).unwrap(),
+ Interval::make(Some(-2_f32), Some(1_f32)).unwrap(),
+ Interval::make(Some(-2_f64), Some(1_f64)).unwrap(),
+ Interval::make_unbounded(&Float32).unwrap(),
+ Interval::make_unbounded(&Float64).unwrap(),
+ ];
+ for range in ranges {
+ let expected_range = range.arithmetic_negate().unwrap();
+ for descending in [false, true] {
+ for nulls_first in [false, true] {
+ let ordered = SortProperties::Ordered(arrow::compute::SortOptions {
+ descending,
+ nulls_first,
+ });
+ let child = ExprProperties::new_unknown()
+ .with_range(range.clone())
+ .with_order(ordered);
+ let result = expr.get_properties(&[child]).unwrap();
+ assert_eq!(result.sort_properties, -ordered);
+ assert_eq!(result.range, expected_range);
+ }
+ }
+ assert_eq!(expr.evaluate_bounds(&[&range]).unwrap(), expected_range);
+ assert_eq!(
+ expr.propagate_constraints(&expected_range, &[&range])
+ .unwrap(),
+ Some(vec![range])
+ );
+ }
+ }
+
+ #[test]
+ fn test_unknown_negation_properties() {
+ let expr = NegativeExpr::new(Arc::new(Column::new("a", 0)));
+ for order in [
+ SortProperties::Ordered(Default::default()),
+ SortProperties::Unordered,
+ SortProperties::Singleton,
+ ] {
+ let child = ExprProperties::new_unknown().with_order(order);
+ let range = child.range.clone();
+ let result = expr.get_properties(&[child]).unwrap();
+ assert_eq!(
+ result.sort_properties,
+ if order == SortProperties::Singleton {
+ SortProperties::Singleton
+ } else {
+ SortProperties::Unordered
+ }
+ );
+ assert_eq!(result.range, range);
+ assert_eq!(expr.evaluate_bounds(&[&range]).unwrap(), range);
+ }
+ }
+
+ #[test]
+ fn test_negation_bounds_errors() {
+ let expr = NegativeExpr::new(Arc::new(Column::new("a", 0)));
+ // Boolean intervals cannot be negated. All callers must propagate the error.
+ let range = Interval::make(Some(false), Some(true)).unwrap();
+ let child = ExprProperties::new_unknown().with_range(range.clone());
+ // Compare the error messages without call-site-dependent backtraces.
+ let expected = range.arithmetic_negate().unwrap_err().strip_backtrace();
+ assert_eq!(
+ expr.evaluate_bounds(&[&range])
+ .unwrap_err()
+ .strip_backtrace(),
+ expected
+ );
+ assert_eq!(
+ expr.get_properties(&[child]).unwrap_err().strip_backtrace(),
+ expected
+ );
+ assert_eq!(
+ expr.propagate_constraints(&range, &[&range])
+ .unwrap_err()
+ .strip_backtrace(),
+ expected
+ );
+ }
+
#[test]
fn test_evaluate_bounds() -> Result<()> {
let negative_expr = NegativeExpr::new(Arc::new(Column::new("a", 0)));
diff --git a/datafusion/sqllogictest/test_files/order.slt b/datafusion/sqllogictest/test_files/order.slt
index d974eb9dd0..9e6bb5d214 100644
--- a/datafusion/sqllogictest/test_files/order.slt
+++ b/datafusion/sqllogictest/test_files/order.slt
@@ -1966,11 +1966,13 @@ EXPLAIN SELECT * from ordered ORDER BY (a + b);
----
physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/data/composite_order.csv]]}, projection=[a, b], output_ordering=[a@0 + b@1 ASC NULLS LAST], file_type=csv, has_header=true
-# Order equivalence handling should make this query a simple table scan
+# Negation may wrap at BIGINT's minimum, so the declared ordering is insufficient.
query TT
EXPLAIN SELECT * from ordered ORDER BY -(a + b) desc nulls last;
----
-physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/data/composite_order.csv]]}, projection=[a, b], output_ordering=[a@0 + b@1 ASC NULLS LAST], file_type=csv, has_header=true
+physical_plan
+01)SortExec: expr=[(- a@0 + b@1) DESC NULLS LAST], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/data/composite_order.csv]]}, projection=[a, b], output_ordering=[a@0 + b@1 ASC NULLS LAST], file_type=csv, has_header=true
# Ordering by another column requires a sort
query TT
@@ -2159,3 +2161,63 @@ DROP TABLE sort_key_sink;
statement ok
DROP TABLE t2;
+
+# Regression for #24683: signed minimum negation wraps.
+query II
+SELECT i, -i AS n
+FROM (
+ SELECT i FROM (VALUES
+ (CAST(-128 AS TINYINT)), (CAST(-1 AS TINYINT)),
+ (CAST(0 AS TINYINT)), (CAST(1 AS TINYINT))
+ ) AS v(i)
+ ORDER BY i DESC LIMIT 4
+) AS t
+ORDER BY n ASC;
+----
+-128 -128
+1 -1
+0 0
+-1 1
+
+# Interval propagation must not overflow when the filter negates MIN.
+statement ok
+CREATE TABLE negative_values (i TINYINT) AS VALUES (-128), (-1), (0), (1);
+
+query I
+SELECT i FROM negative_values WHERE -i = CAST(-128 AS TINYINT);
+----
+-128
+
+statement ok
+DROP TABLE negative_values;
+
+statement ok
+CREATE UNBOUNDED EXTERNAL TABLE negative_stream (i INT)
+STORED AS CSV WITH ORDER (i ASC)
+LOCATION '../core/tests/data/empty.csv'
+OPTIONS ('format.has_header' 'true');
+
+# Double negation must not require sorting an unbounded stream.
+query TT
+EXPLAIN SELECT i FROM negative_stream ORDER BY -(-i) ASC;
+----
+logical_plan
+01)Sort: negative_stream.i AS (- (- negative_stream.i)) ASC NULLS LAST
+02)--TableScan: negative_stream projection=[i]
+physical_plan StreamingTableExec: partition_sizes=1, projection=[i], infinite_source=true, output_ordering=[i@0 ASC NULLS LAST]
+
+statement ok
+DROP TABLE negative_stream;
+
+# Equal non-null values do not remove the need to order NULLs.
+query II
+SELECT i, -i AS n FROM (
+ SELECT i FROM (VALUES
+ (CAST(-128 AS TINYINT)), (CAST(NULL AS TINYINT)), (CAST(-128 AS TINYINT))
+ ) AS v(i)
+ ORDER BY i DESC NULLS FIRST LIMIT 3
+) AS t ORDER BY n ASC NULLS LAST;
+----
+-128 -128
+-128 -128
+NULL NULL