Apache DataFusion SQL Query Engine
fix: preserve sorts for wrapping integer negation (#25568)
Huaijin11c8c3d90b05
diff --git a/datafusion/physical-expr/src/expressions/negative.rs b/datafusion/physical-expr/src/expressions/negative.rsindex a932c8d522..f625deab80 100644--- a/datafusion/physical-expr/src/expressions/negative.rs+++ b/datafusion/physical-expr/src/expressions/negative.rs@@ -28,9 +28,9 @@ use arrow::{datatypes::{DataType, Schema},record_batch::RecordBatch,};-use datafusion_common::{Result, internal_err, plan_err};+use datafusion_common::{Result, ScalarValue, internal_err, plan_err};use datafusion_expr::interval_arithmetic::Interval;-use datafusion_expr::sort_properties::ExprProperties;+use datafusion_expr::sort_properties::{ExprProperties, SortProperties};#[expect(deprecated)]use datafusion_expr::statistics::Distribution::{self, Bernoulli, Exponential, Gaussian, Generic, Uniform,@@ -115,10 +115,9 @@ impl PhysicalExpr for NegativeExpr {}/// Given the child interval of a NegativeExpr, it calculates the NegativeExpr's interval.- /// It replaces the upper and lower bounds after multiplying them with -1.- /// Ex: `(a, b]` => `[-b, -a)`+ /// Reflects the bounds unless signed integer negation can wrap.fn evaluate_bounds(&self, children: &[&Interval]) -> Result<Interval> {- children[0].arithmetic_negate()+ negate_bounds(children[0])}/// Returns a new [`Interval`] of a NegativeExpr that has the existing `interval` given that@@ -128,7 +127,7 @@ impl PhysicalExpr for NegativeExpr {interval: &Interval,children: &[&Interval],) -> Result<Option<Vec<Interval>>> {- let negated_interval = interval.arithmetic_negate()?;+ let negated_interval = negate_bounds(interval)?;Ok(children[0].intersect(negated_interval)?@@ -160,11 +159,17 @@ impl PhysicalExpr for NegativeExpr {}}- /// The ordering of a [`NegativeExpr`] is simply the reverse of its child.+ /// Negation reverses ordering only when the input cannot cross a wrap.fn get_properties(&self, children: &[ExprProperties]) -> Result<ExprProperties> {Ok(ExprProperties {- sort_properties: -children[0].sort_properties,- range: children[0].range.clone().arithmetic_negate()?,+ sort_properties: if children[0].sort_properties != SortProperties::Singleton+ && negation_may_wrap(&children[0].range)+ {+ SortProperties::Unordered+ } else {+ -children[0].sort_properties+ },+ range: negate_bounds(&children[0].range)?,preserves_lex_ordering: false,// Negation is one-to-one but reverses the ordering direction.strictly_order_preserving: false,@@ -196,6 +201,26 @@ impl PhysicalExpr for NegativeExpr {}}+fn negate_bounds(range: &Interval) -> Result<Interval> {+ if range.data_type() == DataType::Null {+ return Ok(range.clone());+ }+ if range.data_type().is_signed_integer() && negation_may_wrap(range) {+ // Do not derive bounds across a possible integer wrap.+ return Interval::make_unbounded(&range.data_type());+ }+ range.arithmetic_negate()+}++// Signed integer array negation wraps at the minimum value.+fn negation_may_wrap(range: &Interval) -> bool {+ let data_type = range.data_type();+ data_type == DataType::Null+ || (data_type.is_signed_integer()+ && (range.lower().is_null()+ || ScalarValue::min(&data_type).as_ref() == Some(range.lower())))+}+#[cfg(feature = "proto")]impl NegativeExpr {/// Reconstruct a [`NegativeExpr`] from its protobuf representation.@@ -290,6 +315,135 @@ mod tests {Ok(())}+ #[test]+ fn test_wrapping_negation_properties() {+ let expr = NegativeExpr::new(Arc::new(Column::new("a", 0)));+ let ordered = SortProperties::Ordered(Default::default());+ for data_type in [Int8, Int16, Int32, Int64] {+ let minimum = ScalarValue::min(&data_type).unwrap();+ let full = Interval::make_unbounded(&data_type).unwrap();+ let singleton = Interval::try_new(minimum.clone(), minimum.clone()).unwrap();+ for range in [+ full.clone(),+ singleton.clone(),+ Interval::try_new(ScalarValue::try_from(&data_type).unwrap(), minimum)+ .unwrap(),+ ] {+ let child = ExprProperties::new_unknown()+ .with_range(range.clone())+ .with_order(ordered);+ let result = expr.get_properties(&[child]).unwrap();+ assert_eq!(result.sort_properties, SortProperties::Unordered);+ assert_eq!(result.range, full);+ assert_eq!(expr.evaluate_bounds(&[&range]).unwrap(), full);+ assert_eq!(+ expr.propagate_constraints(&range, &[&full]).unwrap(),+ Some(vec![full.clone()])+ );+ }+ let child = ExprProperties::new_unknown()+ .with_range(singleton)+ .with_order(SortProperties::Singleton);+ assert_eq!(+ expr.get_properties(&[child]).unwrap().sort_properties,+ SortProperties::Singleton+ );+ }+ }++ #[test]+ fn test_non_wrapping_negation_properties() {+ let expr = NegativeExpr::new(Arc::new(Column::new("a", 0)));+ let ranges = [+ Interval::make(Some(i8::MIN + 1), Some(1_i8)).unwrap(),+ Interval::make(Some(i16::MIN + 1), Some(1_i16)).unwrap(),+ Interval::make(Some(i32::MIN + 1), Some(1_i32)).unwrap(),+ Interval::make(Some(i64::MIN + 1), Some(1_i64)).unwrap(),+ // An unbounded upper bound cannot introduce MIN when the lower bound excludes it.+ Interval::make(Some(i8::MIN + 1), None).unwrap(),+ Interval::make(Some(i16::MIN + 1), None).unwrap(),+ Interval::make(Some(i32::MIN + 1), None).unwrap(),+ Interval::make(Some(i64::MIN + 1), None).unwrap(),+ Interval::make(Some(-2_f32), Some(1_f32)).unwrap(),+ Interval::make(Some(-2_f64), Some(1_f64)).unwrap(),+ Interval::make_unbounded(&Float32).unwrap(),+ Interval::make_unbounded(&Float64).unwrap(),+ ];+ for range in ranges {+ let expected_range = range.arithmetic_negate().unwrap();+ for descending in [false, true] {+ for nulls_first in [false, true] {+ let ordered = SortProperties::Ordered(arrow::compute::SortOptions {+ descending,+ nulls_first,+ });+ let child = ExprProperties::new_unknown()+ .with_range(range.clone())+ .with_order(ordered);+ let result = expr.get_properties(&[child]).unwrap();+ assert_eq!(result.sort_properties, -ordered);+ assert_eq!(result.range, expected_range);+ }+ }+ assert_eq!(expr.evaluate_bounds(&[&range]).unwrap(), expected_range);+ assert_eq!(+ expr.propagate_constraints(&expected_range, &[&range])+ .unwrap(),+ Some(vec![range])+ );+ }+ }++ #[test]+ fn test_unknown_negation_properties() {+ let expr = NegativeExpr::new(Arc::new(Column::new("a", 0)));+ for order in [+ SortProperties::Ordered(Default::default()),+ SortProperties::Unordered,+ SortProperties::Singleton,+ ] {+ let child = ExprProperties::new_unknown().with_order(order);+ let range = child.range.clone();+ let result = expr.get_properties(&[child]).unwrap();+ assert_eq!(+ result.sort_properties,+ if order == SortProperties::Singleton {+ SortProperties::Singleton+ } else {+ SortProperties::Unordered+ }+ );+ assert_eq!(result.range, range);+ assert_eq!(expr.evaluate_bounds(&[&range]).unwrap(), range);+ }+ }++ #[test]+ fn test_negation_bounds_errors() {+ let expr = NegativeExpr::new(Arc::new(Column::new("a", 0)));+ // Boolean intervals cannot be negated. All callers must propagate the error.+ let range = Interval::make(Some(false), Some(true)).unwrap();+ let child = ExprProperties::new_unknown().with_range(range.clone());+ // Compare the error messages without call-site-dependent backtraces.+ let expected = range.arithmetic_negate().unwrap_err().strip_backtrace();+ assert_eq!(+ expr.evaluate_bounds(&[&range])+ .unwrap_err()+ .strip_backtrace(),+ expected+ );+ assert_eq!(+ expr.get_properties(&[child]).unwrap_err().strip_backtrace(),+ expected+ );+ assert_eq!(+ expr.propagate_constraints(&range, &[&range])+ .unwrap_err()+ .strip_backtrace(),+ expected+ );+ }+#[test]fn test_evaluate_bounds() -> Result<()> {let negative_expr = NegativeExpr::new(Arc::new(Column::new("a", 0)));diff --git a/datafusion/sqllogictest/test_files/order.slt b/datafusion/sqllogictest/test_files/order.sltindex d974eb9dd0..9e6bb5d214 100644--- a/datafusion/sqllogictest/test_files/order.slt+++ b/datafusion/sqllogictest/test_files/order.slt@@ -1966,11 +1966,13 @@ EXPLAIN SELECT * from ordered ORDER BY (a + b);----physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/data/composite_order.csv]]}, projection=[a, b], output_ordering=[a@0 + b@1 ASC NULLS LAST], file_type=csv, has_header=true-# Order equivalence handling should make this query a simple table scan+# Negation may wrap at BIGINT's minimum, so the declared ordering is insufficient.query TTEXPLAIN SELECT * from ordered ORDER BY -(a + b) desc nulls last;-----physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/data/composite_order.csv]]}, projection=[a, b], output_ordering=[a@0 + b@1 ASC NULLS LAST], file_type=csv, has_header=true+physical_plan+01)SortExec: expr=[(- a@0 + b@1) DESC NULLS LAST], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/data/composite_order.csv]]}, projection=[a, b], output_ordering=[a@0 + b@1 ASC NULLS LAST], file_type=csv, has_header=true# Ordering by another column requires a sortquery TT@@ -2159,3 +2161,63 @@ DROP TABLE sort_key_sink;statement okDROP TABLE t2;++# Regression for #24683: signed minimum negation wraps.+query II+SELECT i, -i AS n+FROM (+ SELECT i FROM (VALUES+ (CAST(-128 AS TINYINT)), (CAST(-1 AS TINYINT)),+ (CAST(0 AS TINYINT)), (CAST(1 AS TINYINT))+ ) AS v(i)+ ORDER BY i DESC LIMIT 4+) AS t+ORDER BY n ASC;+----+-128 -128+1 -1+0 0+-1 1++# Interval propagation must not overflow when the filter negates MIN.+statement ok+CREATE TABLE negative_values (i TINYINT) AS VALUES (-128), (-1), (0), (1);++query I+SELECT i FROM negative_values WHERE -i = CAST(-128 AS TINYINT);+----+-128++statement ok+DROP TABLE negative_values;++statement ok+CREATE UNBOUNDED EXTERNAL TABLE negative_stream (i INT)+STORED AS CSV WITH ORDER (i ASC)+LOCATION '../core/tests/data/empty.csv'+OPTIONS ('format.has_header' 'true');++# Double negation must not require sorting an unbounded stream.+query TT+EXPLAIN SELECT i FROM negative_stream ORDER BY -(-i) ASC;+----+logical_plan+01)Sort: negative_stream.i AS (- (- negative_stream.i)) ASC NULLS LAST+02)--TableScan: negative_stream projection=[i]+physical_plan StreamingTableExec: partition_sizes=1, projection=[i], infinite_source=true, output_ordering=[i@0 ASC NULLS LAST]++statement ok+DROP TABLE negative_stream;++# Equal non-null values do not remove the need to order NULLs.+query II+SELECT i, -i AS n FROM (+ SELECT i FROM (VALUES+ (CAST(-128 AS TINYINT)), (CAST(NULL AS TINYINT)), (CAST(-128 AS TINYINT))+ ) AS v(i)+ ORDER BY i DESC NULLS FIRST LIMIT 3+) AS t ORDER BY n ASC NULLS LAST;+----+-128 -128+-128 -128+NULL NULL