Apache DataFusion SQL Query Engine
fix(functions): prevent invalid sort omission in log monotonicity when base <= 1 (#25746)
Ignatius Pang46bff840b932
diff --git a/datafusion/functions/src/math/log.rs b/datafusion/functions/src/math/log.rsindex 5538181f1c..c9f5b0c40e 100644--- a/datafusion/functions/src/math/log.rs+++ b/datafusion/functions/src/math/log.rs@@ -27,6 +27,7 @@ use arrow::error::ArrowError;use arrow_buffer::i256;use datafusion_common::types::NativeType;use datafusion_common::{Result, ScalarValue, exec_err, plan_datafusion_err, plan_err};+use datafusion_expr::interval_arithmetic::Interval;use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};use datafusion_expr::sort_properties::{ExprProperties, SortProperties};use datafusion_expr::{@@ -101,6 +102,14 @@ fn is_valid_integer_base(base: f64) -> bool {base.trunc() == base && base >= 2.0 && base <= u32::MAX as f64}+/// Returns true if every value in `range` is provably greater than one.+fn is_gt_one(range: &Interval) -> bool {+ ScalarValue::new_one(&range.data_type())+ .and_then(|one| Interval::try_new(one.clone(), one))+ .and_then(|one| range.gt(one))+ .is_ok_and(|gt| gt == Interval::TRUE)+}+/// Calculate logarithm for Decimal32 values./// For integer bases >= 2 with zero scale, return an exact integer log when the/// value is a perfect power of the base. Otherwise falls back to f64 computation.@@ -209,10 +218,20 @@ impl ScalarUDFImpl for LogFunc {} else {(input[0].sort_properties, input[1].sort_properties)};+ // log_b(x) = ln(x) / ln(b) is increasing in x only when b > 1; for+ // 0 < b < 1 it is decreasing. If the base cannot be proven to be > 1+ // (e.g. `log(0.5, x)`, or a column with unknown bounds), make no claim.+ // `log(x)` always uses base 10.+ if input.len() == 2 && !is_gt_one(&input[0].range) {+ return Ok(SortProperties::Unordered);+ }+match (num_sort_properties, base_sort_properties) {+ // With b > 1, log_b(x) is decreasing in b only when x > 1(first @ SortProperties::Ordered(num), SortProperties::Ordered(base))if num.descending != base.descending- && num.nulls_first == base.nulls_first =>+ && num.nulls_first == base.nulls_first+ && is_gt_one(&input[1].range) =>{Ok(first)}@@ -220,7 +239,9 @@ impl ScalarUDFImpl for LogFunc {first @ (SortProperties::Ordered(_) | SortProperties::Singleton),SortProperties::Singleton,) => Ok(first),- (SortProperties::Singleton, second @ SortProperties::Ordered(_)) => {+ (SortProperties::Singleton, second @ SortProperties::Ordered(_))+ if is_gt_one(&input[1].range) =>+ {Ok(-second)}_ => Ok(SortProperties::Unordered),@@ -725,22 +746,28 @@ mod tests {#[test]fn test_log_output_ordering() {+ // Range proving base > 1.0 and num > 1.0 (e.g. [2.0, 10.0])+ let gt_one_range =+ Interval::try_new(ScalarValue::from(2.0), ScalarValue::from(10.0)).unwrap();+// [Unordered, Ascending, Descending, Literal]let orders = [- ExprProperties::new_unknown(),- ExprProperties::new_unknown().with_order(SortProperties::Ordered(- SortOptions {+ ExprProperties::new_unknown().with_range(gt_one_range.clone()),+ ExprProperties::new_unknown()+ .with_range(gt_one_range.clone())+ .with_order(SortProperties::Ordered(SortOptions {descending: false,nulls_first: true,- },- )),- ExprProperties::new_unknown().with_order(SortProperties::Ordered(- SortOptions {+ })),+ ExprProperties::new_unknown()+ .with_range(gt_one_range.clone())+ .with_order(SortProperties::Ordered(SortOptions {descending: true,nulls_first: true,- },- )),- ExprProperties::new_unknown().with_order(SortProperties::Singleton),+ })),+ ExprProperties::new_unknown()+ .with_range(gt_one_range.clone())+ .with_order(SortProperties::Singleton),];let log = LogFunc::new();@@ -813,22 +840,50 @@ mod tests {assert_eq!(results, expected);// Test with different `nulls_first`- let base_order = ExprProperties::new_unknown().with_order(- SortProperties::Ordered(SortOptions {+ let base_order = ExprProperties::new_unknown()+ .with_range(gt_one_range.clone())+ .with_order(SortProperties::Ordered(SortOptions {descending: true,nulls_first: true,- }),- );- let num_order = ExprProperties::new_unknown().with_order(- SortProperties::Ordered(SortOptions {+ }));+ let num_order = ExprProperties::new_unknown()+ .with_range(gt_one_range.clone())+ .with_order(SortProperties::Ordered(SortOptions {descending: false,nulls_first: false,- }),- );+ }));assert_eq!(log.output_ordering(&[base_order, num_order]).unwrap(),SortProperties::Unordered);++ // Test base in (0, 1), e.g. base = 0.5:+ // When base < 1.0, ln(base) < 0 which inverts monotonicity.+ // It must NOT claim ascending order (must return Unordered to prevent invalid sort omission).+ let sub_one_range =+ Interval::try_new(ScalarValue::from(0.2), ScalarValue::from(0.8)).unwrap();+ let sub_one_base = ExprProperties::new_unknown()+ .with_range(sub_one_range)+ .with_order(SortProperties::Singleton);+ let asc_num = ExprProperties::new_unknown()+ .with_range(gt_one_range.clone())+ .with_order(SortProperties::Ordered(SortOptions {+ descending: false,+ nulls_first: true,+ }));+ assert_eq!(+ log.output_ordering(&[sub_one_base, asc_num.clone()])+ .unwrap(),+ SortProperties::Unordered+ );++ // Test unknown / unbounded base: must return Unordered+ let unknown_base =+ ExprProperties::new_unknown().with_order(SortProperties::Singleton);+ assert_eq!(+ log.output_ordering(&[unknown_base, asc_num]).unwrap(),+ SortProperties::Unordered+ );}#[test]diff --git a/datafusion/sqllogictest/test_files/order.slt b/datafusion/sqllogictest/test_files/order.sltindex 9e6bb5d214..c8c1cd2210 100644--- a/datafusion/sqllogictest/test_files/order.slt+++ b/datafusion/sqllogictest/test_files/order.slt@@ -948,6 +948,8 @@ physical_plan03)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, projection=[c11], output_ordering=[c11@0 ASC NULLS LAST], file_type=csv, has_header=true+# The bounds of c11 and c12 are unknown at plan time (the data is in (0, 1),+# where log(c12, c11) is not increasing in c11), so the sort must be keptquery TTEXPLAIN SELECT LOG(c12, c11) as log_c11_base_c12FROM aggregate_test_100@@ -959,9 +961,10 @@ logical_plan03)----TableScan: aggregate_test_100 projection=[c11, c12]physical_plan01)SortPreservingMergeExec: [log_c11_base_c12@0 ASC NULLS LAST]-02)--ProjectionExec: expr=[log(c12@1, c11@0) as log_c11_base_c12]-03)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true-04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, projection=[c11, c12], output_orderings=[[c11@0 ASC NULLS LAST], [c12@1 DESC NULLS LAST]], file_type=csv, has_header=true+02)--SortExec: expr=[log_c11_base_c12@0 ASC NULLS LAST], preserve_partitioning=[true]+03)----ProjectionExec: expr=[log(c12@1, c11@0) as log_c11_base_c12]+04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true+05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, projection=[c11, c12], output_orderings=[[c11@0 ASC NULLS LAST], [c12@1 DESC NULLS LAST]], file_type=csv, has_header=truequery TTEXPLAIN SELECT LOG(c11, c12) as log_c12_base_c11@@ -974,9 +977,61 @@ logical_plan03)----TableScan: aggregate_test_100 projection=[c11, c12]physical_plan01)SortPreservingMergeExec: [log_c12_base_c11@0 DESC NULLS LAST]-02)--ProjectionExec: expr=[log(c11@0, c12@1) as log_c12_base_c11]-03)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true-04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, projection=[c11, c12], output_orderings=[[c11@0 ASC NULLS LAST], [c12@1 DESC NULLS LAST]], file_type=csv, has_header=true+02)--SortExec: expr=[log_c12_base_c11@0 DESC NULLS LAST], preserve_partitioning=[true]+03)----ProjectionExec: expr=[log(c11@0, c12@1) as log_c12_base_c11]+04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true+05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, projection=[c11, c12], output_orderings=[[c11@0 ASC NULLS LAST], [c12@1 DESC NULLS LAST]], file_type=csv, has_header=true++# log with a base in (0, 1) is decreasing, so the outer sort on `l` must be+# kept even though its input is already sorted by x (the inner TopK)+query TT+EXPLAIN SELECT log(0.5, x) AS l FROM (SELECT x FROM (VALUES (1.0), (2.0), (4.0), (8.0)) AS t(x) ORDER BY x LIMIT 10) ORDER BY l ASC;+----+logical_plan+01)Sort: l ASC NULLS LAST+02)--Projection: log(Float64(0.5), t.x) AS l+03)----Sort: t.x ASC NULLS LAST, fetch=10+04)------SubqueryAlias: t+05)--------Projection: column1 AS x+06)----------Values: (Float64(1)), (Float64(2)), (Float64(4)), (Float64(8))+physical_plan+01)SortExec: expr=[l@0 ASC NULLS LAST], preserve_partitioning=[false]+02)--ProjectionExec: expr=[log(0.5, column1@0) as l]+03)----SortExec: TopK(fetch=10), expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[false]+04)------DataSourceExec: partitions=1, partition_sizes=[1]++query R+SELECT log(0.5, x) AS l FROM (SELECT x FROM (VALUES (1.0), (2.0), (4.0), (8.0)) AS t(x) ORDER BY x LIMIT 10) ORDER BY l ASC;+----+-3+-2+-1+0++# log with a base > 1 is increasing, so the outer sort on `l` can be removed:+# its input is already sorted by x (the inner TopK), so `l` is sorted too+query TT+EXPLAIN SELECT log(2.0, x) AS l FROM (SELECT x FROM (VALUES (1.0), (2.0), (4.0), (8.0)) AS t(x) ORDER BY x LIMIT 10) ORDER BY l ASC;+----+logical_plan+01)Sort: l ASC NULLS LAST+02)--Projection: log(Float64(2), t.x) AS l+03)----Sort: t.x ASC NULLS LAST, fetch=10+04)------SubqueryAlias: t+05)--------Projection: column1 AS x+06)----------Values: (Float64(1)), (Float64(2)), (Float64(4)), (Float64(8))+physical_plan+01)ProjectionExec: expr=[log(2, column1@0) as l]+02)--SortExec: TopK(fetch=10), expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[false]+03)----DataSourceExec: partitions=1, partition_sizes=[1]++query R+SELECT log(2.0, x) AS l FROM (SELECT x FROM (VALUES (1.0), (2.0), (4.0), (8.0)) AS t(x) ORDER BY x LIMIT 10) ORDER BY l ASC;+----+0+1+2+3statement okdrop table aggregate_test_100;