Skip to content

Apache DataFusion SQL Query Engine

fix(functions): prevent invalid sort omission in log monotonicity when base <= 1 (#25746)

Ignatius Pang46bff840b932
diff --git a/datafusion/functions/src/math/log.rs b/datafusion/functions/src/math/log.rs
index 5538181f1c..c9f5b0c40e 100644
--- a/datafusion/functions/src/math/log.rs
+++ b/datafusion/functions/src/math/log.rs
@@ -27,6 +27,7 @@ use arrow::error::ArrowError;
use arrow_buffer::i256;
use datafusion_common::types::NativeType;
use datafusion_common::{Result, ScalarValue, exec_err, plan_datafusion_err, plan_err};
+use datafusion_expr::interval_arithmetic::Interval;
use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};
use datafusion_expr::sort_properties::{ExprProperties, SortProperties};
use datafusion_expr::{
@@ -101,6 +102,14 @@ fn is_valid_integer_base(base: f64) -> bool {
base.trunc() == base && base >= 2.0 && base <= u32::MAX as f64
}
+/// Returns true if every value in `range` is provably greater than one.
+fn is_gt_one(range: &Interval) -> bool {
+ ScalarValue::new_one(&range.data_type())
+ .and_then(|one| Interval::try_new(one.clone(), one))
+ .and_then(|one| range.gt(one))
+ .is_ok_and(|gt| gt == Interval::TRUE)
+}
+
/// Calculate logarithm for Decimal32 values.
/// For integer bases >= 2 with zero scale, return an exact integer log when the
/// value is a perfect power of the base. Otherwise falls back to f64 computation.
@@ -209,10 +218,20 @@ impl ScalarUDFImpl for LogFunc {
} else {
(input[0].sort_properties, input[1].sort_properties)
};
+ // log_b(x) = ln(x) / ln(b) is increasing in x only when b > 1; for
+ // 0 < b < 1 it is decreasing. If the base cannot be proven to be > 1
+ // (e.g. `log(0.5, x)`, or a column with unknown bounds), make no claim.
+ // `log(x)` always uses base 10.
+ if input.len() == 2 && !is_gt_one(&input[0].range) {
+ return Ok(SortProperties::Unordered);
+ }
+
match (num_sort_properties, base_sort_properties) {
+ // With b > 1, log_b(x) is decreasing in b only when x > 1
(first @ SortProperties::Ordered(num), SortProperties::Ordered(base))
if num.descending != base.descending
- && num.nulls_first == base.nulls_first =>
+ && num.nulls_first == base.nulls_first
+ && is_gt_one(&input[1].range) =>
{
Ok(first)
}
@@ -220,7 +239,9 @@ impl ScalarUDFImpl for LogFunc {
first @ (SortProperties::Ordered(_) | SortProperties::Singleton),
SortProperties::Singleton,
) => Ok(first),
- (SortProperties::Singleton, second @ SortProperties::Ordered(_)) => {
+ (SortProperties::Singleton, second @ SortProperties::Ordered(_))
+ if is_gt_one(&input[1].range) =>
+ {
Ok(-second)
}
_ => Ok(SortProperties::Unordered),
@@ -725,22 +746,28 @@ mod tests {
#[test]
fn test_log_output_ordering() {
+ // Range proving base > 1.0 and num > 1.0 (e.g. [2.0, 10.0])
+ let gt_one_range =
+ Interval::try_new(ScalarValue::from(2.0), ScalarValue::from(10.0)).unwrap();
+
// [Unordered, Ascending, Descending, Literal]
let orders = [
- ExprProperties::new_unknown(),
- ExprProperties::new_unknown().with_order(SortProperties::Ordered(
- SortOptions {
+ ExprProperties::new_unknown().with_range(gt_one_range.clone()),
+ ExprProperties::new_unknown()
+ .with_range(gt_one_range.clone())
+ .with_order(SortProperties::Ordered(SortOptions {
descending: false,
nulls_first: true,
- },
- )),
- ExprProperties::new_unknown().with_order(SortProperties::Ordered(
- SortOptions {
+ })),
+ ExprProperties::new_unknown()
+ .with_range(gt_one_range.clone())
+ .with_order(SortProperties::Ordered(SortOptions {
descending: true,
nulls_first: true,
- },
- )),
- ExprProperties::new_unknown().with_order(SortProperties::Singleton),
+ })),
+ ExprProperties::new_unknown()
+ .with_range(gt_one_range.clone())
+ .with_order(SortProperties::Singleton),
];
let log = LogFunc::new();
@@ -813,22 +840,50 @@ mod tests {
assert_eq!(results, expected);
// Test with different `nulls_first`
- let base_order = ExprProperties::new_unknown().with_order(
- SortProperties::Ordered(SortOptions {
+ let base_order = ExprProperties::new_unknown()
+ .with_range(gt_one_range.clone())
+ .with_order(SortProperties::Ordered(SortOptions {
descending: true,
nulls_first: true,
- }),
- );
- let num_order = ExprProperties::new_unknown().with_order(
- SortProperties::Ordered(SortOptions {
+ }));
+ let num_order = ExprProperties::new_unknown()
+ .with_range(gt_one_range.clone())
+ .with_order(SortProperties::Ordered(SortOptions {
descending: false,
nulls_first: false,
- }),
- );
+ }));
assert_eq!(
log.output_ordering(&[base_order, num_order]).unwrap(),
SortProperties::Unordered
);
+
+ // Test base in (0, 1), e.g. base = 0.5:
+ // When base < 1.0, ln(base) < 0 which inverts monotonicity.
+ // It must NOT claim ascending order (must return Unordered to prevent invalid sort omission).
+ let sub_one_range =
+ Interval::try_new(ScalarValue::from(0.2), ScalarValue::from(0.8)).unwrap();
+ let sub_one_base = ExprProperties::new_unknown()
+ .with_range(sub_one_range)
+ .with_order(SortProperties::Singleton);
+ let asc_num = ExprProperties::new_unknown()
+ .with_range(gt_one_range.clone())
+ .with_order(SortProperties::Ordered(SortOptions {
+ descending: false,
+ nulls_first: true,
+ }));
+ assert_eq!(
+ log.output_ordering(&[sub_one_base, asc_num.clone()])
+ .unwrap(),
+ SortProperties::Unordered
+ );
+
+ // Test unknown / unbounded base: must return Unordered
+ let unknown_base =
+ ExprProperties::new_unknown().with_order(SortProperties::Singleton);
+ assert_eq!(
+ log.output_ordering(&[unknown_base, asc_num]).unwrap(),
+ SortProperties::Unordered
+ );
}
#[test]
diff --git a/datafusion/sqllogictest/test_files/order.slt b/datafusion/sqllogictest/test_files/order.slt
index 9e6bb5d214..c8c1cd2210 100644
--- a/datafusion/sqllogictest/test_files/order.slt
+++ b/datafusion/sqllogictest/test_files/order.slt
@@ -948,6 +948,8 @@ physical_plan
03)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true
04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, projection=[c11], output_ordering=[c11@0 ASC NULLS LAST], file_type=csv, has_header=true
+# The bounds of c11 and c12 are unknown at plan time (the data is in (0, 1),
+# where log(c12, c11) is not increasing in c11), so the sort must be kept
query TT
EXPLAIN SELECT LOG(c12, c11) as log_c11_base_c12
FROM aggregate_test_100
@@ -959,9 +961,10 @@ logical_plan
03)----TableScan: aggregate_test_100 projection=[c11, c12]
physical_plan
01)SortPreservingMergeExec: [log_c11_base_c12@0 ASC NULLS LAST]
-02)--ProjectionExec: expr=[log(c12@1, c11@0) as log_c11_base_c12]
-03)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true
-04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, projection=[c11, c12], output_orderings=[[c11@0 ASC NULLS LAST], [c12@1 DESC NULLS LAST]], file_type=csv, has_header=true
+02)--SortExec: expr=[log_c11_base_c12@0 ASC NULLS LAST], preserve_partitioning=[true]
+03)----ProjectionExec: expr=[log(c12@1, c11@0) as log_c11_base_c12]
+04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true
+05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, projection=[c11, c12], output_orderings=[[c11@0 ASC NULLS LAST], [c12@1 DESC NULLS LAST]], file_type=csv, has_header=true
query TT
EXPLAIN SELECT LOG(c11, c12) as log_c12_base_c11
@@ -974,9 +977,61 @@ logical_plan
03)----TableScan: aggregate_test_100 projection=[c11, c12]
physical_plan
01)SortPreservingMergeExec: [log_c12_base_c11@0 DESC NULLS LAST]
-02)--ProjectionExec: expr=[log(c11@0, c12@1) as log_c12_base_c11]
-03)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true
-04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, projection=[c11, c12], output_orderings=[[c11@0 ASC NULLS LAST], [c12@1 DESC NULLS LAST]], file_type=csv, has_header=true
+02)--SortExec: expr=[log_c12_base_c11@0 DESC NULLS LAST], preserve_partitioning=[true]
+03)----ProjectionExec: expr=[log(c11@0, c12@1) as log_c12_base_c11]
+04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true
+05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, projection=[c11, c12], output_orderings=[[c11@0 ASC NULLS LAST], [c12@1 DESC NULLS LAST]], file_type=csv, has_header=true
+
+# log with a base in (0, 1) is decreasing, so the outer sort on `l` must be
+# kept even though its input is already sorted by x (the inner TopK)
+query TT
+EXPLAIN SELECT log(0.5, x) AS l FROM (SELECT x FROM (VALUES (1.0), (2.0), (4.0), (8.0)) AS t(x) ORDER BY x LIMIT 10) ORDER BY l ASC;
+----
+logical_plan
+01)Sort: l ASC NULLS LAST
+02)--Projection: log(Float64(0.5), t.x) AS l
+03)----Sort: t.x ASC NULLS LAST, fetch=10
+04)------SubqueryAlias: t
+05)--------Projection: column1 AS x
+06)----------Values: (Float64(1)), (Float64(2)), (Float64(4)), (Float64(8))
+physical_plan
+01)SortExec: expr=[l@0 ASC NULLS LAST], preserve_partitioning=[false]
+02)--ProjectionExec: expr=[log(0.5, column1@0) as l]
+03)----SortExec: TopK(fetch=10), expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[false]
+04)------DataSourceExec: partitions=1, partition_sizes=[1]
+
+query R
+SELECT log(0.5, x) AS l FROM (SELECT x FROM (VALUES (1.0), (2.0), (4.0), (8.0)) AS t(x) ORDER BY x LIMIT 10) ORDER BY l ASC;
+----
+-3
+-2
+-1
+0
+
+# log with a base > 1 is increasing, so the outer sort on `l` can be removed:
+# its input is already sorted by x (the inner TopK), so `l` is sorted too
+query TT
+EXPLAIN SELECT log(2.0, x) AS l FROM (SELECT x FROM (VALUES (1.0), (2.0), (4.0), (8.0)) AS t(x) ORDER BY x LIMIT 10) ORDER BY l ASC;
+----
+logical_plan
+01)Sort: l ASC NULLS LAST
+02)--Projection: log(Float64(2), t.x) AS l
+03)----Sort: t.x ASC NULLS LAST, fetch=10
+04)------SubqueryAlias: t
+05)--------Projection: column1 AS x
+06)----------Values: (Float64(1)), (Float64(2)), (Float64(4)), (Float64(8))
+physical_plan
+01)ProjectionExec: expr=[log(2, column1@0) as l]
+02)--SortExec: TopK(fetch=10), expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[false]
+03)----DataSourceExec: partitions=1, partition_sizes=[1]
+
+query R
+SELECT log(2.0, x) AS l FROM (SELECT x FROM (VALUES (1.0), (2.0), (4.0), (8.0)) AS t(x) ORDER BY x LIMIT 10) ORDER BY l ASC;
+----
+0
+1
+2
+3
statement ok
drop table aggregate_test_100;