Skip to content

Apache DataFusion SQL Query Engine

feat: add `strictly_order_preserving` for more expr (#23922)

Raz Luvaton22cf99d82bd1
diff --git a/datafusion/functions-nested/src/make_array.rs b/datafusion/functions-nested/src/make_array.rs
index ba746cd9cf..a63217cd5c 100644
--- a/datafusion/functions-nested/src/make_array.rs
+++ b/datafusion/functions-nested/src/make_array.rs
@@ -37,6 +37,7 @@ use datafusion_expr::{
ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
Volatility,
};
+use datafusion_expr_common::sort_properties::ExprProperties;
use datafusion_macros::user_doc;
use itertools::Itertools as _;
@@ -113,6 +114,11 @@ impl ScalarUDFImpl for MakeArray {
&self.aliases
}
+ fn strictly_order_preserving(&self, _inputs: &[ExprProperties]) -> Result<bool> {
+ // Not strictly order preserving since null input does not correspond to null output
+ Ok(false)
+ }
+
fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
if arg_types.is_empty() {
Ok(vec![])
diff --git a/datafusion/functions-nested/src/repeat.rs b/datafusion/functions-nested/src/repeat.rs
index b275ae661e..dc99d78036 100644
--- a/datafusion/functions-nested/src/repeat.rs
+++ b/datafusion/functions-nested/src/repeat.rs
@@ -37,6 +37,7 @@ use datafusion_expr::{
Volatility,
};
use datafusion_expr_common::signature::{Coercion, TypeSignatureClass};
+use datafusion_expr_common::sort_properties::ExprProperties;
use datafusion_macros::user_doc;
use std::mem::size_of;
use std::sync::Arc;
@@ -137,6 +138,12 @@ impl ScalarUDFImpl for ArrayRepeat {
make_scalar_function(array_repeat_inner)(&args.args)
}
+ fn strictly_order_preserving(&self, _inputs: &[ExprProperties]) -> Result<bool> {
+ // this is not strictly order preserving
+ // since array_repeat(NULL, 2) = [NULL, NULL] and not NULL
+ Ok(false)
+ }
+
fn aliases(&self) -> &[String] {
&self.aliases
}
diff --git a/datafusion/functions/src/math/factorial.rs b/datafusion/functions/src/math/factorial.rs
index f4e9b60dd3..d2a41a2dfb 100644
--- a/datafusion/functions/src/math/factorial.rs
+++ b/datafusion/functions/src/math/factorial.rs
@@ -28,6 +28,7 @@ use datafusion_expr::{
ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
Volatility,
};
+use datafusion_expr_common::sort_properties::{ExprProperties, SortProperties};
use datafusion_macros::user_doc;
#[user_doc(
@@ -116,6 +117,16 @@ impl ScalarUDFImpl for FactorialFunc {
}
}
+ fn output_ordering(&self, inputs: &[ExprProperties]) -> Result<SortProperties> {
+ // Keep the same ordering as the input
+ Ok(inputs[0].sort_properties)
+ }
+
+ fn strictly_order_preserving(&self, _inputs: &[ExprProperties]) -> Result<bool> {
+ // factorial(0) == factorial(1) == 1 so ordering is not preserved
+ Ok(false)
+ }
+
fn documentation(&self) -> Option<&Documentation> {
self.doc()
}
diff --git a/datafusion/functions/src/math/floor.rs b/datafusion/functions/src/math/floor.rs
index 24a93fa7e5..9c01238d4a 100644
--- a/datafusion/functions/src/math/floor.rs
+++ b/datafusion/functions/src/math/floor.rs
@@ -261,6 +261,11 @@ impl ScalarUDFImpl for FloorFunc {
Ok(input[0].sort_properties)
}
+ fn strictly_order_preserving(&self, _inputs: &[ExprProperties]) -> Result<bool> {
+ // floor allows distinct inputs to collapse into equal outputs
+ Ok(false)
+ }
+
fn evaluate_bounds(&self, inputs: &[&Interval]) -> Result<Interval> {
let data_type = decimal_floor_ceil_return_type(&inputs[0].data_type());
Interval::make_unbounded(&data_type)
diff --git a/datafusion/functions/src/string/repeat.rs b/datafusion/functions/src/string/repeat.rs
index a53f1e2e4f..cf2b8fd25c 100644
--- a/datafusion/functions/src/string/repeat.rs
+++ b/datafusion/functions/src/string/repeat.rs
@@ -32,6 +32,7 @@ use datafusion_common::{
use datafusion_expr::{ColumnarValue, Documentation, Volatility};
use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl, Signature};
use datafusion_expr_common::signature::{Coercion, TypeSignatureClass};
+use datafusion_expr_common::sort_properties::ExprProperties;
use datafusion_macros::user_doc;
#[user_doc(
@@ -160,6 +161,11 @@ impl ScalarUDFImpl for RepeatFunc {
fn documentation(&self) -> Option<&Documentation> {
self.doc()
}
+
+ fn strictly_order_preserving(&self, _inputs: &[ExprProperties]) -> Result<bool> {
+ // repeat('bc', 2) = 'bcbc' > repeat('bca', 2) = 'bcabca', so ordering is not preserved
+ Ok(false)
+ }
}
/// Computes repeat for a single string value with max size check
diff --git a/datafusion/sqllogictest/test_files/order.slt b/datafusion/sqllogictest/test_files/order.slt
index 1a02f5f59a..d080139bc7 100644
--- a/datafusion/sqllogictest/test_files/order.slt
+++ b/datafusion/sqllogictest/test_files/order.slt
@@ -799,6 +799,36 @@ logical_plan
03)----TableScan: multiple_ordered_table projection=[a]
physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[floor(CAST(a@1 AS Float64)) as a_], file_type=csv, has_header=true
+# A merely monotone (`preserves_lex_ordering`, but not strictly order-preserving) function such as factorial()
+# does NOT justify the suffix keys:
+# factorial(0) == factorial(1) which means that it collapses distinct inputs into one output value, and `b`
+# is not sorted within such a run, so a SortExec must remain.
+query TT
+EXPLAIN SELECT factorial(a) AS a_, b
+FROM multiple_ordered_table
+ORDER BY a_, b;
+----
+logical_plan
+01)Sort: a_ ASC NULLS LAST, multiple_ordered_table.b ASC NULLS LAST
+02)--Projection: factorial(CAST(multiple_ordered_table.a AS Int64)) AS a_, multiple_ordered_table.b
+03)----TableScan: multiple_ordered_table projection=[a, b]
+physical_plan
+01)SortExec: expr=[a_@0 ASC NULLS LAST, b@1 ASC NULLS LAST], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[factorial(CAST(a@1 AS Int64)) as a_, b], file_type=csv, has_header=true
+
+# Monotonicity alone is still enough when the expression is the *only* sort
+# key, so here the SortExec is removed even though factorial() is not strict:
+query TT
+EXPLAIN SELECT factorial(a) AS a_
+FROM multiple_ordered_table
+ORDER BY a_;
+----
+logical_plan
+01)Sort: a_ ASC NULLS LAST
+02)--Projection: factorial(CAST(multiple_ordered_table.a AS Int64)) AS a_
+03)----TableScan: multiple_ordered_table projection=[a]
+physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[factorial(CAST(a@1 AS Int64)) as a_], file_type=csv, has_header=true
+
# The SLT runner sets `target_partitions` to 4 instead of using the default, so
# reset it explicitly.
statement ok