Apache DataFusion SQL Query Engine
feat: add `strictly_order_preserving` for more expr (#23922)
Raz Luvaton22cf99d82bd1
diff --git a/datafusion/functions-nested/src/make_array.rs b/datafusion/functions-nested/src/make_array.rsindex ba746cd9cf..a63217cd5c 100644--- a/datafusion/functions-nested/src/make_array.rs+++ b/datafusion/functions-nested/src/make_array.rs@@ -37,6 +37,7 @@ use datafusion_expr::{ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,Volatility,};+use datafusion_expr_common::sort_properties::ExprProperties;use datafusion_macros::user_doc;use itertools::Itertools as _;@@ -113,6 +114,11 @@ impl ScalarUDFImpl for MakeArray {&self.aliases}+ fn strictly_order_preserving(&self, _inputs: &[ExprProperties]) -> Result<bool> {+ // Not strictly order preserving since null input does not correspond to null output+ Ok(false)+ }+fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {if arg_types.is_empty() {Ok(vec![])diff --git a/datafusion/functions-nested/src/repeat.rs b/datafusion/functions-nested/src/repeat.rsindex b275ae661e..dc99d78036 100644--- a/datafusion/functions-nested/src/repeat.rs+++ b/datafusion/functions-nested/src/repeat.rs@@ -37,6 +37,7 @@ use datafusion_expr::{Volatility,};use datafusion_expr_common::signature::{Coercion, TypeSignatureClass};+use datafusion_expr_common::sort_properties::ExprProperties;use datafusion_macros::user_doc;use std::mem::size_of;use std::sync::Arc;@@ -137,6 +138,12 @@ impl ScalarUDFImpl for ArrayRepeat {make_scalar_function(array_repeat_inner)(&args.args)}+ fn strictly_order_preserving(&self, _inputs: &[ExprProperties]) -> Result<bool> {+ // this is not strictly order preserving+ // since array_repeat(NULL, 2) = [NULL, NULL] and not NULL+ Ok(false)+ }+fn aliases(&self) -> &[String] {&self.aliases}diff --git a/datafusion/functions/src/math/factorial.rs b/datafusion/functions/src/math/factorial.rsindex f4e9b60dd3..d2a41a2dfb 100644--- a/datafusion/functions/src/math/factorial.rs+++ b/datafusion/functions/src/math/factorial.rs@@ -28,6 +28,7 @@ use datafusion_expr::{ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,Volatility,};+use datafusion_expr_common::sort_properties::{ExprProperties, SortProperties};use datafusion_macros::user_doc;#[user_doc(@@ -116,6 +117,16 @@ impl ScalarUDFImpl for FactorialFunc {}}+ fn output_ordering(&self, inputs: &[ExprProperties]) -> Result<SortProperties> {+ // Keep the same ordering as the input+ Ok(inputs[0].sort_properties)+ }++ fn strictly_order_preserving(&self, _inputs: &[ExprProperties]) -> Result<bool> {+ // factorial(0) == factorial(1) == 1 so ordering is not preserved+ Ok(false)+ }+fn documentation(&self) -> Option<&Documentation> {self.doc()}diff --git a/datafusion/functions/src/math/floor.rs b/datafusion/functions/src/math/floor.rsindex 24a93fa7e5..9c01238d4a 100644--- a/datafusion/functions/src/math/floor.rs+++ b/datafusion/functions/src/math/floor.rs@@ -261,6 +261,11 @@ impl ScalarUDFImpl for FloorFunc {Ok(input[0].sort_properties)}+ fn strictly_order_preserving(&self, _inputs: &[ExprProperties]) -> Result<bool> {+ // floor allows distinct inputs to collapse into equal outputs+ Ok(false)+ }+fn evaluate_bounds(&self, inputs: &[&Interval]) -> Result<Interval> {let data_type = decimal_floor_ceil_return_type(&inputs[0].data_type());Interval::make_unbounded(&data_type)diff --git a/datafusion/functions/src/string/repeat.rs b/datafusion/functions/src/string/repeat.rsindex a53f1e2e4f..cf2b8fd25c 100644--- a/datafusion/functions/src/string/repeat.rs+++ b/datafusion/functions/src/string/repeat.rs@@ -32,6 +32,7 @@ use datafusion_common::{use datafusion_expr::{ColumnarValue, Documentation, Volatility};use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl, Signature};use datafusion_expr_common::signature::{Coercion, TypeSignatureClass};+use datafusion_expr_common::sort_properties::ExprProperties;use datafusion_macros::user_doc;#[user_doc(@@ -160,6 +161,11 @@ impl ScalarUDFImpl for RepeatFunc {fn documentation(&self) -> Option<&Documentation> {self.doc()}++ fn strictly_order_preserving(&self, _inputs: &[ExprProperties]) -> Result<bool> {+ // repeat('bc', 2) = 'bcbc' > repeat('bca', 2) = 'bcabca', so ordering is not preserved+ Ok(false)+ }}/// Computes repeat for a single string value with max size checkdiff --git a/datafusion/sqllogictest/test_files/order.slt b/datafusion/sqllogictest/test_files/order.sltindex 1a02f5f59a..d080139bc7 100644--- a/datafusion/sqllogictest/test_files/order.slt+++ b/datafusion/sqllogictest/test_files/order.slt@@ -799,6 +799,36 @@ logical_plan03)----TableScan: multiple_ordered_table projection=[a]physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[floor(CAST(a@1 AS Float64)) as a_], file_type=csv, has_header=true+# A merely monotone (`preserves_lex_ordering`, but not strictly order-preserving) function such as factorial()+# does NOT justify the suffix keys:+# factorial(0) == factorial(1) which means that it collapses distinct inputs into one output value, and `b`+# is not sorted within such a run, so a SortExec must remain.+query TT+EXPLAIN SELECT factorial(a) AS a_, b+FROM multiple_ordered_table+ORDER BY a_, b;+----+logical_plan+01)Sort: a_ ASC NULLS LAST, multiple_ordered_table.b ASC NULLS LAST+02)--Projection: factorial(CAST(multiple_ordered_table.a AS Int64)) AS a_, multiple_ordered_table.b+03)----TableScan: multiple_ordered_table projection=[a, b]+physical_plan+01)SortExec: expr=[a_@0 ASC NULLS LAST, b@1 ASC NULLS LAST], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[factorial(CAST(a@1 AS Int64)) as a_, b], file_type=csv, has_header=true++# Monotonicity alone is still enough when the expression is the *only* sort+# key, so here the SortExec is removed even though factorial() is not strict:+query TT+EXPLAIN SELECT factorial(a) AS a_+FROM multiple_ordered_table+ORDER BY a_;+----+logical_plan+01)Sort: a_ ASC NULLS LAST+02)--Projection: factorial(CAST(multiple_ordered_table.a AS Int64)) AS a_+03)----TableScan: multiple_ordered_table projection=[a]+physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[factorial(CAST(a@1 AS Int64)) as a_], file_type=csv, has_header=true+# The SLT runner sets `target_partitions` to 4 instead of using the default, so# reset it explicitly.statement ok