Apache DataFusion SQL Query Engine
fix: do not fail when an expression's ordering cannot be derived (#26884)
Jay Zhan69a7fef952f7
diff --git a/datafusion/physical-expr/src/equivalence/properties/dependency.rs b/datafusion/physical-expr/src/equivalence/properties/dependency.rsindex ab796ace5d..5a7b757896 100644--- a/datafusion/physical-expr/src/equivalence/properties/dependency.rs+++ b/datafusion/physical-expr/src/equivalence/properties/dependency.rs@@ -592,6 +592,54 @@ mod tests {Ok(())}+ /// `sqrt(a)` has no ordering that can be derived from `a`, which may be+ /// negative, so its ordering function fails. A plan sorted on `sqrt(a)`+ /// still satisfies a requirement on `sqrt(a)` with the same sort options,+ /// and nothing else makes `sqrt(a)` ordered.+ #[test]+ fn test_known_ordering_when_expr_properties_fail() -> Result<()> {+ let schema =+ Arc::new(Schema::new(vec![Field::new("a", DataType::Float64, true)]));+ let a = col("a", &schema)?;+ let sqrt_a: Arc<dyn PhysicalExpr> = Arc::new(ScalarFunctionExpr::try_new(+ datafusion_functions::math::sqrt(),+ vec![Arc::clone(&a)],+ &schema,+ Arc::new(ConfigOptions::default()),+ )?);++ // `sqrt(a)` is nullable, so both the direction and the NULLS position+ // must match:+ let all_options = [(false, false), (false, true), (true, false), (true, true)]+ .map(|(descending, nulls_first)| SortOptions {+ descending,+ nulls_first,+ });+ for known in all_options {+ let mut eq_properties = EquivalenceProperties::new(Arc::clone(&schema));+ eq_properties+ .add_ordering([PhysicalSortExpr::new(Arc::clone(&sqrt_a), known)]);+ for required in all_options {+ let required_ordering =+ [PhysicalSortExpr::new(Arc::clone(&sqrt_a), required)];+ assert_eq!(+ eq_properties.ordering_satisfy(required_ordering)?,+ known == required,+ "known {known:?}, required {required:?}"+ );+ }+ }++ // Without a known ordering on `sqrt(a)`, it is not ordered, even when+ // `a` is:+ let required_ordering = [PhysicalSortExpr::new_default(Arc::clone(&sqrt_a))];+ let mut eq_properties = EquivalenceProperties::new(Arc::clone(&schema));+ assert!(!eq_properties.ordering_satisfy(required_ordering.clone())?);+ eq_properties.add_ordering([PhysicalSortExpr::new_default(a)]);+ assert!(!eq_properties.ordering_satisfy(required_ordering)?);+ Ok(())+ }+#[test]fn test_update_properties() -> Result<()> {let schema = Schema::new(vec![diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rsindex 4d3dc89f7a..4d21480a81 100644--- a/datafusion/physical-expr/src/equivalence/properties/mod.rs+++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs@@ -508,9 +508,13 @@ impl EquivalenceProperties {strictly_order_preserving: true,});}- // Check if the expression is monotonic in all arguments:- let expr_properties =- equivalent_expr.get_properties(&child_properties)?;+ // Check if the expression is monotonic in all arguments. If+ // that cannot be derived, no new ordering is found:+ let Ok(expr_properties) =+ equivalent_expr.get_properties(&child_properties)+ else {+ continue;+ };if expr_properties.preserves_lex_ordering&& expr_properties.sort_properties== SortProperties::Ordered(leading_ordering_options)@@ -1548,7 +1552,14 @@ fn update_properties(if !node.expr.children().is_empty() {// We have an intermediate (non-leaf) node, account for its children:let children_props = node.children.iter().map(|c| c.data.clone()).collect_vec();- node.data = node.expr.get_properties(&children_props)?;+ // Properties that cannot be derived from the children, for example of a+ // function whose input may be outside its domain, are unknown. The+ // ordering of the node itself can still be known, so do not fail here,+ // and keep the type of its range so that parents can derive from it.+ node.data = node.expr.get_properties(&children_props).or_else(|_| {+ Interval::make_unbounded(&node.expr.data_type(eq_properties.schema())?)+ .map(|range| ExprProperties::new_unknown().with_range(range))+ })?;} else if node.expr.is::<Literal>() {// We have a Literal, which is one of the two possible leaf node types:node.data = node.expr.get_properties(&[])?;diff --git a/datafusion/sqllogictest/test_files/equivalence_properties.slt b/datafusion/sqllogictest/test_files/equivalence_properties.sltindex d5806538fe..4b32ba2c05 100644--- a/datafusion/sqllogictest/test_files/equivalence_properties.slt+++ b/datafusion/sqllogictest/test_files/equivalence_properties.slt@@ -134,6 +134,76 @@ SELECT l.c, s.x FROM (SELECT c FROM t) l CROSS JOIN (SELECT x FROM small WHERE x1 12 1+###+### Functions whose ordering cannot be derived from their input+###++# sqrt has no ordering that can be derived from c, because c may be negative.+# The inner ORDER BY still makes the subquery sorted by sqrt(c), so the+# window's ORDER BY sqrt(c) needs no second sort.+query TT+EXPLAIN SELECT c, row_number() OVER (ORDER BY sqrt(c)) FROM (SELECT c FROM t ORDER BY sqrt(c) LIMIT 5);+----+physical_plan+01)BoundedWindowAggExec: wdw=[row_number() ORDER BY [sqrt(t.c) ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [sqrt(t.c) ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]+02)--SortExec: TopK(fetch=5), expr=[sqrt(CAST(c@0 AS Float64)) ASC NULLS LAST], preserve_partitioning=[false]+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[c], output_ordering=[c@0 ASC NULLS LAST], file_type=csv, has_header=true++query II+SELECT c, row_number() OVER (ORDER BY sqrt(c)) FROM (SELECT c FROM t ORDER BY sqrt(c) LIMIT 5);+----+0 1+1 2+2 3+3 4+4 5++# -sqrt(c) is ordered opposite to sqrt(c), whose order is known only from the+# inner ORDER BY. So the window's ORDER BY -sqrt(c) needs no second sort.+query TT+EXPLAIN SELECT c, row_number() OVER (ORDER BY -sqrt(c)) FROM (SELECT c FROM t ORDER BY sqrt(c) DESC NULLS LAST LIMIT 5);+----+physical_plan+01)BoundedWindowAggExec: wdw=[row_number() ORDER BY [(- sqrt(t.c)) ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [(- sqrt(t.c)) ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]+02)--SortExec: TopK(fetch=5), expr=[sqrt(CAST(c@0 AS Float64)) DESC NULLS LAST], preserve_partitioning=[false]+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[c], output_ordering=[c@0 ASC NULLS LAST], file_type=csv, has_header=true++query II+SELECT c, row_number() OVER (ORDER BY -sqrt(c)) FROM (SELECT c FROM t ORDER BY sqrt(c) DESC NULLS LAST LIMIT 5);+----+99 1+98 2+97 3+96 4+95 5++# On a table sorted by (a, b), the filter a = sqrt(b) would make it sorted by+# b alone too, if sqrt(b) were known to be increasing in b. That cannot be+# derived, because b may be negative, so no new ordering is found and the+# query runs.+statement ok+CREATE EXTERNAL TABLE t_double (a0 DOUBLE, a DOUBLE, b DOUBLE, c DOUBLE, d DOUBLE)+STORED AS CSV+WITH ORDER (a ASC, b ASC)+LOCATION '../core/tests/data/window_2.csv'+OPTIONS ('format.has_header' 'true');++query TT+EXPLAIN SELECT a, b FROM t_double WHERE a = sqrt(b);+----+physical_plan+01)FilterExec: a@0 = sqrt(b@1)+02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[a, b], output_ordering=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST], file_type=csv, has_header=true++query I+SELECT count(*) FROM t_double WHERE a = sqrt(b);+----+25++statement ok+DROP TABLE t_double;+###### Order-preserving repartition of an unbounded input###