Skip to content

Apache DataFusion SQL Query Engine

fix: do not fail when an expression's ordering cannot be derived (#26884)

Jay Zhan69a7fef952f7
diff --git a/datafusion/physical-expr/src/equivalence/properties/dependency.rs b/datafusion/physical-expr/src/equivalence/properties/dependency.rs
index ab796ace5d..5a7b757896 100644
--- a/datafusion/physical-expr/src/equivalence/properties/dependency.rs
+++ b/datafusion/physical-expr/src/equivalence/properties/dependency.rs
@@ -592,6 +592,54 @@ mod tests {
Ok(())
}
+ /// `sqrt(a)` has no ordering that can be derived from `a`, which may be
+ /// negative, so its ordering function fails. A plan sorted on `sqrt(a)`
+ /// still satisfies a requirement on `sqrt(a)` with the same sort options,
+ /// and nothing else makes `sqrt(a)` ordered.
+ #[test]
+ fn test_known_ordering_when_expr_properties_fail() -> Result<()> {
+ let schema =
+ Arc::new(Schema::new(vec![Field::new("a", DataType::Float64, true)]));
+ let a = col("a", &schema)?;
+ let sqrt_a: Arc<dyn PhysicalExpr> = Arc::new(ScalarFunctionExpr::try_new(
+ datafusion_functions::math::sqrt(),
+ vec![Arc::clone(&a)],
+ &schema,
+ Arc::new(ConfigOptions::default()),
+ )?);
+
+ // `sqrt(a)` is nullable, so both the direction and the NULLS position
+ // must match:
+ let all_options = [(false, false), (false, true), (true, false), (true, true)]
+ .map(|(descending, nulls_first)| SortOptions {
+ descending,
+ nulls_first,
+ });
+ for known in all_options {
+ let mut eq_properties = EquivalenceProperties::new(Arc::clone(&schema));
+ eq_properties
+ .add_ordering([PhysicalSortExpr::new(Arc::clone(&sqrt_a), known)]);
+ for required in all_options {
+ let required_ordering =
+ [PhysicalSortExpr::new(Arc::clone(&sqrt_a), required)];
+ assert_eq!(
+ eq_properties.ordering_satisfy(required_ordering)?,
+ known == required,
+ "known {known:?}, required {required:?}"
+ );
+ }
+ }
+
+ // Without a known ordering on `sqrt(a)`, it is not ordered, even when
+ // `a` is:
+ let required_ordering = [PhysicalSortExpr::new_default(Arc::clone(&sqrt_a))];
+ let mut eq_properties = EquivalenceProperties::new(Arc::clone(&schema));
+ assert!(!eq_properties.ordering_satisfy(required_ordering.clone())?);
+ eq_properties.add_ordering([PhysicalSortExpr::new_default(a)]);
+ assert!(!eq_properties.ordering_satisfy(required_ordering)?);
+ Ok(())
+ }
+
#[test]
fn test_update_properties() -> Result<()> {
let schema = Schema::new(vec![
diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rs
index 4d3dc89f7a..4d21480a81 100644
--- a/datafusion/physical-expr/src/equivalence/properties/mod.rs
+++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs
@@ -508,9 +508,13 @@ impl EquivalenceProperties {
strictly_order_preserving: true,
});
}
- // Check if the expression is monotonic in all arguments:
- let expr_properties =
- equivalent_expr.get_properties(&child_properties)?;
+ // Check if the expression is monotonic in all arguments. If
+ // that cannot be derived, no new ordering is found:
+ let Ok(expr_properties) =
+ equivalent_expr.get_properties(&child_properties)
+ else {
+ continue;
+ };
if expr_properties.preserves_lex_ordering
&& expr_properties.sort_properties
== SortProperties::Ordered(leading_ordering_options)
@@ -1548,7 +1552,14 @@ fn update_properties(
if !node.expr.children().is_empty() {
// We have an intermediate (non-leaf) node, account for its children:
let children_props = node.children.iter().map(|c| c.data.clone()).collect_vec();
- node.data = node.expr.get_properties(&children_props)?;
+ // Properties that cannot be derived from the children, for example of a
+ // function whose input may be outside its domain, are unknown. The
+ // ordering of the node itself can still be known, so do not fail here,
+ // and keep the type of its range so that parents can derive from it.
+ node.data = node.expr.get_properties(&children_props).or_else(|_| {
+ Interval::make_unbounded(&node.expr.data_type(eq_properties.schema())?)
+ .map(|range| ExprProperties::new_unknown().with_range(range))
+ })?;
} else if node.expr.is::<Literal>() {
// We have a Literal, which is one of the two possible leaf node types:
node.data = node.expr.get_properties(&[])?;
diff --git a/datafusion/sqllogictest/test_files/equivalence_properties.slt b/datafusion/sqllogictest/test_files/equivalence_properties.slt
index d5806538fe..4b32ba2c05 100644
--- a/datafusion/sqllogictest/test_files/equivalence_properties.slt
+++ b/datafusion/sqllogictest/test_files/equivalence_properties.slt
@@ -134,6 +134,76 @@ SELECT l.c, s.x FROM (SELECT c FROM t) l CROSS JOIN (SELECT x FROM small WHERE x
1 1
2 1
+###
+### Functions whose ordering cannot be derived from their input
+###
+
+# sqrt has no ordering that can be derived from c, because c may be negative.
+# The inner ORDER BY still makes the subquery sorted by sqrt(c), so the
+# window's ORDER BY sqrt(c) needs no second sort.
+query TT
+EXPLAIN SELECT c, row_number() OVER (ORDER BY sqrt(c)) FROM (SELECT c FROM t ORDER BY sqrt(c) LIMIT 5);
+----
+physical_plan
+01)BoundedWindowAggExec: wdw=[row_number() ORDER BY [sqrt(t.c) ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [sqrt(t.c) ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]
+02)--SortExec: TopK(fetch=5), expr=[sqrt(CAST(c@0 AS Float64)) ASC NULLS LAST], preserve_partitioning=[false]
+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[c], output_ordering=[c@0 ASC NULLS LAST], file_type=csv, has_header=true
+
+query II
+SELECT c, row_number() OVER (ORDER BY sqrt(c)) FROM (SELECT c FROM t ORDER BY sqrt(c) LIMIT 5);
+----
+0 1
+1 2
+2 3
+3 4
+4 5
+
+# -sqrt(c) is ordered opposite to sqrt(c), whose order is known only from the
+# inner ORDER BY. So the window's ORDER BY -sqrt(c) needs no second sort.
+query TT
+EXPLAIN SELECT c, row_number() OVER (ORDER BY -sqrt(c)) FROM (SELECT c FROM t ORDER BY sqrt(c) DESC NULLS LAST LIMIT 5);
+----
+physical_plan
+01)BoundedWindowAggExec: wdw=[row_number() ORDER BY [(- sqrt(t.c)) ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [(- sqrt(t.c)) ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]
+02)--SortExec: TopK(fetch=5), expr=[sqrt(CAST(c@0 AS Float64)) DESC NULLS LAST], preserve_partitioning=[false]
+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[c], output_ordering=[c@0 ASC NULLS LAST], file_type=csv, has_header=true
+
+query II
+SELECT c, row_number() OVER (ORDER BY -sqrt(c)) FROM (SELECT c FROM t ORDER BY sqrt(c) DESC NULLS LAST LIMIT 5);
+----
+99 1
+98 2
+97 3
+96 4
+95 5
+
+# On a table sorted by (a, b), the filter a = sqrt(b) would make it sorted by
+# b alone too, if sqrt(b) were known to be increasing in b. That cannot be
+# derived, because b may be negative, so no new ordering is found and the
+# query runs.
+statement ok
+CREATE EXTERNAL TABLE t_double (a0 DOUBLE, a DOUBLE, b DOUBLE, c DOUBLE, d DOUBLE)
+STORED AS CSV
+WITH ORDER (a ASC, b ASC)
+LOCATION '../core/tests/data/window_2.csv'
+OPTIONS ('format.has_header' 'true');
+
+query TT
+EXPLAIN SELECT a, b FROM t_double WHERE a = sqrt(b);
+----
+physical_plan
+01)FilterExec: a@0 = sqrt(b@1)
+02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true
+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[a, b], output_ordering=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST], file_type=csv, has_header=true
+
+query I
+SELECT count(*) FROM t_double WHERE a = sqrt(b);
+----
+25
+
+statement ok
+DROP TABLE t_double;
+
###
### Order-preserving repartition of an unbounded input
###