Apache DataFusion SQL Query Engine
Split and deprecate `and_or` (#24276)
Nagato Yuzuruceaef16778f3
diff --git a/datafusion/expr-common/src/sort_properties.rs b/datafusion/expr-common/src/sort_properties.rsindex 6f18a8a46a..af417ed16d 100644--- a/datafusion/expr-common/src/sort_properties.rs+++ b/datafusion/expr-common/src/sort_properties.rs@@ -100,23 +100,32 @@ impl SortProperties {}}- pub fn and_or(&self, rhs: &Self) -> Self {+ /// `AND` keeps an ordering only when nulls sort after `true`+ /// (`ASC NULLS LAST`, `DESC NULLS FIRST`).+ pub fn and(&self, rhs: &Self) -> Self {+ self.kleene(rhs, |opt| opt.descending == opt.nulls_first)+ }++ /// `OR` keeps an ordering only when nulls sort before `false`+ /// (`ASC NULLS FIRST`, `DESC NULLS LAST`).+ pub fn or(&self, rhs: &Self) -> Self {+ self.kleene(rhs, |opt| opt.descending != opt.nulls_first)+ }++ /// `preserves` reports whether the operator keeps data in that ordering.+ /// A `Singleton` may be a literal NULL, so it carries the same guard.+ fn kleene(&self, rhs: &Self, preserves: fn(&SortOptions) -> bool) -> Self {match (self, rhs) {- (Self::Ordered(lhs), Self::Ordered(rhs))- if lhs.descending == rhs.descending- && lhs.nulls_first == rhs.nulls_first =>+ (Self::Singleton, Self::Singleton) => Self::Singleton,+ (Self::Ordered(opt), Self::Singleton)+ | (Self::Singleton, Self::Ordered(opt))+ if preserves(opt) =>{- Self::Ordered(SortOptions {- descending: lhs.descending,- nulls_first: lhs.nulls_first,- })+ Self::Ordered(*opt)+ }+ (Self::Ordered(lhs), Self::Ordered(rhs)) if lhs == rhs && preserves(lhs) => {+ Self::Ordered(*lhs)}- (Self::Ordered(opt), Self::Singleton)- | (Self::Singleton, Self::Ordered(opt)) => Self::Ordered(SortOptions {- descending: opt.descending,- nulls_first: opt.nulls_first,- }),- (Self::Singleton, Self::Singleton) => Self::Singleton,_ => Self::Unordered,}}@@ -188,39 +197,116 @@ mod sort_properties_test {SINGLETON,SINGLETON,),- // `and_or` backs both `AND` and `OR`, whose rules coincide. Same- // shape as `add`, and likewise commutative.+ // `and` keeps ASC NULLS LAST / DESC NULLS FIRST, `or` keeps ASC+ // NULLS FIRST / DESC NULLS LAST. Both are commutative.+ (+ "and: ASC NULLS LAST is preserved",+ SortProperties::and,+ ASC_NL,+ ASC_NL,+ ASC_NL,+ ),+ (+ "and: DESC NULLS FIRST is preserved",+ SortProperties::and,+ DESC_NF,+ DESC_NF,+ DESC_NF,+ ),+ (+ "and: ASC NULLS FIRST is unordered",+ SortProperties::and,+ ASC_NF,+ ASC_NF,+ UNORDERED,+ ),+ (+ "and: DESC NULLS LAST is unordered",+ SortProperties::and,+ DESC_NL,+ DESC_NL,+ UNORDERED,+ ),+ (+ "and: mixed directions are unordered",+ SortProperties::and,+ ASC_NL,+ DESC_NF,+ UNORDERED,+ ),+ (+ "and: literal with ASC NULLS LAST",+ SortProperties::and,+ SINGLETON,+ ASC_NL,+ ASC_NL,+ ),+ (+ "and: literal with ASC NULLS FIRST is unordered",+ SortProperties::and,+ SINGLETON,+ ASC_NF,+ UNORDERED,+ ),+ (+ "and: two literals stay a literal",+ SortProperties::and,+ SINGLETON,+ SINGLETON,+ SINGLETON,+ ),(- "and_or: same direction is preserved",- SortProperties::and_or,+ "or: ASC NULLS FIRST is preserved",+ SortProperties::or,ASC_NF,ASC_NF,ASC_NF,),(- "and_or: nulls_last placement is preserved",- SortProperties::and_or,+ "or: DESC NULLS LAST is preserved",+ SortProperties::or,DESC_NL,DESC_NL,DESC_NL,),(- "and_or: opposing directions are unordered",- SortProperties::and_or,- ASC_NF,+ "or: ASC NULLS LAST is unordered",+ SortProperties::or,+ ASC_NL,+ ASC_NL,+ UNORDERED,+ ),+ (+ "or: DESC NULLS FIRST is unordered",+ SortProperties::or,DESC_NF,+ DESC_NF,+ UNORDERED,+ ),+ (+ "or: mixed directions are unordered",+ SortProperties::or,+ ASC_NF,+ DESC_NL,UNORDERED,),(- "and_or: literal with ordered",- SortProperties::and_or,+ "or: literal with ASC NULLS FIRST",+ SortProperties::or,SINGLETON,ASC_NF,ASC_NF,),(- "and_or: two literals stay a literal",- SortProperties::and_or,+ "or: literal with ASC NULLS LAST is unordered",+ SortProperties::or,+ SINGLETON,+ ASC_NL,+ UNORDERED,+ ),+ (+ "or: two literals stay a literal",+ SortProperties::or,SINGLETON,SINGLETON,SINGLETON,@@ -333,11 +419,17 @@ mod sort_properties_test {assert_eq!(op(&lhs, &rhs), expected, "case: {name}");}- // `add` and `and_or` are commutative, which is what lets the table- // above cover only one argument order for them.- for (lhs, rhs) in [(ASC_NF, DESC_NL), (ASC_NF, SINGLETON), (ASC_NL, DESC_NF)] {+ // `add`, `and` and `or` are commutative, so the table above covers+ // only one argument order.+ for (lhs, rhs) in [+ (ASC_NF, DESC_NL),+ (ASC_NF, SINGLETON),+ (ASC_NL, SINGLETON),+ (ASC_NL, DESC_NF),+ ] {assert_eq!(lhs.add(&rhs), rhs.add(&lhs), "add is commutative");- assert_eq!(lhs.and_or(&rhs), rhs.and_or(&lhs), "and_or is commutative");+ assert_eq!(lhs.and(&rhs), rhs.and(&lhs), "and is commutative");+ assert_eq!(lhs.or(&rhs), rhs.or(&lhs), "or is commutative");}}@@ -345,9 +437,11 @@ mod sort_properties_test {/// always Unordered, no matter which operator or direction is used./// Checked below for every combination.///- /// Nulls propagate: the result is null wherever either operand is- /// null. `nulls_first` treats those rows as a prefix, `nulls_last` as- /// a suffix. A set that's both can't be described by any `SortOptions`.+ /// Nulls propagate through the arithmetic and comparison operators: the+ /// result is null wherever either operand is null. `nulls_first` treats+ /// those rows as a prefix, `nulls_last` as a suffix. A set that's both+ /// can't be described by any `SortOptions`. `and`/`or` keep an ordering+ /// only when both operands already share it.////// The assertion only checks "not Ordered", not which ordering/// results. That keeps the test from just repeating the logic it's@@ -356,7 +450,8 @@ mod sort_properties_test {#[case::add("add", SortProperties::add)]#[case::sub("sub", SortProperties::sub)]#[case::gt_or_gteq("gt_or_gteq", SortProperties::gt_or_gteq)]- #[case::and_or("and_or", SortProperties::and_or)]+ #[case::and("and", SortProperties::and)]+ #[case::or("or", SortProperties::or)]fn conflicting_null_placement_is_never_ordered(#[values(false, true)] l_descending: bool,#[values(false, true)] r_descending: bool,diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rsindex ee61d80565..c956b83178 100644--- a/datafusion/physical-expr/src/expressions/binary.rs+++ b/datafusion/physical-expr/src/expressions/binary.rs@@ -878,13 +878,13 @@ impl PhysicalExpr for BinaryExpr {strictly_order_preserving: false,}),Operator::And => Ok(ExprProperties {- sort_properties: r_order.and_or(&l_order),+ sort_properties: l_order.and(&r_order),range: l_range.and(r_range)?,preserves_lex_ordering: false,strictly_order_preserving: false,}),Operator::Or => Ok(ExprProperties {- sort_properties: r_order.and_or(&l_order),+ sort_properties: l_order.or(&r_order),range: l_range.or(r_range)?,preserves_lex_ordering: false,strictly_order_preserving: false,diff --git a/datafusion/sqllogictest/test_files/order.slt b/datafusion/sqllogictest/test_files/order.sltindex dfe5c0faab..350e340fdd 100644--- a/datafusion/sqllogictest/test_files/order.slt+++ b/datafusion/sqllogictest/test_files/order.slt@@ -1591,6 +1591,77 @@ physical_planstatement okDROP TABLE mixed_null_placement;+# ASC NULLS FIRST survives OR but not AND. The per-operator rules are+# unit-tested in sort_properties.rs; this checks the planner uses them.+query I+COPY (VALUES+ (CAST(NULL AS BOOLEAN), false),+ (CAST(NULL AS BOOLEAN), true),+ (false, true),+ (true, true))+TO 'test_files/scratch/order/kleene_nulls_first.csv'+OPTIONS ('format.has_header' 'true');+----+4++statement ok+CREATE EXTERNAL TABLE kleene_nulls_first (a BOOLEAN, b BOOLEAN)+STORED AS CSV+WITH ORDER (a ASC NULLS FIRST)+WITH ORDER (b ASC NULLS FIRST)+LOCATION 'test_files/scratch/order/kleene_nulls_first.csv'+OPTIONS ('format.has_header' 'true');++# a AND b is (false, NULL, false, true) in scan order, so it needs sorting.+query B+SELECT a AND b AS c FROM kleene_nulls_first ORDER BY c ASC NULLS FIRST;+----+NULL+false+false+true++# AND must not claim ASC NULLS FIRST.+query TT+EXPLAIN SELECT a AND b AS c FROM kleene_nulls_first ORDER BY c ASC NULLS FIRST;+----+logical_plan+01)Sort: c ASC NULLS FIRST+02)--Projection: kleene_nulls_first.a AND kleene_nulls_first.b AS c+03)----TableScan: kleene_nulls_first projection=[a, b]+physical_plan+01)SortPreservingMergeExec: [c@0 ASC]+02)--SortExec: expr=[c@0 ASC], preserve_partitioning=[true]+03)----ProjectionExec: expr=[a@0 AND b@1 as c]+04)------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1, maintains_sort_order=true+05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/order/kleene_nulls_first.csv]]}, projection=[a, b], output_orderings=[[a@0 ASC], [b@1 ASC]], file_type=csv, has_header=true++# a OR b is (NULL, true, true, true) in scan order, already ASC NULLS FIRST,+# so no SortExec.+query B+SELECT a OR b AS c FROM kleene_nulls_first ORDER BY c ASC NULLS FIRST;+----+NULL+true+true+true++query TT+EXPLAIN SELECT a OR b AS c FROM kleene_nulls_first ORDER BY c ASC NULLS FIRST;+----+logical_plan+01)Sort: c ASC NULLS FIRST+02)--Projection: kleene_nulls_first.a OR kleene_nulls_first.b AS c+03)----TableScan: kleene_nulls_first projection=[a, b]+physical_plan+01)SortPreservingMergeExec: [c@0 ASC]+02)--ProjectionExec: expr=[a@0 OR b@1 as c]+03)----RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1, maintains_sort_order=true+04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/order/kleene_nulls_first.csv]]}, projection=[a, b], output_orderings=[[a@0 ASC], [b@1 ASC]], file_type=csv, has_header=true++statement ok+DROP TABLE kleene_nulls_first;+# Union a query with the actual data and one with a constantquery ISELECT (SELECT c from ordered_table ORDER BY c LIMIT 1) UNION ALL (SELECT 23 as c from ordered_table ORDER BY c LIMIT 1) ORDER BY c;diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.mdindex c87dbc3b8a..e88722bbf4 100644--- a/docs/source/library-user-guide/upgrading/55.0.0.md+++ b/docs/source/library-user-guide/upgrading/55.0.0.md@@ -1372,6 +1372,28 @@ following the pattern in [`parquet/examples/object_store.rs`]See [PR #24030](https://github.com/apache/datafusion/pull/24030) for details.+### `SortProperties::and_or` replaced by `and` and `or`++`datafusion_expr_common::sort_properties::SortProperties::and_or` has been+removed. Call `and` for `AND` and `or` for `OR`:++```rust,ignore+// Before (54.0.0)+let props = lhs.and_or(&rhs);++// After (55.0.0)+let props = lhs.and(&rhs); // for AND+let props = lhs.or(&rhs); // for OR+```++The two operators do not preserve the same sort orders. `AND` keeps an ordering+only when nulls sort after `true` (`ASC NULLS LAST` or `DESC NULLS FIRST`), and+`OR` only when nulls sort before `false` (`ASC NULLS FIRST` or+`DESC NULLS LAST`). `and_or` did not say which operator the caller meant, so it+could not be right for both.++See [PR #24276](https://github.com/apache/datafusion/pull/24276) for details.+[`parquet` crate]: https://crates.io/crates/parquet[`parquetobjectreader`]: https://docs.rs/parquet/59.1.0/parquet/arrow/async_reader/struct.ParquetObjectReader.html[`parquetobjectwriter`]: https://docs.rs/parquet/59.1.0/parquet/arrow/async_writer/struct.ParquetObjectWriter.html