Skip to content

Apache DataFusion SQL Query Engine

Split and deprecate `and_or` (#24276)

Nagato Yuzuruceaef16778f3
diff --git a/datafusion/expr-common/src/sort_properties.rs b/datafusion/expr-common/src/sort_properties.rs
index 6f18a8a46a..af417ed16d 100644
--- a/datafusion/expr-common/src/sort_properties.rs
+++ b/datafusion/expr-common/src/sort_properties.rs
@@ -100,23 +100,32 @@ impl SortProperties {
}
}
- pub fn and_or(&self, rhs: &Self) -> Self {
+ /// `AND` keeps an ordering only when nulls sort after `true`
+ /// (`ASC NULLS LAST`, `DESC NULLS FIRST`).
+ pub fn and(&self, rhs: &Self) -> Self {
+ self.kleene(rhs, |opt| opt.descending == opt.nulls_first)
+ }
+
+ /// `OR` keeps an ordering only when nulls sort before `false`
+ /// (`ASC NULLS FIRST`, `DESC NULLS LAST`).
+ pub fn or(&self, rhs: &Self) -> Self {
+ self.kleene(rhs, |opt| opt.descending != opt.nulls_first)
+ }
+
+ /// `preserves` reports whether the operator keeps data in that ordering.
+ /// A `Singleton` may be a literal NULL, so it carries the same guard.
+ fn kleene(&self, rhs: &Self, preserves: fn(&SortOptions) -> bool) -> Self {
match (self, rhs) {
- (Self::Ordered(lhs), Self::Ordered(rhs))
- if lhs.descending == rhs.descending
- && lhs.nulls_first == rhs.nulls_first =>
+ (Self::Singleton, Self::Singleton) => Self::Singleton,
+ (Self::Ordered(opt), Self::Singleton)
+ | (Self::Singleton, Self::Ordered(opt))
+ if preserves(opt) =>
{
- Self::Ordered(SortOptions {
- descending: lhs.descending,
- nulls_first: lhs.nulls_first,
- })
+ Self::Ordered(*opt)
+ }
+ (Self::Ordered(lhs), Self::Ordered(rhs)) if lhs == rhs && preserves(lhs) => {
+ Self::Ordered(*lhs)
}
- (Self::Ordered(opt), Self::Singleton)
- | (Self::Singleton, Self::Ordered(opt)) => Self::Ordered(SortOptions {
- descending: opt.descending,
- nulls_first: opt.nulls_first,
- }),
- (Self::Singleton, Self::Singleton) => Self::Singleton,
_ => Self::Unordered,
}
}
@@ -188,39 +197,116 @@ mod sort_properties_test {
SINGLETON,
SINGLETON,
),
- // `and_or` backs both `AND` and `OR`, whose rules coincide. Same
- // shape as `add`, and likewise commutative.
+ // `and` keeps ASC NULLS LAST / DESC NULLS FIRST, `or` keeps ASC
+ // NULLS FIRST / DESC NULLS LAST. Both are commutative.
+ (
+ "and: ASC NULLS LAST is preserved",
+ SortProperties::and,
+ ASC_NL,
+ ASC_NL,
+ ASC_NL,
+ ),
+ (
+ "and: DESC NULLS FIRST is preserved",
+ SortProperties::and,
+ DESC_NF,
+ DESC_NF,
+ DESC_NF,
+ ),
+ (
+ "and: ASC NULLS FIRST is unordered",
+ SortProperties::and,
+ ASC_NF,
+ ASC_NF,
+ UNORDERED,
+ ),
+ (
+ "and: DESC NULLS LAST is unordered",
+ SortProperties::and,
+ DESC_NL,
+ DESC_NL,
+ UNORDERED,
+ ),
+ (
+ "and: mixed directions are unordered",
+ SortProperties::and,
+ ASC_NL,
+ DESC_NF,
+ UNORDERED,
+ ),
+ (
+ "and: literal with ASC NULLS LAST",
+ SortProperties::and,
+ SINGLETON,
+ ASC_NL,
+ ASC_NL,
+ ),
+ (
+ "and: literal with ASC NULLS FIRST is unordered",
+ SortProperties::and,
+ SINGLETON,
+ ASC_NF,
+ UNORDERED,
+ ),
+ (
+ "and: two literals stay a literal",
+ SortProperties::and,
+ SINGLETON,
+ SINGLETON,
+ SINGLETON,
+ ),
(
- "and_or: same direction is preserved",
- SortProperties::and_or,
+ "or: ASC NULLS FIRST is preserved",
+ SortProperties::or,
ASC_NF,
ASC_NF,
ASC_NF,
),
(
- "and_or: nulls_last placement is preserved",
- SortProperties::and_or,
+ "or: DESC NULLS LAST is preserved",
+ SortProperties::or,
DESC_NL,
DESC_NL,
DESC_NL,
),
(
- "and_or: opposing directions are unordered",
- SortProperties::and_or,
- ASC_NF,
+ "or: ASC NULLS LAST is unordered",
+ SortProperties::or,
+ ASC_NL,
+ ASC_NL,
+ UNORDERED,
+ ),
+ (
+ "or: DESC NULLS FIRST is unordered",
+ SortProperties::or,
DESC_NF,
+ DESC_NF,
+ UNORDERED,
+ ),
+ (
+ "or: mixed directions are unordered",
+ SortProperties::or,
+ ASC_NF,
+ DESC_NL,
UNORDERED,
),
(
- "and_or: literal with ordered",
- SortProperties::and_or,
+ "or: literal with ASC NULLS FIRST",
+ SortProperties::or,
SINGLETON,
ASC_NF,
ASC_NF,
),
(
- "and_or: two literals stay a literal",
- SortProperties::and_or,
+ "or: literal with ASC NULLS LAST is unordered",
+ SortProperties::or,
+ SINGLETON,
+ ASC_NL,
+ UNORDERED,
+ ),
+ (
+ "or: two literals stay a literal",
+ SortProperties::or,
SINGLETON,
SINGLETON,
SINGLETON,
@@ -333,11 +419,17 @@ mod sort_properties_test {
assert_eq!(op(&lhs, &rhs), expected, "case: {name}");
}
- // `add` and `and_or` are commutative, which is what lets the table
- // above cover only one argument order for them.
- for (lhs, rhs) in [(ASC_NF, DESC_NL), (ASC_NF, SINGLETON), (ASC_NL, DESC_NF)] {
+ // `add`, `and` and `or` are commutative, so the table above covers
+ // only one argument order.
+ for (lhs, rhs) in [
+ (ASC_NF, DESC_NL),
+ (ASC_NF, SINGLETON),
+ (ASC_NL, SINGLETON),
+ (ASC_NL, DESC_NF),
+ ] {
assert_eq!(lhs.add(&rhs), rhs.add(&lhs), "add is commutative");
- assert_eq!(lhs.and_or(&rhs), rhs.and_or(&lhs), "and_or is commutative");
+ assert_eq!(lhs.and(&rhs), rhs.and(&lhs), "and is commutative");
+ assert_eq!(lhs.or(&rhs), rhs.or(&lhs), "or is commutative");
}
}
@@ -345,9 +437,11 @@ mod sort_properties_test {
/// always Unordered, no matter which operator or direction is used.
/// Checked below for every combination.
///
- /// Nulls propagate: the result is null wherever either operand is
- /// null. `nulls_first` treats those rows as a prefix, `nulls_last` as
- /// a suffix. A set that's both can't be described by any `SortOptions`.
+ /// Nulls propagate through the arithmetic and comparison operators: the
+ /// result is null wherever either operand is null. `nulls_first` treats
+ /// those rows as a prefix, `nulls_last` as a suffix. A set that's both
+ /// can't be described by any `SortOptions`. `and`/`or` keep an ordering
+ /// only when both operands already share it.
///
/// The assertion only checks "not Ordered", not which ordering
/// results. That keeps the test from just repeating the logic it's
@@ -356,7 +450,8 @@ mod sort_properties_test {
#[case::add("add", SortProperties::add)]
#[case::sub("sub", SortProperties::sub)]
#[case::gt_or_gteq("gt_or_gteq", SortProperties::gt_or_gteq)]
- #[case::and_or("and_or", SortProperties::and_or)]
+ #[case::and("and", SortProperties::and)]
+ #[case::or("or", SortProperties::or)]
fn conflicting_null_placement_is_never_ordered(
#[values(false, true)] l_descending: bool,
#[values(false, true)] r_descending: bool,
diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rs
index ee61d80565..c956b83178 100644
--- a/datafusion/physical-expr/src/expressions/binary.rs
+++ b/datafusion/physical-expr/src/expressions/binary.rs
@@ -878,13 +878,13 @@ impl PhysicalExpr for BinaryExpr {
strictly_order_preserving: false,
}),
Operator::And => Ok(ExprProperties {
- sort_properties: r_order.and_or(&l_order),
+ sort_properties: l_order.and(&r_order),
range: l_range.and(r_range)?,
preserves_lex_ordering: false,
strictly_order_preserving: false,
}),
Operator::Or => Ok(ExprProperties {
- sort_properties: r_order.and_or(&l_order),
+ sort_properties: l_order.or(&r_order),
range: l_range.or(r_range)?,
preserves_lex_ordering: false,
strictly_order_preserving: false,
diff --git a/datafusion/sqllogictest/test_files/order.slt b/datafusion/sqllogictest/test_files/order.slt
index dfe5c0faab..350e340fdd 100644
--- a/datafusion/sqllogictest/test_files/order.slt
+++ b/datafusion/sqllogictest/test_files/order.slt
@@ -1591,6 +1591,77 @@ physical_plan
statement ok
DROP TABLE mixed_null_placement;
+# ASC NULLS FIRST survives OR but not AND. The per-operator rules are
+# unit-tested in sort_properties.rs; this checks the planner uses them.
+query I
+COPY (VALUES
+ (CAST(NULL AS BOOLEAN), false),
+ (CAST(NULL AS BOOLEAN), true),
+ (false, true),
+ (true, true))
+TO 'test_files/scratch/order/kleene_nulls_first.csv'
+OPTIONS ('format.has_header' 'true');
+----
+4
+
+statement ok
+CREATE EXTERNAL TABLE kleene_nulls_first (a BOOLEAN, b BOOLEAN)
+STORED AS CSV
+WITH ORDER (a ASC NULLS FIRST)
+WITH ORDER (b ASC NULLS FIRST)
+LOCATION 'test_files/scratch/order/kleene_nulls_first.csv'
+OPTIONS ('format.has_header' 'true');
+
+# a AND b is (false, NULL, false, true) in scan order, so it needs sorting.
+query B
+SELECT a AND b AS c FROM kleene_nulls_first ORDER BY c ASC NULLS FIRST;
+----
+NULL
+false
+false
+true
+
+# AND must not claim ASC NULLS FIRST.
+query TT
+EXPLAIN SELECT a AND b AS c FROM kleene_nulls_first ORDER BY c ASC NULLS FIRST;
+----
+logical_plan
+01)Sort: c ASC NULLS FIRST
+02)--Projection: kleene_nulls_first.a AND kleene_nulls_first.b AS c
+03)----TableScan: kleene_nulls_first projection=[a, b]
+physical_plan
+01)SortPreservingMergeExec: [c@0 ASC]
+02)--SortExec: expr=[c@0 ASC], preserve_partitioning=[true]
+03)----ProjectionExec: expr=[a@0 AND b@1 as c]
+04)------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1, maintains_sort_order=true
+05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/order/kleene_nulls_first.csv]]}, projection=[a, b], output_orderings=[[a@0 ASC], [b@1 ASC]], file_type=csv, has_header=true
+
+# a OR b is (NULL, true, true, true) in scan order, already ASC NULLS FIRST,
+# so no SortExec.
+query B
+SELECT a OR b AS c FROM kleene_nulls_first ORDER BY c ASC NULLS FIRST;
+----
+NULL
+true
+true
+true
+
+query TT
+EXPLAIN SELECT a OR b AS c FROM kleene_nulls_first ORDER BY c ASC NULLS FIRST;
+----
+logical_plan
+01)Sort: c ASC NULLS FIRST
+02)--Projection: kleene_nulls_first.a OR kleene_nulls_first.b AS c
+03)----TableScan: kleene_nulls_first projection=[a, b]
+physical_plan
+01)SortPreservingMergeExec: [c@0 ASC]
+02)--ProjectionExec: expr=[a@0 OR b@1 as c]
+03)----RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1, maintains_sort_order=true
+04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/order/kleene_nulls_first.csv]]}, projection=[a, b], output_orderings=[[a@0 ASC], [b@1 ASC]], file_type=csv, has_header=true
+
+statement ok
+DROP TABLE kleene_nulls_first;
+
# Union a query with the actual data and one with a constant
query I
SELECT (SELECT c from ordered_table ORDER BY c LIMIT 1) UNION ALL (SELECT 23 as c from ordered_table ORDER BY c LIMIT 1) ORDER BY c;
diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.md
index c87dbc3b8a..e88722bbf4 100644
--- a/docs/source/library-user-guide/upgrading/55.0.0.md
+++ b/docs/source/library-user-guide/upgrading/55.0.0.md
@@ -1372,6 +1372,28 @@ following the pattern in [`parquet/examples/object_store.rs`]
See [PR #24030](https://github.com/apache/datafusion/pull/24030) for details.
+### `SortProperties::and_or` replaced by `and` and `or`
+
+`datafusion_expr_common::sort_properties::SortProperties::and_or` has been
+removed. Call `and` for `AND` and `or` for `OR`:
+
+```rust,ignore
+// Before (54.0.0)
+let props = lhs.and_or(&rhs);
+
+// After (55.0.0)
+let props = lhs.and(&rhs); // for AND
+let props = lhs.or(&rhs); // for OR
+```
+
+The two operators do not preserve the same sort orders. `AND` keeps an ordering
+only when nulls sort after `true` (`ASC NULLS LAST` or `DESC NULLS FIRST`), and
+`OR` only when nulls sort before `false` (`ASC NULLS FIRST` or
+`DESC NULLS LAST`). `and_or` did not say which operator the caller meant, so it
+could not be right for both.
+
+See [PR #24276](https://github.com/apache/datafusion/pull/24276) for details.
+
[`parquet` crate]: https://crates.io/crates/parquet
[`parquetobjectreader`]: https://docs.rs/parquet/59.1.0/parquet/arrow/async_reader/struct.ParquetObjectReader.html
[`parquetobjectwriter`]: https://docs.rs/parquet/59.1.0/parquet/arrow/async_writer/struct.ParquetObjectWriter.html