Apache DataFusion SQL Query Engine
fix: consider nulls_first when propagating ordered SortProperties (#24206)
Nagato Yuzuru4042812a86f4
diff --git a/Cargo.lock b/Cargo.lockindex d7576659f0..cca0d1fdbf 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2196,6 +2196,7 @@ dependencies = ["indexmap 2.14.0","insta","itertools 0.15.0",+ "rstest",][[package]]diff --git a/datafusion/expr-common/Cargo.toml b/datafusion/expr-common/Cargo.tomlindex 072c8f14da..026c1e0bc3 100644--- a/datafusion/expr-common/Cargo.toml+++ b/datafusion/expr-common/Cargo.toml@@ -48,3 +48,4 @@ itertools = { workspace = true }[dev-dependencies]insta = { workspace = true }+rstest = { workspace = true }diff --git a/datafusion/expr-common/src/sort_properties.rs b/datafusion/expr-common/src/sort_properties.rsindex 74d644f79f..6f18a8a46a 100644--- a/datafusion/expr-common/src/sort_properties.rs+++ b/datafusion/expr-common/src/sort_properties.rs@@ -50,11 +50,12 @@ impl SortProperties {(Self::Singleton, _) => *rhs,(_, Self::Singleton) => *self,(Self::Ordered(lhs), Self::Ordered(rhs))- if lhs.descending == rhs.descending =>+ if lhs.descending == rhs.descending+ && lhs.nulls_first == rhs.nulls_first =>{Self::Ordered(SortOptions {descending: lhs.descending,- nulls_first: lhs.nulls_first || rhs.nulls_first,+ nulls_first: lhs.nulls_first,})}_ => Self::Unordered,@@ -70,11 +71,12 @@ impl SortProperties {}),(_, Self::Singleton) => *self,(Self::Ordered(lhs), Self::Ordered(rhs))- if lhs.descending != rhs.descending =>+ if lhs.descending != rhs.descending+ && lhs.nulls_first == rhs.nulls_first =>{Self::Ordered(SortOptions {descending: lhs.descending,- nulls_first: lhs.nulls_first || rhs.nulls_first,+ nulls_first: lhs.nulls_first,})}_ => Self::Unordered,@@ -89,7 +91,8 @@ impl SortProperties {}),(_, Self::Singleton) => *self,(Self::Ordered(lhs), Self::Ordered(rhs))- if lhs.descending != rhs.descending =>+ if lhs.descending != rhs.descending+ && lhs.nulls_first == rhs.nulls_first =>{*self}@@ -100,11 +103,12 @@ impl SortProperties {pub fn and_or(&self, rhs: &Self) -> Self {match (self, rhs) {(Self::Ordered(lhs), Self::Ordered(rhs))- if lhs.descending == rhs.descending =>+ if lhs.descending == rhs.descending+ && lhs.nulls_first == rhs.nulls_first =>{Self::Ordered(SortOptions {descending: lhs.descending,- nulls_first: lhs.nulls_first || rhs.nulls_first,+ nulls_first: lhs.nulls_first,})}(Self::Ordered(opt), Self::Singleton)@@ -118,6 +122,255 @@ impl SortProperties {}}+#[cfg(test)]+mod sort_properties_test {+ use super::{SortOptions, SortProperties};+ use rstest::rstest;++ const fn ordered(descending: bool, nulls_first: bool) -> SortProperties {+ SortProperties::Ordered(SortOptions {+ descending,+ nulls_first,+ })+ }++ const ASC_NF: SortProperties = ordered(false, true);+ const ASC_NL: SortProperties = ordered(false, false);+ const DESC_NF: SortProperties = ordered(true, true);+ const DESC_NL: SortProperties = ordered(true, false);+ const UNORDERED: SortProperties = SortProperties::Unordered;+ const SINGLETON: SortProperties = SortProperties::Singleton;++ type BinOp = fn(&SortProperties, &SortProperties) -> SortProperties;++ /// Each method's direction rule and its `Singleton` arms.+ ///+ /// Operands that *disagree* on null placement are deliberately absent:+ /// that half is covered exhaustively by+ /// [`conflicting_null_placement_is_never_ordered`].+ #[test]+ fn ordering_propagation() {+ let cases: &[(&str, BinOp, SortProperties, SortProperties, SortProperties)] = &[+ // `add` preserves ordering when both operands run in the same+ // direction. It is commutative, so one argument order suffices.+ (+ "add: same direction is preserved",+ SortProperties::add,+ ASC_NF,+ ASC_NF,+ ASC_NF,+ ),+ (+ "add: nulls_last placement is preserved",+ SortProperties::add,+ ASC_NL,+ ASC_NL,+ ASC_NL,+ ),+ (+ "add: opposing directions are unordered",+ SortProperties::add,+ ASC_NF,+ DESC_NF,+ UNORDERED,+ ),+ (+ "add: literal with ordered",+ SortProperties::add,+ SINGLETON,+ ASC_NF,+ ASC_NF,+ ),+ (+ "add: two literals stay a literal",+ SortProperties::add,+ SINGLETON,+ SINGLETON,+ SINGLETON,+ ),+ // `and_or` backs both `AND` and `OR`, whose rules coincide. Same+ // shape as `add`, and likewise commutative.+ (+ "and_or: same direction is preserved",+ SortProperties::and_or,+ ASC_NF,+ ASC_NF,+ ASC_NF,+ ),+ (+ "and_or: nulls_last placement is preserved",+ SortProperties::and_or,+ DESC_NL,+ DESC_NL,+ DESC_NL,+ ),+ (+ "and_or: opposing directions are unordered",+ SortProperties::and_or,+ ASC_NF,+ DESC_NF,+ UNORDERED,+ ),+ (+ "and_or: literal with ordered",+ SortProperties::and_or,+ SINGLETON,+ ASC_NF,+ ASC_NF,+ ),+ (+ "and_or: two literals stay a literal",+ SortProperties::and_or,+ SINGLETON,+ SINGLETON,+ SINGLETON,+ ),+ // `sub` needs the *opposite* rule: an ascending column minus a+ // descending one still ascends. It is not commutative+ (+ "sub: opposing directions are preserved",+ SortProperties::sub,+ ASC_NF,+ DESC_NF,+ ASC_NF,+ ),+ (+ "sub: result follows the left operand",+ SortProperties::sub,+ DESC_NF,+ ASC_NF,+ DESC_NF,+ ),+ (+ "sub: nulls_last placement is preserved",+ SortProperties::sub,+ ASC_NL,+ DESC_NL,+ ASC_NL,+ ),+ (+ "sub: same direction is unordered",+ SortProperties::sub,+ ASC_NF,+ ASC_NF,+ UNORDERED,+ ),+ (+ "sub: literal minus ordered flips the direction",+ SortProperties::sub,+ SINGLETON,+ ASC_NF,+ DESC_NF,+ ),+ (+ "sub: ordered minus literal keeps the direction",+ SortProperties::sub,+ ASC_NF,+ SINGLETON,+ ASC_NF,+ ),+ (+ "sub: two literals stay a literal",+ SortProperties::sub,+ SINGLETON,+ SINGLETON,+ SINGLETON,+ ),+ // `gt_or_gteq` compares into a boolean column, which is ordered by+ // `false < true`. Same direction rule as `sub`, also asymmetric.+ (+ "gt_or_gteq: opposing directions are preserved",+ SortProperties::gt_or_gteq,+ ASC_NF,+ DESC_NF,+ ASC_NF,+ ),+ (+ "gt_or_gteq: result follows the left operand",+ SortProperties::gt_or_gteq,+ DESC_NF,+ ASC_NF,+ DESC_NF,+ ),+ (+ "gt_or_gteq: nulls_last placement is preserved",+ SortProperties::gt_or_gteq,+ DESC_NL,+ ASC_NL,+ DESC_NL,+ ),+ (+ "gt_or_gteq: same direction is unordered",+ SortProperties::gt_or_gteq,+ ASC_NF,+ ASC_NF,+ UNORDERED,+ ),+ (+ "gt_or_gteq: literal on the left flips the direction",+ SortProperties::gt_or_gteq,+ SINGLETON,+ ASC_NF,+ DESC_NF,+ ),+ (+ "gt_or_gteq: literal on the right keeps the direction",+ SortProperties::gt_or_gteq,+ ASC_NF,+ SINGLETON,+ ASC_NF,+ ),+ (+ "gt_or_gteq: two literals stay a literal",+ SortProperties::gt_or_gteq,+ SINGLETON,+ SINGLETON,+ SINGLETON,+ ),+ ];++ for &(name, op, lhs, rhs, expected) in cases {+ assert_eq!(op(&lhs, &rhs), expected, "case: {name}");+ }++ // `add` and `and_or` are commutative, which is what lets the table+ // above cover only one argument order for them.+ for (lhs, rhs) in [(ASC_NF, DESC_NL), (ASC_NF, SINGLETON), (ASC_NL, DESC_NF)] {+ assert_eq!(lhs.add(&rhs), rhs.add(&lhs), "add is commutative");+ assert_eq!(lhs.and_or(&rhs), rhs.and_or(&lhs), "and_or is commutative");+ }+ }++ /// If two ordered operands disagree on null placement, the result is+ /// always Unordered, no matter which operator or direction is used.+ /// Checked below for every combination.+ ///+ /// Nulls propagate: the result is null wherever either operand is+ /// null. `nulls_first` treats those rows as a prefix, `nulls_last` as+ /// a suffix. A set that's both can't be described by any `SortOptions`.+ ///+ /// The assertion only checks "not Ordered", not which ordering+ /// results. That keeps the test from just repeating the logic it's+ /// checking.+ #[rstest]+ #[case::add("add", SortProperties::add)]+ #[case::sub("sub", SortProperties::sub)]+ #[case::gt_or_gteq("gt_or_gteq", SortProperties::gt_or_gteq)]+ #[case::and_or("and_or", SortProperties::and_or)]+ fn conflicting_null_placement_is_never_ordered(+ #[values(false, true)] l_descending: bool,+ #[values(false, true)] r_descending: bool,+ #[values(false, true)] l_nulls_first: bool,+ #[case] op_name: &str,+ #[case] op: BinOp,+ ) {+ // Negating `l_nulls_first` makes the operands disagree by construction.+ let lhs = ordered(l_descending, l_nulls_first);+ let rhs = ordered(r_descending, !l_nulls_first);+ assert_eq!(op(&lhs, &rhs), UNORDERED, "{op_name}: {lhs:?} and {rhs:?}");+ }+}+impl Neg for SortProperties {type Output = Self;diff --git a/datafusion/sqllogictest/test_files/order.slt b/datafusion/sqllogictest/test_files/order.sltindex 4b136d24b0..dfe5c0faab 100644--- a/datafusion/sqllogictest/test_files/order.slt+++ b/datafusion/sqllogictest/test_files/order.slt@@ -1485,7 +1485,9 @@ physical_plan03)----RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1, maintains_sort_order=true04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[c], output_ordering=[c@0 ASC NULLS LAST], file_type=csv, has_header=true-# Boolean to integer casts preserve the order.+# The cast preserves the order, but the comparison does not: `inc_col` is+# ASC NULLS LAST while `desc_col` is DESC NULLS FIRST, so `inc_col > desc_col`+# is null at both ends and the sort has to stay.statement okCREATE EXTERNAL TABLE annotated_data_finite (ts INTEGER,@@ -1507,9 +1509,87 @@ logical_plan03)----TableScan: annotated_data_finite projection=[inc_col, desc_col]physical_plan01)SortPreservingMergeExec: [c@0 ASC NULLS LAST]+02)--SortExec: expr=[c@0 ASC NULLS LAST], preserve_partitioning=[true]+03)----ProjectionExec: expr=[CAST(inc_col@0 > desc_col@1 AS Int32) as c]+04)------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1, maintains_sort_order=true+05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_1.csv]]}, projection=[inc_col, desc_col], output_orderings=[[inc_col@0 ASC NULLS LAST], [desc_col@1 DESC]], file_type=csv, has_header=true++# With matching null placement the comparison keeps its order: no sort needed.+statement ok+CREATE EXTERNAL TABLE annotated_data_finite_nulls_last (+ ts INTEGER,+ inc_col INTEGER,+ desc_col INTEGER,+)+STORED AS CSV+WITH ORDER (inc_col ASC NULLS LAST)+WITH ORDER (desc_col DESC NULLS LAST)+LOCATION '../core/tests/data/window_1.csv'+OPTIONS ('format.has_header' 'true');++query TT+EXPLAIN SELECT CAST((inc_col>desc_col) as integer) as c from annotated_data_finite_nulls_last order by c;+----+logical_plan+01)Sort: c ASC NULLS LAST+02)--Projection: CAST(annotated_data_finite_nulls_last.inc_col > annotated_data_finite_nulls_last.desc_col AS Int32) AS c+03)----TableScan: annotated_data_finite_nulls_last projection=[inc_col, desc_col]+physical_plan+01)SortPreservingMergeExec: [c@0 ASC NULLS LAST]02)--ProjectionExec: expr=[CAST(inc_col@0 > desc_col@1 AS Int32) as c]03)----RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1, maintains_sort_order=true-04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_1.csv]]}, projection=[inc_col, desc_col], output_orderings=[[inc_col@0 ASC NULLS LAST], [desc_col@1 DESC]], file_type=csv, has_header=true+04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_1.csv]]}, projection=[inc_col, desc_col], output_orderings=[[inc_col@0 ASC NULLS LAST], [desc_col@1 DESC NULLS LAST]], file_type=csv, has_header=true++# Regression test for #11596. `a` is ASC NULLS FIRST and `b` is ASC NULLS LAST,+# so `a + b` is null at both ends of the scan: (NULL, 3, 6, NULL). No nulls+# placement describes that.+query I+COPY (VALUES (NULL, 1), (1, 2), (2, 4), (3, NULL))+TO 'test_files/scratch/order/mixed_null_placement.csv'+OPTIONS ('format.has_header' 'true');+----+4++statement ok+CREATE EXTERNAL TABLE mixed_null_placement (+ a BIGINT,+ b BIGINT+)+STORED AS CSV+WITH ORDER (a ASC NULLS FIRST)+WITH ORDER (b ASC NULLS LAST)+LOCATION 'test_files/scratch/order/mixed_null_placement.csv'+OPTIONS ('format.has_header' 'true');++# The sort has to happen. Without it, this comes back in scan order.+query I+SELECT a + b AS s FROM mixed_null_placement ORDER BY s;+----+3+6+NULL+NULL++# Same at the plan level. Two separate things keep this sort: `add()` won't+# merge mismatched `nulls_first`, and `arithmetic_sort_properties` gives up on+# any `col + col` with unbounded ranges, which is every column read from a+# file. The second masks the first today.+query TT+EXPLAIN SELECT a + b AS s FROM mixed_null_placement ORDER BY s;+----+logical_plan+01)Sort: s ASC NULLS LAST+02)--Projection: mixed_null_placement.a + mixed_null_placement.b AS s+03)----TableScan: mixed_null_placement projection=[a, b]+physical_plan+01)SortPreservingMergeExec: [s@0 ASC NULLS LAST]+02)--SortExec: expr=[s@0 ASC NULLS LAST], preserve_partitioning=[true]+03)----ProjectionExec: expr=[a@0 + b@1 as s]+04)------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1, maintains_sort_order=true+05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/order/mixed_null_placement.csv]]}, projection=[a, b], output_orderings=[[a@0 ASC], [b@1 ASC NULLS LAST]], file_type=csv, has_header=true++statement ok+DROP TABLE mixed_null_placement;# Union a query with the actual data and one with a constantquery I