Apache DataFusion SQL Query Engine
fix: `SanityCheckPlan` error when a per-partition constant leads an ordering (#27191)
Jay Zhanac1a63d2ce20
diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rsindex b45ee313ca..8cc37f464a 100644--- a/datafusion/physical-expr/src/equivalence/properties/mod.rs+++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs@@ -630,7 +630,8 @@ impl EquivalenceProperties {}}- /// Adds new orderings into the existing ordering equivalence class.+ /// Adds new orderings into the existing ordering equivalence class, along+ /// with the orderings they imply through known equalities.pub fn add_orderings(&mut self,orderings: impl IntoIterator<Item = impl IntoIterator<Item = PhysicalSortExpr>>,@@ -642,14 +643,32 @@ impl EquivalenceProperties {.cloned().filter_map(|o| self.normalize_sort_exprs(o)).collect();- if !normal_orderings.is_empty() {- self.oeq_class.extend(orderings);- // Normalize given orderings to update the cache:- self.oeq_cache.normal_cls.extend(normal_orderings);- // TODO: If no ordering is found to be redundant during extension, we- // can use a shortcut algorithm to update the leading map.- self.oeq_cache.update_map();+ if normal_orderings.is_empty() {+ return;}+ // `add_constants` and `add_equal_conditions` discover implied orderings+ // too, and whether an ordering is satisfied must not depend on which of+ // them ran last. Deriving one fails only on an expression that does not+ // match the schema; that ordering is then not implied, which can cost a+ // sort but not a wrong result.+ let mut implied: Vec<LexOrdering> = vec![];+ for ordering in &normal_orderings {+ // Two orderings can imply the same one; expand it only once:+ for ordering in self.implied_by(ordering).unwrap_or_default() {+ if !implied.contains(&ordering) {+ implied.push(ordering);+ }+ }+ }+ self.oeq_class.extend(orderings);+ // Normalize given orderings to update the cache:+ self.oeq_cache.normal_cls.extend(normal_orderings);+ // TODO: If no ordering is found to be redundant during extension, we+ // can use a shortcut algorithm to update the leading map.+ self.oeq_cache.update_map();+ // Each implied ordering is shorter than the one implying it, so this+ // terminates:+ self.add_orderings(implied);}/// Adds a single ordering to the existing ordering equivalence class.@@ -819,15 +838,76 @@ impl EquivalenceProperties {let mut new_orderings = vec![];for idx in ordering_idxs {let ordering = &self.oeq_cache.normal_cls[*idx];- new_orderings- .extend(self.implied_ordering(&ordering[0], ordering[1..].iter())?);- }- if !new_orderings.is_empty() {- self.add_orderings(new_orderings);+ // As in `add_orderings`, an ordering that cannot be derived is+ // not implied, so the result does not depend on the call order:+ new_orderings.extend(self.implied_by(ordering).unwrap_or_default());}+ self.add_orderings(new_orderings);Ok(())}+ /// Returns the orderings that the normal ordering `ordering` implies+ /// through known equalities: for each key that is a function of the keys+ /// after it (see [`Self::implied_ordering`]), `ordering` without that key.+ ///+ /// # Example: a key that is a function of the keys after it+ ///+ /// * Ordering: `[k ASC, a ASC, c ASC]`+ /// * Equality: `a = CAST(c AS DATE)`+ /// * Returns: `[k ASC, c ASC]`+ ///+ /// Among the rows that tie on `k`, the rows are sorted by `a`, and the+ /// rows that also tie on `a` are sorted by `c`. `a` is the date of `c`, so+ /// a row with an earlier `a` also has an earlier `c`, and the rows that+ /// tie on `k` are sorted by `c` alone:+ ///+ /// ```text+ /// ┌───┬────────────┬──────────────────┐+ /// │ k │ a │ c │+ /// ├───┼────────────┼──────────────────┤+ /// │ x │ 2026-01-01 │ 2026-01-01 09:00 │+ /// │ x │ 2026-01-02 │ 2026-01-02 01:00 │+ /// │ x │ 2026-01-02 │ 2026-01-02 05:00 │+ /// │ y │ 2026-01-01 │ 2026-01-01 03:00 │+ /// └───┴────────────┴──────────────────┘+ /// ```+ ///+ /// # Example: a constant that differs between partitions+ ///+ /// * Ordering: `[k ASC, a ASC, c ASC]`, where `k` is a constant whose value+ /// differs between partitions+ /// * Equality: `a = CAST(c AS DATE)`+ /// * Returns: `[c ASC]`+ ///+ /// A normal ordering can keep such a constant (see+ /// `EquivalenceGroup::normalize_sort_exprs`). Within a partition `k` has+ /// one value, so it is left out of the keys and of the result.+ ///+ /// # Example: directions that differ+ ///+ /// * Ordering: `[a ASC, c DESC]`+ /// * Equality: `a = CAST(c AS DATE)`+ /// * Returns: nothing+ ///+ /// `c` descends within a date but ascends across dates, so the rows are+ /// not sorted by `c` in either direction.+ fn implied_by(&self, ordering: &LexOrdering) -> Result<Vec<LexOrdering>> {+ let keys: Vec<_> = ordering+ .iter()+ .filter(|sort_expr| self.eq_group.is_expr_constant(&sort_expr.expr).is_none())+ .collect();+ let mut implied = vec![];+ for (idx, key) in keys.iter().enumerate() {+ if let Some(rest) =+ self.implied_ordering(key, keys[idx + 1..].iter().copied())?+ {+ let prefix = keys[..idx].iter().copied().cloned();+ implied.extend(LexOrdering::new(prefix.chain(rest)));+ }+ }+ Ok(implied)+ }+/// Returns the ordering that the normal ordering `[leading, rest..]`/// implies through an equality on its leading expression, if any.///diff --git a/datafusion/physical-expr/src/equivalence/properties/walk.rs b/datafusion/physical-expr/src/equivalence/properties/walk.rsindex e0487cdd23..0eb27ad002 100644--- a/datafusion/physical-expr/src/equivalence/properties/walk.rs+++ b/datafusion/physical-expr/src/equivalence/properties/walk.rs@@ -43,19 +43,34 @@ pub(super) struct SortKeyWalk<'a> {/// found by [`EquivalenceProperties::implied_ordering`] as the walk goes.implied: Vec<LexOrdering>,/// For each ordering (the normal orderings of `props`, then `implied`),- /// the index of its head. Empty until the first pin, so that a walk with- /// nothing pinned allocates nothing; until then every head is at index 0.+ /// the index of its head. Empty until a head moves, so that a walk that+ /// moves no head allocates nothing; until then every head is at index 0.head_idxs: Vec<usize>,}impl<'a> SortKeyWalk<'a> {pub(super) fn new(props: &'a EquivalenceProperties) -> Self {- Self {+ let mut walk = Self {props,pinned: HashSet::new(),implied: vec![],head_idxs: vec![],+ };+ // A normal ordering keeps a constant whose value differs between+ // partitions (see `EquivalenceGroup::normalize_sort_exprs`), and such+ // a constant may lead it. Within a partition it has one value, so it+ // is fixed and the head is the first expression after it.+ let normal = &props.oeq_cache.normal_cls;+ if normal+ .iter()+ .any(|ordering| walk.is_fixed(&ordering[0].expr))+ {+ walk.head_idxs = normal+ .iter()+ .map(|ordering| walk.skip_fixed(ordering, 0))+ .collect();}+ walk}/// Returns how many leading keys (in normal form) the existing orderings@@ -153,11 +168,7 @@ impl<'a> SortKeyWalk<'a> {let mut found = vec![];for idx in first..count {let ordering = self.ordering(idx);- let mut head_idx = self.head_idxs[idx];- while head_idx < ordering.len() && self.is_fixed(&ordering[head_idx].expr)- {- head_idx += 1;- }+ let head_idx = self.skip_fixed(ordering, self.head_idxs[idx]);if let Some(head) = ordering.get(head_idx) {let rest = ordering[head_idx + 1..].iter()@@ -264,6 +275,16 @@ impl<'a> SortKeyWalk<'a> {Ok(Transformed::yes(node))}+ /// Returns the index of the head of `ordering` (see [`SortKeyWalk`]),+ /// searching from `start`, or `ordering.len()` if it has no head there.+ fn skip_fixed(&self, ordering: &LexOrdering, start: usize) -> usize {+ let mut idx = start;+ while idx < ordering.len() && self.is_fixed(&ordering[idx].expr) {+ idx += 1;+ }+ idx+ }+/// Whether `expr` is fixed (see [`SortKeyWalk`]). Callers pass `expr` in/// normal form. Pins are stored as given, so a pinned expression that is/// not in normal form is found only through its equivalence class, whose@@ -276,6 +297,12 @@ impl<'a> SortKeyWalk<'a> {if class.is_some_and(|class| class.constant.is_some()) {return true;}+ // A column that is neither pinned nor constant can have a different+ // value in each row, so it is not fixed. The leaf check below gives+ // the same answer; returning here skips it:+ if expr.is::<Column>() {+ return false;+ }// A volatile expression can change from row to row, even when all of// its inputs are fixed:if expr.is_volatile_node() {@@ -314,7 +341,7 @@ impl<'a> SortKeyWalk<'a> {mod tests {use std::sync::Arc;- use crate::expressions::{BinaryExpr, CastExpr, col, lit};+ use crate::expressions::{BinaryExpr, CastExpr, Column, col, lit};use crate::scalar_subquery::ScalarSubqueryExpr;use crate::{AcrossPartitions, ConstExpr, EquivalenceProperties};use crate::{PhysicalExpr, PhysicalSortExpr, ScalarFunctionExpr};@@ -328,6 +355,289 @@ mod tests {};use datafusion_expr::{ColumnarValue, Operator, Volatility, create_udf};+ /// With `CAST(c AS DATE) = a`, the ordering `[a, c]` implies `[c]` when+ /// `a` and `c` have the same sort options, whether the ordering or the+ /// equality comes first. Otherwise it implies nothing: with+ /// `[a DESC, c ASC]`, `c` descends across dates and ascends within one.+ #[test]+ fn implied_ordering_does_not_depend_on_call_order() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Date32, true),+ Field::new("c", DataType::Timestamp(TimeUnit::Nanosecond, None), true),+ ]));+ let [a, c] = ["a", "c"].map(|name| col(name, &schema).unwrap());+ let cast_c = Arc::new(CastExpr::new_with_target_field(+ Arc::clone(&c),+ Arc::new(Field::new("c", DataType::Date32, true)),+ None,+ )) as Arc<dyn PhysicalExpr>;+ let all_options = [(false, false), (false, true), (true, false), (true, true)]+ .map(|(descending, nulls_first)| SortOptions::new(descending, nulls_first));++ for a_options in all_options {+ for c_options in all_options {+ let ordering = [+ PhysicalSortExpr::new(Arc::clone(&a), a_options),+ PhysicalSortExpr::new(Arc::clone(&c), c_options),+ ];++ let mut equality_first = EquivalenceProperties::new(Arc::clone(&schema));+ equality_first+ .add_equal_conditions(Arc::clone(&cast_c), Arc::clone(&a))?;+ equality_first.add_ordering(ordering.clone());++ let mut ordering_first = EquivalenceProperties::new(Arc::clone(&schema));+ ordering_first.add_ordering(ordering);+ ordering_first+ .add_equal_conditions(Arc::clone(&cast_c), Arc::clone(&a))?;++ let required = [PhysicalSortExpr::new(Arc::clone(&c), c_options)];+ for properties in [equality_first, ordering_first] {+ assert_eq!(+ properties.ordering_satisfy(required.clone())?,+ a_options == c_options,+ "ordering [a {a_options:?}, c {c_options:?}]"+ );+ }+ }+ }+ Ok(())+ }++ /// A normal ordering can keep a leading per-partition constant, and the+ /// equality behind it still counts: with `k` such a constant and+ /// `CAST(c AS DATE) = a`, the ordering `[k, a, c]` implies `[c]`, whichever+ /// order the constant, the equality and the ordering are added in.+ #[test]+ fn implied_ordering_behind_a_leading_constant() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("k", DataType::Int32, true),+ Field::new("a", DataType::Date32, true),+ Field::new("c", DataType::Timestamp(TimeUnit::Nanosecond, None), true),+ ]));+ let [k, a, c] = ["k", "a", "c"].map(|name| col(name, &schema).unwrap());+ let cast_c = Arc::new(CastExpr::new_with_target_field(+ Arc::clone(&c),+ Arc::new(Field::new("c", DataType::Date32, true)),+ None,+ )) as Arc<dyn PhysicalExpr>;+ let ordering =+ [&k, &a, &c].map(|expr| PhysicalSortExpr::new_default(Arc::clone(expr)));+ let constant = || ConstExpr::new(Arc::clone(&k), AcrossPartitions::Heterogeneous);++ let mut constant_equality_ordering =+ EquivalenceProperties::new(Arc::clone(&schema));+ constant_equality_ordering.add_constants([constant()])?;+ constant_equality_ordering+ .add_equal_conditions(Arc::clone(&cast_c), Arc::clone(&a))?;+ constant_equality_ordering.add_ordering(ordering.clone());++ let mut constant_ordering_equality =+ EquivalenceProperties::new(Arc::clone(&schema));+ constant_ordering_equality.add_constants([constant()])?;+ constant_ordering_equality.add_ordering(ordering.clone());+ constant_ordering_equality+ .add_equal_conditions(Arc::clone(&cast_c), Arc::clone(&a))?;++ let mut equality_ordering_constant =+ EquivalenceProperties::new(Arc::clone(&schema));+ equality_ordering_constant.add_equal_conditions(cast_c, Arc::clone(&a))?;+ equality_ordering_constant.add_ordering(ordering);+ equality_ordering_constant.add_constants([constant()])?;++ for properties in [+ constant_equality_ordering,+ constant_ordering_equality,+ equality_ordering_constant,+ ] {+ assert!(+ properties+ .ordering_satisfy([PhysicalSortExpr::new_default(Arc::clone(&c))])?+ );+ }+ Ok(())+ }++ /// An implied ordering can imply another: with `CAST(c AS DATE) = a` and+ /// `CAST(a AS DATE64) = d`, the ordering `[d, a, c]` implies `[a, c]`,+ /// which implies `[c]`.+ #[test]+ fn implied_orderings_chain() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("d", DataType::Date64, true),+ Field::new("a", DataType::Date32, true),+ Field::new("c", DataType::Timestamp(TimeUnit::Nanosecond, None), true),+ ]));+ let [d, a, c] = ["d", "a", "c"].map(|name| col(name, &schema).unwrap());+ let cast = |expr: &Arc<dyn PhysicalExpr>, data_type| {+ Arc::new(CastExpr::new_with_target_field(+ Arc::clone(expr),+ Arc::new(Field::new("cast", data_type, true)),+ None,+ )) as Arc<dyn PhysicalExpr>+ };++ let mut properties = EquivalenceProperties::new(Arc::clone(&schema));+ properties.add_equal_conditions(Arc::clone(&a), cast(&c, DataType::Date32))?;+ properties.add_equal_conditions(cast(&a, DataType::Date64), Arc::clone(&d))?;+ properties.add_ordering(+ [&d, &a, &c].map(|expr| PhysicalSortExpr::new_default(Arc::clone(expr))),+ );++ assert!(+ properties+ .ordering_satisfy([PhysicalSortExpr::new_default(Arc::clone(&c))])?+ );+ Ok(())+ }++ /// A key that is a function of the keys after it implies the ordering+ /// without it, after any prefix: with `CAST(c AS DATE) = a`, the ordering+ /// `[k, a, c]` implies `[k, c]`, whichever is added first. The implied+ /// ordering is stored, so it holds after `a` is gone.+ #[test]+ fn implied_ordering_after_a_prefix() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("k", DataType::Utf8, true),+ Field::new("a", DataType::Date32, true),+ Field::new("c", DataType::Timestamp(TimeUnit::Nanosecond, None), true),+ ]));+ let [k, a, c] = ["k", "a", "c"].map(|name| col(name, &schema).unwrap());+ let cast_c = Arc::new(CastExpr::new_with_target_field(+ Arc::clone(&c),+ Arc::new(Field::new("c", DataType::Date32, true)),+ None,+ )) as Arc<dyn PhysicalExpr>;+ let ordering =+ [&k, &a, &c].map(|expr| PhysicalSortExpr::new_default(Arc::clone(expr)));+ let implied =+ [&k, &c].map(|expr| PhysicalSortExpr::new_default(Arc::clone(expr)));++ let mut equality_first = EquivalenceProperties::new(Arc::clone(&schema));+ equality_first.add_equal_conditions(Arc::clone(&cast_c), Arc::clone(&a))?;+ equality_first.add_ordering(ordering.clone());++ let mut ordering_first = EquivalenceProperties::new(Arc::clone(&schema));+ ordering_first.add_ordering(ordering);+ ordering_first.add_equal_conditions(cast_c, Arc::clone(&a))?;++ for properties in [equality_first, ordering_first] {+ assert!(+ properties+ .oeq_class()+ .iter()+ .any(|ordering| ordering.iter().eq(implied.iter()))+ );+ }+ Ok(())+ }++ /// A constant between a key and its arguments doesn't hide the equality:+ /// with `k` a per-partition constant and `CAST(c AS DATE) = a`, the+ /// ordering `[a, k, c]` implies `[c]`, because within a partition `k` has+ /// one value. The constant is added first, so the normal ordering keeps it.+ #[test]+ fn implied_ordering_past_a_constant_between_keys() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("k", DataType::Int32, true),+ Field::new("a", DataType::Date32, true),+ Field::new("c", DataType::Timestamp(TimeUnit::Nanosecond, None), true),+ ]));+ let [k, a, c] = ["k", "a", "c"].map(|name| col(name, &schema).unwrap());+ let cast_c = Arc::new(CastExpr::new_with_target_field(+ Arc::clone(&c),+ Arc::new(Field::new("c", DataType::Date32, true)),+ None,+ )) as Arc<dyn PhysicalExpr>;+ let ordering =+ [&a, &k, &c].map(|expr| PhysicalSortExpr::new_default(Arc::clone(expr)));+ let constant = || ConstExpr::new(Arc::clone(&k), AcrossPartitions::Heterogeneous);++ let mut equality_then_ordering = EquivalenceProperties::new(Arc::clone(&schema));+ equality_then_ordering.add_constants([constant()])?;+ equality_then_ordering+ .add_equal_conditions(Arc::clone(&cast_c), Arc::clone(&a))?;+ equality_then_ordering.add_ordering(ordering.clone());++ let mut ordering_then_equality = EquivalenceProperties::new(Arc::clone(&schema));+ ordering_then_equality.add_constants([constant()])?;+ ordering_then_equality.add_ordering(ordering);+ ordering_then_equality.add_equal_conditions(cast_c, Arc::clone(&a))?;++ for properties in [equality_then_ordering, ordering_then_equality] {+ assert!(+ properties+ .ordering_satisfy([PhysicalSortExpr::new_default(Arc::clone(&c))])?+ );+ }+ Ok(())+ }++ /// An implied ordering that cannot be derived is left out, and the given+ /// ordering is kept, whether the ordering or the equality comes first:+ /// here `c` is not in the schema, so its type is unknown.+ #[test]+ fn underivable_implied_ordering_is_skipped_in_either_order() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Date32, true)]));+ let a = col("a", &schema)?;+ let c = Arc::new(Column::new("c", 1)) as Arc<dyn PhysicalExpr>;+ let cast_c = Arc::new(CastExpr::new_with_target_field(+ Arc::clone(&c),+ Arc::new(Field::new("c", DataType::Date32, true)),+ None,+ )) as Arc<dyn PhysicalExpr>;++ let ordering =+ [&a, &c].map(|expr| PhysicalSortExpr::new_default(Arc::clone(expr)));++ let mut equality_first = EquivalenceProperties::new(Arc::clone(&schema));+ equality_first.add_equal_conditions(Arc::clone(&cast_c), Arc::clone(&a))?;+ equality_first.add_ordering(ordering.clone());++ let mut ordering_first = EquivalenceProperties::new(Arc::clone(&schema));+ ordering_first.add_ordering(ordering);+ ordering_first.add_equal_conditions(cast_c, Arc::clone(&a))?;++ for properties in [equality_first, ordering_first] {+ assert_eq!(properties.oeq_class().len(), 1);+ }+ Ok(())+ }++ /// A normal ordering keeps a constant whose value differs between+ /// partitions (`AcrossPartitions::Heterogeneous`), and such a constant+ /// can lead it. Within a partition the constant has one value, so it is+ /// fixed and the head is the expression after it. With `k` such a+ /// constant and ordering `[k, b]`, the requirements `[b]` and `[b, k]`+ /// hold before anything is pinned.+ /// See <https://github.com/apache/datafusion/issues/26762>.+ #[test]+ fn leading_per_partition_constant_is_not_a_head() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("k", DataType::Int32, true),+ Field::new("b", DataType::Int32, true),+ ]));+ let [k, b] = ["k", "b"].map(|name| col(name, &schema).unwrap());++ let mut properties = EquivalenceProperties::new(Arc::clone(&schema));+ properties.add_constants([ConstExpr::new(+ Arc::clone(&k),+ AcrossPartitions::Heterogeneous,+ )])?;+ // Added after the constant, so that `k` stays in the normal ordering:+ properties.add_ordering(+ [&k, &b].map(|expr| PhysicalSortExpr::new_default(Arc::clone(expr))),+ );++ for required in [vec![&b], vec![&b, &k], vec![&k, &b]] {+ let required = required+ .into_iter()+ .map(|expr| PhysicalSortExpr::new_default(Arc::clone(expr)));+ assert!(properties.ordering_satisfy(required)?);+ }+ Ok(())+ }+/// Pinning a key can move the head of an ordering to an expression that/// is equal to a function of the expressions after it, which implies a/// new ordering. With ordering `[k, a, c]` and `CAST(c AS DATE) = a`,diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rsindex 985ec3632d..4ca469f307 100644--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs@@ -1289,10 +1289,34 @@ fn handle_projection_pushdown(// (expressed in the projection's output schema) must be remapped into the// child schema before being pushed down. Forwarding `score@1` unchanged// could refer to a different input column and sort on the wrong key.- Ok(+ let Some(requirements) =remap_requirement_through_projection(projection_exec, parent_required)- .map(|requirements| vec![Some(requirements)]),- )+ else {+ return Ok(None);+ };+ // If the input already has this ordering, no sort will be added below the+ // projection. The parent still needs the projection to prove it, which it+ // may not be able to do: the proof can depend on a column that it drops,+ // such as a leading per-partition constant. Keep the sort above the+ // projection in that case. Alternatives that cannot be remapped are not+ // pushed down, so compare against the first one that can.+ let pushed = parent_required.iter().find_map(|required| {+ remap_lex_requirement_through_projection(projection_exec.expr(), required)+ .map(|remapped| (required, remapped))+ });+ if let Some((required, remapped)) = pushed+ && projection_exec+ .input()+ .equivalence_properties()+ .ordering_satisfy_requirement(remapped)?+ && !projection_exec+ .properties()+ .equivalence_properties()+ .ordering_satisfy_requirement(required.clone())?+ {+ return Ok(None);+ }+ Ok(Some(vec![Some(requirements)]))}#[cfg(test)]@@ -1306,7 +1330,7 @@ mod tests {ScalarSubqueryResults, SubqueryIndex,};use datafusion_physical_expr::PhysicalExpr;- use datafusion_physical_expr::expressions::{BinaryExpr, col};+ use datafusion_physical_expr::expressions::{BinaryExpr, col, lit};use datafusion_physical_plan::empty::EmptyExec;use datafusion_physical_plan::limit::GlobalLimitExec;use datafusion_physical_plan::scalar_subquery::{@@ -1490,6 +1514,58 @@ mod tests {assert!(matches!(remapped, OrderingRequirements::Soft(_)));}+ /// The guard keeps a sort above a projection when the input already has+ /// the ordering that would be pushed down, but the projection cannot show+ /// that ordering to its parent. When the parent accepts several+ /// orderings, the guard must check the one that is pushed down, not the+ /// first one.+ ///+ /// * Input: sorted by `[c DESC]`; the projection outputs `c` as `score`+ /// * Parent requires: `[score % 2 ASC]` or `[score DESC]`+ /// * Expected: `[c DESC]` is pushed down+ ///+ /// `score % 2` is not a column, so `[score % 2 ASC]` cannot be rewritten+ /// in terms of the input and is dropped; `[score DESC]` becomes+ /// `[c DESC]`. The projection does provide `[score DESC]`, so the guard+ /// does not apply. Checking the first alternative, `[score % 2 ASC]`,+ /// which the projection does not provide, would keep an unneeded sort+ /// above the projection.+ #[test]+ fn projection_guard_compares_the_pushed_alternative() -> Result<()> {+ let child = child_schema();+ let empty: Arc<dyn ExecutionPlan> = Arc::new(EmptyExec::new(Arc::clone(&child)));+ let sorted: Arc<dyn ExecutionPlan> = Arc::new(SortExec::new(+ [PhysicalSortExpr::new(col("c", &child)?, DESC)].into(),+ empty,+ ));+ let projection = ProjectionExec::try_new(+ vec![+ (col("a", &child)?, "a".to_string()),+ (col("c", &child)?, "score".to_string()),+ (col("b", &child)?, "value".to_string()),+ ],+ sorted,+ )?;+ let out = projection.schema();++ let computed: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(+ col("score", &out)?,+ Operator::Modulo,+ lit(2),+ ));+ let mut required =+ OrderingRequirements::new(lex([PhysicalSortRequirement::new(+ computed,+ Some(ASC),+ )]));+ required.add_alternative(lex([req("score", &out, DESC)]));++ let pushed = handle_projection_pushdown(&projection, &required)?;+ let expected = OrderingRequirements::new(lex([req("c", &child, DESC)]));+ assert_eq!(pushed, Some(vec![Some(expected)]));+ Ok(())+ }+#[test]fn remap_preserves_all_hard_alternatives() {let projection = reordering_projection();diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rsindex 13274d26e1..95f26bd2b8 100644--- a/datafusion/physical-plan/src/repartition/mod.rs+++ b/datafusion/physical-plan/src/repartition/mod.rs@@ -2206,11 +2206,13 @@ impl RepartitionExec {// and per-partition constants no longer hold.if input.output_partitioning().partition_count() > 1 {eq_properties.clear_orderings();+ eq_properties.clear_groupings();+ // Clear the constants first: an ordering added while they hold+ // implies orderings that hold only within each input partition.+ eq_properties.clear_per_partition_constants();if preserve_order && let Some(ordering) = input.output_ordering() {eq_properties.add_ordering(ordering.clone());}- eq_properties.clear_groupings();- eq_properties.clear_per_partition_constants();}eq_properties}@@ -4942,6 +4944,105 @@ mod test {Ok(())}+ /// Merging partitions keeps only the merge key ordering. Here each input+ /// partition has its own constant `k`, `a = CAST(c AS DATE)`, and is sorted+ /// by `[k, a, c]`, so within a partition the rows are also sorted by `[c]`.+ /// Partition `k = 0` has later dates than partition `k = 1`, so the merged+ /// rows are not sorted by `[c]`.+ #[tokio::test]+ async fn test_preserve_order_drops_per_partition_implied_ordering() -> Result<()> {+ use crate::filter::FilterExec;+ use crate::sorts::sort::SortExec;+ use arrow::array::{Date32Array, TimestampSecondArray};+ use arrow::datatypes::TimeUnit;+ use datafusion_expr::Operator;+ use datafusion_physical_expr::expressions::{BinaryExpr, CastExpr, lit};++ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Date32, true),+ Field::new("c", DataType::Timestamp(TimeUnit::Second, None), true),+ ]));+ let partition = |k: i32, days: [i32; 2]| -> Result<Arc<dyn ExecutionPlan>> {+ // One o'clock on each day:+ let times = days.map(|day| i64::from(day) * 86_400 + 3_600);+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(Date32Array::from(days.to_vec())),+ Arc::new(TimestampSecondArray::from(times.to_vec())),+ ],+ )?;+ let source =+ TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?;+ Ok(Arc::new(ProjectionExec::try_new(+ vec![+ (lit(ScalarValue::Int32(Some(k))), "k".to_string()),+ (col("a", &schema)?, "a".to_string()),+ (col("c", &schema)?, "c".to_string()),+ ],+ source,+ )?))+ };+ let union =+ UnionExec::try_new(vec![partition(0, [10, 11])?, partition(1, [1, 2])?])?;+ let union_schema = union.schema();+ let cast_c = Arc::new(CastExpr::new_with_target_field(+ col("c", &union_schema)?,+ Arc::new(Field::new("c", DataType::Date32, true)),+ None,+ ));+ let filter = Arc::new(FilterExec::try_new(+ Arc::new(BinaryExpr::new(+ col("a", &union_schema)?,+ Operator::Eq,+ cast_c,+ )),+ union,+ )?);+ let ordering = |names: &[&str]| {+ names+ .iter()+ .map(|name| Ok(PhysicalSortExpr::new_default(col(name, &union_schema)?)))+ .collect::<Result<Vec<_>>>()+ };+ let sort = Arc::new(+ SortExec::new(+ LexOrdering::new(ordering(&["k", "a", "c"])?).unwrap(),+ filter,+ )+ .with_preserve_partitioning(true),+ );+ assert!(+ sort.properties()+ .equivalence_properties()+ .ordering_satisfy(ordering(&["c"])?)?+ );++ let exec = Arc::new(+ RepartitionExec::try_new(sort, Partitioning::RoundRobinBatch(1))?+ .with_preserve_order(),+ );+ let properties = exec.properties().equivalence_properties();+ assert!(properties.ordering_satisfy(ordering(&["k", "a", "c"])?)?);+ assert!(!properties.ordering_satisfy(ordering(&["c"])?)?);++ let batches = crate::collect(exec, Arc::new(TaskContext::default())).await?;+ assert_batches_eq!(+ [+ "+---+------------+---------------------+",+ "| k | a | c |",+ "+---+------------+---------------------+",+ "| 0 | 1970-01-11 | 1970-01-11T01:00:00 |",+ "| 0 | 1970-01-12 | 1970-01-12T01:00:00 |",+ "| 1 | 1970-01-02 | 1970-01-02T01:00:00 |",+ "| 1 | 1970-01-03 | 1970-01-03T01:00:00 |",+ "+---+------------+---------------------+",+ ],+ &batches+ );+ Ok(())+ }+#[tokio::test]async fn test_preserve_order_with_spilling() -> Result<()> {use datafusion_execution::runtime_env::RuntimeEnvBuilder;diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.sltindex 8508ca6ff6..2a9e3084c1 100644--- a/datafusion/sqllogictest/test_files/window.slt+++ b/datafusion/sqllogictest/test_files/window.slt@@ -7834,3 +7834,299 @@ DROP TABLE range_null_peers;statement okDROP TABLE range_nan_peers;++# A window ordered by `(total DESC, k)` over an input sorted by `[k, total]`,+# where `k` is constant within each partition (`lower(g)` under `WHERE g = 'a'`).+# Within a partition `k` has one value, so the input ordering satisfies the+# window's requirement and no sort is added.+# https://github.com/apache/datafusion/issues/26762+statement ok+set datafusion.execution.target_partitions = 1;++statement ok+CREATE TABLE per_partition_constant(g VARCHAR, v BIGINT) AS VALUES ('a', 1), ('b', 2);++query TI+SELECT k, DENSE_RANK() OVER (ORDER BY total DESC, k) AS r+FROM (+ SELECT k, SUM(s) OVER (PARTITION BY k) AS total+ FROM (SELECT lower(g) AS k, SUM(v) AS s FROM per_partition_constant WHERE g = 'a' GROUP BY lower(g))+);+----+a 1++statement ok+set datafusion.explain.physical_plan_only = true;++query TT+EXPLAIN SELECT k, DENSE_RANK() OVER (ORDER BY total DESC, k) AS r+FROM (+ SELECT k, SUM(s) OVER (PARTITION BY k) AS total+ FROM (SELECT lower(g) AS k, SUM(v) AS s FROM per_partition_constant WHERE g = 'a' GROUP BY lower(g))+);+----+physical_plan+01)ProjectionExec: expr=[k@0 as k, dense_rank() ORDER BY [total DESC NULLS FIRST, k ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as r]+02)--BoundedWindowAggExec: wdw=[dense_rank() ORDER BY [total DESC NULLS FIRST, k ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "dense_rank() ORDER BY [total DESC NULLS FIRST, k ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]+03)----ProjectionExec: expr=[k@0 as k, sum(s) PARTITION BY [k] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 as total]+04)------WindowAggExec: wdw=[sum(s) PARTITION BY [k] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(s) PARTITION BY [k] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }]+05)--------ProjectionExec: expr=[lower(per_partition_constant.g)@0 as k, sum(per_partition_constant.v)@1 as s]+06)----------AggregateExec: mode=Single, gby=[lower(g@0) as lower(per_partition_constant.g)], aggr=[sum(per_partition_constant.v)]+07)------------FilterExec: g@0 = a+08)--------------DataSourceExec: partitions=1, partition_sizes=[1]++statement ok+reset datafusion.explain.physical_plan_only;++# Same, with `k` a per-partition constant that is not derived from a filter+# (one row per partition), at more than one target partition.+statement ok+set datafusion.execution.target_partitions = 4;++query IIII+SELECT k, c, total, DENSE_RANK() OVER (ORDER BY total DESC, k) AS r+FROM (+ SELECT k, c, SUM(c) OVER (PARTITION BY k) AS total+ FROM (SELECT SUM(v) AS k, COUNT(*) AS c FROM per_partition_constant)+);+----+3 2 2 1++statement ok+set datafusion.explain.physical_plan_only = true;++query TT+EXPLAIN SELECT k, c, total, DENSE_RANK() OVER (ORDER BY total DESC, k) AS r+FROM (+ SELECT k, c, SUM(c) OVER (PARTITION BY k) AS total+ FROM (SELECT SUM(v) AS k, COUNT(*) AS c FROM per_partition_constant)+);+----+physical_plan+01)ProjectionExec: expr=[k@0 as k, c@1 as c, total@2 as total, dense_rank() ORDER BY [total DESC NULLS FIRST, k ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as r]+02)--BoundedWindowAggExec: wdw=[dense_rank() ORDER BY [total DESC NULLS FIRST, k ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "dense_rank() ORDER BY [total DESC NULLS FIRST, k ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]+03)----ProjectionExec: expr=[k@0 as k, c@1 as c, sum(c) PARTITION BY [k] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 as total]+04)------WindowAggExec: wdw=[sum(c) PARTITION BY [k] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(c) PARTITION BY [k] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }]+05)--------ProjectionExec: expr=[sum(per_partition_constant.v)@0 as k, count(Int64(1))@1 as c]+06)----------AggregateExec: mode=Single, gby=[], aggr=[sum(per_partition_constant.v), count(Int64(1))]+07)------------DataSourceExec: partitions=1, partition_sizes=[1]++statement ok+reset datafusion.explain.physical_plan_only;++statement ok+set datafusion.execution.target_partitions = 1;++# When a projection drops the leading constant, it can no longer prove the+# ordering that its input has. The input satisfies `[total]`, because within a+# partition `k` has one value, so a sort pushed below the projection would be+# removed, and `SanityCheckPlan` would reject the window. The sort stays above+# the projection.+query II+SELECT total, DENSE_RANK() OVER (ORDER BY total DESC) AS r+FROM (+ SELECT SUM(s) OVER (PARTITION BY k) AS total+ FROM (SELECT lower(g) AS k, SUM(v) AS s FROM per_partition_constant WHERE g = 'a' GROUP BY lower(g))+);+----+1 1++statement ok+set datafusion.explain.physical_plan_only = true;++query TT+EXPLAIN SELECT total, DENSE_RANK() OVER (ORDER BY total DESC) AS r+FROM (+ SELECT SUM(s) OVER (PARTITION BY k) AS total+ FROM (SELECT lower(g) AS k, SUM(v) AS s FROM per_partition_constant WHERE g = 'a' GROUP BY lower(g))+);+----+physical_plan+01)ProjectionExec: expr=[total@0 as total, dense_rank() ORDER BY [total DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 as r]+02)--BoundedWindowAggExec: wdw=[dense_rank() ORDER BY [total DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "dense_rank() ORDER BY [total DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]+03)----SortExec: expr=[total@0 DESC], preserve_partitioning=[false]+04)------ProjectionExec: expr=[sum(s) PARTITION BY [k] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 as total]+05)--------WindowAggExec: wdw=[sum(s) PARTITION BY [k] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(s) PARTITION BY [k] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }]+06)----------ProjectionExec: expr=[lower(per_partition_constant.g)@0 as k, sum(per_partition_constant.v)@1 as s]+07)------------AggregateExec: mode=Single, gby=[lower(g@0) as lower(per_partition_constant.g)], aggr=[sum(per_partition_constant.v)]+08)--------------FilterExec: g@0 = a+09)----------------DataSourceExec: partitions=1, partition_sizes=[1]++statement ok+reset datafusion.explain.physical_plan_only;++# A key that is a function of the keys after it can be left out, after any+# prefix: `[k, a, c]` with `a = CAST(c AS DATE)` implies `[k, c]`. That ordering+# is stored, so a projection that drops `a` keeps it, and neither the window nor+# an `ORDER BY k, c` above the projection needs a sort.+statement ok+CREATE TABLE ordered_through_cast(k VARCHAR, a DATE, c TIMESTAMP) AS VALUES+ ('a', DATE '2026-01-02', TIMESTAMP '2026-01-02 05:00:00'),+ ('a', DATE '2026-01-01', TIMESTAMP '2026-01-01 09:00:00'),+ ('a', DATE '2026-01-02', TIMESTAMP '2026-01-02 01:00:00'),+ ('b', DATE '2026-01-03', TIMESTAMP '2026-01-03 01:00:00');++query TPI+SELECT k, c, ROW_NUMBER() OVER (ORDER BY k, c) AS r+FROM (SELECT k, a, c FROM ordered_through_cast WHERE a = CAST(c AS DATE) ORDER BY k, a, c LIMIT 10)+ORDER BY r;+----+a 2026-01-01T09:00:00 1+a 2026-01-02T01:00:00 2+a 2026-01-02T05:00:00 3+b 2026-01-03T01:00:00 4++statement ok+set datafusion.explain.physical_plan_only = true;++query TT+EXPLAIN SELECT k, c, ROW_NUMBER() OVER (ORDER BY k, c) AS r+FROM (SELECT k, a, c FROM ordered_through_cast WHERE a = CAST(c AS DATE) ORDER BY k, a, c LIMIT 10)+ORDER BY r;+----+physical_plan+01)ProjectionExec: expr=[k@0 as k, c@1 as c, row_number() ORDER BY [ordered_through_cast.k ASC NULLS LAST, ordered_through_cast.c ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as r]+02)--BoundedWindowAggExec: wdw=[row_number() ORDER BY [ordered_through_cast.k ASC NULLS LAST, ordered_through_cast.c ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [ordered_through_cast.k ASC NULLS LAST, ordered_through_cast.c ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]+03)----ProjectionExec: expr=[k@0 as k, c@2 as c]+04)------SortExec: TopK(fetch=10), expr=[k@0 ASC NULLS LAST, a@1 ASC NULLS LAST, c@2 ASC NULLS LAST], preserve_partitioning=[false]+05)--------FilterExec: a@1 = CAST(c@2 AS Date32)+06)----------DataSourceExec: partitions=1, partition_sizes=[1]++query TT+EXPLAIN SELECT k, c+FROM (SELECT k, a, c FROM ordered_through_cast WHERE a = CAST(c AS DATE) ORDER BY k, a, c LIMIT 10)+ORDER BY k, c;+----+physical_plan+01)ProjectionExec: expr=[k@0 as k, c@2 as c]+02)--SortExec: TopK(fetch=10), expr=[k@0 ASC NULLS LAST, a@1 ASC NULLS LAST, c@2 ASC NULLS LAST], preserve_partitioning=[false]+03)----FilterExec: a@1 = CAST(c@2 AS Date32)+04)------DataSourceExec: partitions=1, partition_sizes=[1]++statement ok+reset datafusion.explain.physical_plan_only;++# An ordering that a sort adds implies other orderings through known equalities,+# as it does when the equality comes after the sort: `[a, c]` with+# `a = CAST(c AS DATE)` has `[c]`, so the window needs no sort of its own.+query DPI+SELECT a, c, ROW_NUMBER() OVER (ORDER BY c) AS r+FROM (SELECT a, c FROM ordered_through_cast WHERE a = CAST(c AS DATE) ORDER BY a, c LIMIT 10)+ORDER BY r;+----+2026-01-01 2026-01-01T09:00:00 1+2026-01-02 2026-01-02T01:00:00 2+2026-01-02 2026-01-02T05:00:00 3+2026-01-03 2026-01-03T01:00:00 4++statement ok+set datafusion.explain.physical_plan_only = true;++query TT+EXPLAIN SELECT a, c, ROW_NUMBER() OVER (ORDER BY c) AS r+FROM (SELECT a, c FROM ordered_through_cast WHERE a = CAST(c AS DATE) ORDER BY a, c LIMIT 10)+ORDER BY r;+----+physical_plan+01)ProjectionExec: expr=[a@0 as a, c@1 as c, row_number() ORDER BY [ordered_through_cast.c ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as r]+02)--BoundedWindowAggExec: wdw=[row_number() ORDER BY [ordered_through_cast.c ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [ordered_through_cast.c ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]+03)----SortExec: TopK(fetch=10), expr=[a@0 ASC NULLS LAST, c@1 ASC NULLS LAST], preserve_partitioning=[false]+04)------FilterExec: a@0 = CAST(c@1 AS Date32)+05)--------DataSourceExec: partitions=1, partition_sizes=[1]++# The same with a constant leading the sort. `lower(k)` under `k = 'a'` has the+# same value in every partition, so the TopK leaves it out of its sort keys.+query TDPI+SELECT k, a, c, ROW_NUMBER() OVER (ORDER BY c) AS r+FROM (SELECT lower(k) AS k, a, c FROM ordered_through_cast WHERE k = 'a' AND a = CAST(c AS DATE) ORDER BY k, a, c LIMIT 10)+ORDER BY r;+----+a 2026-01-01 2026-01-01T09:00:00 1+a 2026-01-02 2026-01-02T01:00:00 2+a 2026-01-02 2026-01-02T05:00:00 3++query TT+EXPLAIN SELECT k, a, c, ROW_NUMBER() OVER (ORDER BY c) AS r+FROM (SELECT lower(k) AS k, a, c FROM ordered_through_cast WHERE k = 'a' AND a = CAST(c AS DATE) ORDER BY k, a, c LIMIT 10)+ORDER BY r;+----+physical_plan+01)ProjectionExec: expr=[k@0 as k, a@1 as a, c@2 as c, row_number() ORDER BY [ordered_through_cast.c ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as r]+02)--BoundedWindowAggExec: wdw=[row_number() ORDER BY [ordered_through_cast.c ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [ordered_through_cast.c ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]+03)----SortExec: TopK(fetch=10), expr=[a@1 ASC NULLS LAST, c@2 ASC NULLS LAST], preserve_partitioning=[false]+04)------ProjectionExec: expr=[lower(k@0) as k, a@1 as a, c@2 as c]+05)--------FilterExec: k@0 = a AND a@1 = CAST(c@2 AS Date32)+06)----------DataSourceExec: partitions=1, partition_sizes=[1]++# The same when a projection drops `a`: the implied `[c]` is stored, so the+# projection can prove it and no sort is added above the projection.+query TPI+SELECT k, c, ROW_NUMBER() OVER (ORDER BY c) AS r+FROM (SELECT lower(k) AS k, a, c FROM ordered_through_cast WHERE k = 'a' AND a = CAST(c AS DATE) ORDER BY k, a, c LIMIT 10)+ORDER BY r;+----+a 2026-01-01T09:00:00 1+a 2026-01-02T01:00:00 2+a 2026-01-02T05:00:00 3++query TT+EXPLAIN SELECT k, c, ROW_NUMBER() OVER (ORDER BY c) AS r+FROM (SELECT lower(k) AS k, a, c FROM ordered_through_cast WHERE k = 'a' AND a = CAST(c AS DATE) ORDER BY k, a, c LIMIT 10)+ORDER BY r;+----+physical_plan+01)ProjectionExec: expr=[k@0 as k, c@1 as c, row_number() ORDER BY [ordered_through_cast.c ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as r]+02)--BoundedWindowAggExec: wdw=[row_number() ORDER BY [ordered_through_cast.c ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [ordered_through_cast.c ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]+03)----ProjectionExec: expr=[k@0 as k, c@2 as c]+04)------SortExec: TopK(fetch=10), expr=[a@1 ASC NULLS LAST, c@2 ASC NULLS LAST], preserve_partitioning=[false]+05)--------ProjectionExec: expr=[lower(k@0) as k, a@1 as a, c@2 as c]+06)----------FilterExec: k@0 = a AND a@1 = CAST(c@2 AS Date32)+07)------------DataSourceExec: partitions=1, partition_sizes=[1]++# And when a merge of partitions is added above the sort, which keeps only its+# merge key. `lower(k)` is computed, so the input is repartitioned and needs the+# merge.+statement ok+set datafusion.execution.target_partitions = 4;++query TDPI+SELECT l, a, c, ROW_NUMBER() OVER (ORDER BY c) AS r+FROM (SELECT lower(k) AS l, a, c FROM ordered_through_cast WHERE a = CAST(c AS DATE) ORDER BY a, c LIMIT 10)+ORDER BY r;+----+a 2026-01-01 2026-01-01T09:00:00 1+a 2026-01-02 2026-01-02T01:00:00 2+a 2026-01-02 2026-01-02T05:00:00 3+b 2026-01-03 2026-01-03T01:00:00 4++query TT+EXPLAIN SELECT l, a, c, ROW_NUMBER() OVER (ORDER BY c) AS r+FROM (SELECT lower(k) AS l, a, c FROM ordered_through_cast WHERE a = CAST(c AS DATE) ORDER BY a, c LIMIT 10)+ORDER BY r;+----+physical_plan+01)ProjectionExec: expr=[l@0 as l, a@1 as a, c@2 as c, row_number() ORDER BY [ordered_through_cast.c ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as r]+02)--BoundedWindowAggExec: wdw=[row_number() ORDER BY [ordered_through_cast.c ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [ordered_through_cast.c ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]+03)----SortPreservingMergeExec: [a@1 ASC NULLS LAST, c@2 ASC NULLS LAST], fetch=10+04)------SortExec: TopK(fetch=10), expr=[a@1 ASC NULLS LAST, c@2 ASC NULLS LAST], preserve_partitioning=[true]+05)--------ProjectionExec: expr=[lower(k@0) as l, a@1 as a, c@2 as c]+06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+07)------------FilterExec: a@1 = CAST(c@2 AS Date32)+08)--------------DataSourceExec: partitions=1, partition_sizes=[1]++statement ok+reset datafusion.explain.physical_plan_only;++statement ok+set datafusion.execution.target_partitions = 1;++statement ok+DROP TABLE ordered_through_cast;++statement ok+DROP TABLE per_partition_constant;++# The SLT runner sets `target_partitions` to 4 instead of using the default, so+# restore that value rather than resetting.+statement ok+set datafusion.execution.target_partitions = 4;