Apache DataFusion SQL Query Engine
fix: preserve sorts before limits when projections hide sort keys (#26763)
Kumar Ujjawal16c1f26e6ffe
diff --git a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs b/datafusion/core/tests/physical_optimizer/enforce_sorting.rsindex 084ae59a76..45dabaac90 100644--- a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs+++ b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs@@ -43,7 +43,7 @@ use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr, PhysicalSortRequirement, OrderingRequirements};use datafusion_physical_expr::{Distribution, Partitioning, PhysicalExpr};-use datafusion_physical_expr::expressions::{col, BinaryExpr, Column, NotExpr};+use datafusion_physical_expr::expressions::{col, is_not_null, BinaryExpr, Column, NotExpr};use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};use datafusion_physical_plan::repartition::RepartitionExec;@@ -359,6 +359,150 @@ async fn test_do_not_remove_sort_with_limit() -> Result<()> {Ok(())}+#[test]+fn test_do_not_remove_sort_with_limit_after_projection() -> Result<()> {+ let schema = create_test_schema()?;+ let source = parquet_exec(Arc::clone(&schema));+ let sort = sort_exec([sort_expr("non_nullable_col", &schema)].into(), source);+ // Dropping the sort key hides the output ordering, but preserves the row+ // sequence that determines which rows a limit skips or returns.+ let projection = projection_exec(+ vec![(col("nullable_col", &schema)?, "nullable_col".to_string())],+ sort,+ )?;+ assert!(projection.output_ordering().is_none());++ let mut config = ConfigOptions::new();+ config.execution.target_partitions = 1;+ let filter = filter_exec(+ is_not_null(col("nullable_col", &projection.schema())?)?,+ Arc::clone(&projection),+ );+ for input in [projection, filter] {+ for plan in [+ global_limit_exec(Arc::clone(&input), 1, None),+ global_limit_exec(Arc::clone(&input), 1, Some(1)),+ global_limit_exec(Arc::clone(&input), 0, Some(1)),+ local_limit_exec(input, 1),+ ] {+ let optimized =+ EnsureRequirements::new().optimize(Arc::clone(&plan), &config)?;+ let optimized = SanityCheckPlan::new().optimize(optimized, &config)?;+ assert_eq!(get_plan_string(&optimized), get_plan_string(&plan));+ }+ }+ Ok(())+}++#[test]+fn test_required_sort_after_projection() -> Result<()> {+ let schema = create_test_schema()?;+ let source = parquet_exec(Arc::clone(&schema));+ let sort = sort_exec([sort_expr("non_nullable_col", &schema)].into(), source);+ let projection = projection_exec(+ vec![(col("nullable_col", &schema)?, "nullable_col".to_string())],+ sort,+ )?;+ let filter = filter_exec(+ is_not_null(col("nullable_col", &projection.schema())?)?,+ projection,+ );+ let mut config = ConfigOptions::new();+ config.execution.target_partitions = 1;++ for with_limit in [false, true] {+ let input = if with_limit {+ global_limit_exec(Arc::clone(&filter), 1, None)+ } else {+ Arc::clone(&filter)+ };+ let window = bounded_window_exec(+ "nullable_col",+ [sort_expr("nullable_col", &input.schema())],+ input,+ );+ let optimized = EnsureRequirements::new().optimize(window, &config)?;+ let optimized = SanityCheckPlan::new().optimize(optimized, &config)?;+ let twice =+ EnsureRequirements::new().optimize(Arc::clone(&optimized), &config)?;+ assert_eq!(get_plan_string(&twice), get_plan_string(&optimized));++ if with_limit {+ // The window needs its own sort, but OFFSET still depends on the+ // hidden sort to select rows.+ assert_snapshot!(displayable(optimized.as_ref()).indent(true), @r#"+ BoundedWindowAggExec: wdw=[count: Field { "count": Int64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]+ SortExec: expr=[nullable_col@0 ASC], preserve_partitioning=[false]+ GlobalLimitExec: skip=1, fetch=None+ FilterExec: nullable_col@0 IS NOT NULL+ ProjectionExec: expr=[nullable_col@0 as nullable_col]+ SortExec: expr=[non_nullable_col@1 ASC], preserve_partitioning=[false]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[nullable_col, non_nullable_col], file_type=parquet+ "#);+ } else {+ // Without a limit, the window's ordering supersedes the hidden sort.+ assert_snapshot!(displayable(optimized.as_ref()).indent(true), @r#"+ BoundedWindowAggExec: wdw=[count: Field { "count": Int64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]+ SortExec: expr=[nullable_col@0 ASC], preserve_partitioning=[false]+ FilterExec: nullable_col@0 IS NOT NULL+ ProjectionExec: expr=[nullable_col@0 as nullable_col]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[nullable_col, non_nullable_col], file_type=parquet+ "#);+ }+ }+ Ok(())+}++#[tokio::test]+async fn test_do_not_remove_sort_below_fetch_after_projection() -> Result<()> {+ let batch =+ record_batch!(("a", Int32, [30, 10, 20, 40]), ("b", Int32, [2, 4, 1, 3]))?;+ let schema = batch.schema();+ let source = Arc::new(DataSourceExec::new(Arc::new(+ datafusion::datasource::memory::MemorySourceConfig::try_new(+ &[vec![batch]],+ Arc::clone(&schema),+ None,+ )?,+ ))) as Arc<dyn ExecutionPlan>;+ let sort = sort_exec([sort_expr("b", &schema)].into(), source);+ let filter = filter_exec(is_not_null(col("a", &schema)?)?, sort)+ .with_fetch(Some(2))+ .unwrap();+ let projection =+ projection_exec(vec![(col("a", &schema)?, "a".to_string())], filter)?;+ assert!(projection.output_ordering().is_none());++ // The window needs a new ordering, but the filter's embedded limit still+ // depends on the hidden sort to select a=20 and a=30.+ let window =+ bounded_window_exec("a", [sort_expr("a", &projection.schema())], projection);+ let mut config = ConfigOptions::new();+ config.execution.target_partitions = 1;+ let optimized = EnsureRequirements::new().optimize(window, &config)?;+ let optimized = SanityCheckPlan::new().optimize(optimized, &config)?;+ let twice = EnsureRequirements::new().optimize(Arc::clone(&optimized), &config)?;+ assert_eq!(get_plan_string(&twice), get_plan_string(&optimized));++ for plan in [optimized, twice] {+ let results =+ datafusion_physical_plan::collect(plan, Arc::new(TaskContext::default()))+ .await?;+ datafusion_common::assert_batches_eq!(+ [+ "+----+-------+",+ "| a | count |",+ "+----+-------+",+ "| 20 | 1 |",+ "| 30 | 2 |",+ "+----+-------+",+ ],+ &results+ );+ }+ Ok(())+}+#[tokio::test]async fn test_union_inputs_sorted() -> Result<()> {let schema = create_test_schema()?;diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/mod.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/mod.rsindex 8a4905365b..4d35061b7f 100644--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/mod.rs+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/mod.rs@@ -102,8 +102,9 @@ fn update_sort_ctx_children_data(child_node.data = if is_sort(child_plan) {// child is sorttrue- } else if is_limit(child_plan) {- // There is no sort linkage for this path, it starts at a limit.+ } else if is_limit(child_plan) || child_plan.fetch().is_some() {+ // Removing a sort below a row limit can change which rows survive,+ // including limits embedded in operators such as FilterExec.false} else {// If a descendent is a sort, and the child maintains the sort.@@ -427,16 +428,14 @@ pub fn ensure_sorting(.zip(requirements.children).enumerate(){- let physical_ordering = child.plan.output_ordering();-if let Some(required) = required_ordering {let eq_properties = child.plan.equivalence_properties();let req = required.into_single();if !eq_properties.ordering_satisfy_requirement(req.clone())? {- // Make sure we preserve the ordering requirements:- if physical_ordering.is_some() {- child = update_child_to_remove_unnecessary_sort(idx, child, plan)?;- }+ // The new sort supersedes linked sorts even when a projection+ // hides their keys. Explicit and embedded limits stop sort+ // linkage, preserving sorts that determine which rows survive.+ child = update_child_to_remove_unnecessary_sort(idx, child, plan)?;child = add_sort_above(child,req,@@ -446,9 +445,11 @@ pub fn ensure_sorting();child = update_sort_ctx_children_data(child, true)?;}- } else if physical_ordering.is_none() || !plan.maintains_input_order()[idx] {- // We have a `SortExec` whose effect may be neutralized by another- // order-imposing operator, remove this sort:+ } else if !plan.maintains_input_order()[idx] {+ // Remove linked sorts only when this operator does not preserve+ // their row order. A projection can hide the sort key, making+ // output_ordering() None, while a later LIMIT or OFFSET still+ // depends on that order to select rows.child = update_child_to_remove_unnecessary_sort(idx, child, plan)?;}updated_children.push(child);diff --git a/datafusion/sqllogictest/test_files/limit.slt b/datafusion/sqllogictest/test_files/limit.sltindex 06d1c85f5f..d5f1cdbc7b 100644--- a/datafusion/sqllogictest/test_files/limit.slt+++ b/datafusion/sqllogictest/test_files/limit.slt@@ -1257,3 +1257,58 @@ physical_planstatement okDROP TABLE t22489;++# Regression test for https://github.com/apache/datafusion/issues/23534+# The sort below OFFSET selects rows even when a projection hides its sort key.+statement ok+CREATE TABLE t23534 (grp VARCHAR NOT NULL, sort_key BIGINT NOT NULL)+AS VALUES ('physical_second', 2), ('sorted_first', 1);++query TI+SELECT grp, sort_key FROM t23534 ORDER BY sort_key ASC NULLS LAST OFFSET 1;+----+physical_second 2++query TI+SELECT grp, COUNT(*) AS n+FROM (+ SELECT grp, sort_key FROM t23534+ ORDER BY sort_key ASC NULLS LAST OFFSET 1+) q+GROUP BY grp+ORDER BY grp;+----+physical_second 1++# One pass leaves a projection between the limit and sort. Correctness must+# not depend on a later logical rewrite moving that projection.+statement ok+SET datafusion.optimizer.max_passes = 1;++query TI+SELECT grp, COUNT(*) AS n+FROM (+ SELECT grp, sort_key FROM t23534+ ORDER BY sort_key ASC NULLS LAST OFFSET 1+) q+GROUP BY grp+ORDER BY grp;+----+physical_second 1++query TI+SELECT grp, COUNT(*) AS n+FROM (+ SELECT grp, sort_key FROM t23534+ ORDER BY sort_key ASC NULLS LAST LIMIT 1 OFFSET 1+) q+GROUP BY grp+ORDER BY grp;+----+physical_second 1++statement ok+SET datafusion.optimizer.max_passes = 3;++statement ok+DROP TABLE t23534;