Apache DataFusion SQL Query Engine
fix(`EnsureRequirements`): remap sort requirement through `ProjectionExec` on pushdown (#23199)
Jack Eadie31198bace873
diff --git a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs b/datafusion/core/tests/physical_optimizer/enforce_sorting.rsindex 9a459f2049..ecff2edbbe 100644--- a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs+++ b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs@@ -42,7 +42,7 @@ use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr, PhysicalSortRequirement, OrderingRequirements};-use datafusion_physical_expr::{Distribution, Partitioning};+use datafusion_physical_expr::{Distribution, Partitioning, PhysicalExpr};use datafusion_physical_expr::expressions::{col, BinaryExpr, Column, NotExpr};use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};use datafusion_physical_plan::repartition::RepartitionExec;@@ -52,6 +52,7 @@ use datafusion_physical_plan::{displayable, get_plan_string, ExecutionPlan};use datafusion::datasource::physical_plan::CsvSource;use datafusion::datasource::listing::PartitionedFile;use datafusion_physical_optimizer::enforce_sorting::{PlanWithCorrespondingCoalescePartitions, PlanWithCorrespondingSort, parallelize_sorts, ensure_sorting};+use datafusion_physical_optimizer::sanity_checker::SanityCheckPlan;use datafusion_physical_optimizer::enforce_sorting::replace_with_order_preserving_variants::{replace_with_order_preserving_variants, OrderPreservationContext};use datafusion_physical_optimizer::enforce_sorting::sort_pushdown::{SortPushDown, assign_initial_requirements, pushdown_sorts};use datafusion_physical_optimizer::ensure_requirements::EnsureRequirements;@@ -2845,3 +2846,131 @@ async fn test_sort_with_streaming_table() -> Result<()> {Ok(())}++/// Regression: `parallelize_sorts` must not relocate a per-partition `SortExec`+/// below an order-preserving `ProjectionExec` that *reorders* columns without+/// remapping the sort-key column indices.+///+/// Builds the minimal physical plan that reproduces the bug (as it looks after+/// distribution enforcement, before sorting enforcement):+///+/// ```text+/// SortExec(fetch=4) [score@1 DESC, a@0 ASC] (global, single-partition)+/// CoalescePartitionsExec+/// ProjectionExec [a@0, score@2 as score, b@1 as value] <- reorder: score 2 -> 1+/// ProjectionExec [a@0, b@1, c+d as score] <- computes score at index 2+/// SortExec(fetch=1000) [c+d DESC] preserve_partitioning=[true] <- inner ordering+/// RepartitionExec(RoundRobinBatch) <- multi-partition+/// DataSourceExec+/// ```+///+/// `parallelize_sorts` turns the `CoalescePartitionsExec` + global `SortExec`+/// into a `SortPreservingMergeExec` + per-partition `SortExec`, sinking the+/// per-partition sort *below* the reordering projection. The sort key+/// `score@1` is valid in the projection's output schema, but in the child+/// schema `[a, b, score]` index 1 is `b` and `score` is at index 2. If the+/// index is not remapped, the relocated `SortExec` references the wrong column+/// and `SanityCheckPlan` rejects the plan with+/// "does not satisfy order requirements ... Child-0 order: []".+fn reorder_projection_physical_plan() -> Result<Arc<dyn ExecutionPlan>> {+ let schema = create_test_schema3()?; // [a, b, c, d, e]+ let source = parquet_exec(schema.clone());+ let repartitioned = repartition_exec(source); // RoundRobinBatch -> multi-partition++ // The score-source expression `c + d`. The inner sort below orders by this+ // expression, and the lower projection aliases the *same* expression to+ // `score`, so the projection output is already ordered by `score`. This+ // existing ordering is what drives sort enforcement to relocate the outer+ // sort below the reorder projection.+ let score_expr = Arc::new(BinaryExpr::new(+ col("c", &schema)?,+ Operator::Plus,+ col("d", &schema)?,+ )) as Arc<dyn PhysicalExpr>;++ // Inner per-partition, fetch-bearing sort on `c + d`.+ let inner_ordering: LexOrdering = [PhysicalSortExpr::new(+ Arc::clone(&score_expr),+ SortOptions {+ descending: true,+ nulls_first: false,+ },+ )]+ .into();+ let inner_sort = Arc::new(+ SortExec::new(inner_ordering, repartitioned)+ .with_fetch(Some(1000))+ .with_preserve_partitioning(true),+ );++ // Lower projection: compute `score` (= c + d) as the last column. Output+ // schema: [a, b, score]; output is ordered by `score`.+ let lower = projection_exec(+ vec![+ (col("a", &schema)?, "a".to_string()),+ (col("b", &schema)?, "b".to_string()),+ (Arc::clone(&score_expr), "score".to_string()),+ ],+ inner_sort,+ )?;++ // Upper projection: reorder so `score` moves from input index 2 to output+ // index 1, and rename `b` to `value`. Output schema: [a, score, value].+ let lower_schema = lower.schema();+ let upper = projection_exec(+ vec![+ (col("a", &lower_schema)?, "a".to_string()),+ (col("score", &lower_schema)?, "score".to_string()),+ (col("b", &lower_schema)?, "value".to_string()),+ ],+ lower,+ )?;++ // Global, fetch-bearing sort on the renamed column + a tiebreaker, expressed+ // in the upper projection's output schema (score@1 DESC NULLS LAST, a@0 ASC).+ let upper_schema = upper.schema();+ let ordering: LexOrdering = [+ sort_expr_options(+ "score",+ &upper_schema,+ SortOptions {+ descending: true,+ nulls_first: false,+ },+ ),+ sort_expr("a", &upper_schema),+ ]+ .into();+ let coalesced = coalesce_partitions_exec(upper);+ Ok(sort_exec_with_fetch(ordering, Some(4), coalesced))+}++#[tokio::test]+async fn test_parallelize_sorts_remaps_index_through_reordering_projection() -> Result<()>+{+ let physical_plan = reorder_projection_physical_plan()?;++ // `EnsureRequirements` (with sort repartitioning enabled) runs the sort+ // enforcement pass, including `parallelize_sorts`.+ let mut config = ConfigOptions::new();+ config.optimizer.repartition_sorts = true;+ let optimized = EnsureRequirements::new().optimize(physical_plan, &config)?;++ // The optimized plan must be physically valid. Before the fix this fails:+ // the per-partition `SortExec` was relocated below the reordering projection+ // but kept the key `score@1` (valid only in the projection output), while its+ // child schema `[a, b, score]` has `score` at index 2 — so `SanityCheckPlan`+ // reports `does not satisfy order requirements: [...]. Child-0 order: []`.+ SanityCheckPlan::new()+ .optimize(Arc::clone(&optimized), &ConfigOptions::default())+ .unwrap_or_else(|e| {+ panic!(+ "Sort enforcement produced a plan that fails SanityCheckPlan \+ (stale sort-key index after relocating the SortExec below a \+ reordering ProjectionExec): {e}\n\nPlan:\n{}",+ displayable(optimized.as_ref()).indent(true)+ )+ });++ Ok(())+}diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rsindex 43ec3eabbf..159c5a9e50 100644--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs@@ -43,7 +43,7 @@ use datafusion_physical_plan::joins::utils::{ColumnIndex, calculate_join_output_ordering,};use datafusion_physical_plan::joins::{HashJoinExec, SortMergeJoinExec};-use datafusion_physical_plan::projection::ProjectionExec;+use datafusion_physical_plan::projection::{ProjectionExec, ProjectionExpr};use datafusion_physical_plan::repartition::RepartitionExec;use datafusion_physical_plan::sorts::sort::SortExec;use datafusion_physical_plan::tree_node::PlanContext;@@ -397,14 +397,34 @@ fn pushdown_requirement_to_children(// Push down through operator with fetch when:// - requirement is aligned with output ordering// - it preserves ordering during execution+ //+ // A `ProjectionExec` reports a `fetch()` forwarded from its input and+ // can renumber/reorder columns, so the requirement (expressed in the+ // projection's output schema) must be remapped into the child schema+ // before being pushed down — forwarding it unchanged would let a key+ // such as `score@1` (valid in the output schema) refer to a different+ // column in the child schema, producing a `SortExec` whose key points+ // at the wrong column ("does not satisfy order requirements ...+ // Child-0 order: []"). If a required column maps to a computed+ // (non-`Column`) projection expression it cannot be expressed below the+ // projection, so the sort is kept above it.+ let child_required =+ if let Some(projection) = plan.downcast_ref::<ProjectionExec>() {+ match remap_requirement_through_projection(projection, &parent_required) {+ Some(remapped) => remapped,+ None => return Ok(None),+ }+ } else {+ parent_required.clone()+ };let Some(ordering) = plan.properties().output_ordering() else {- return Ok(Some(vec![Some(parent_required)]));+ return Ok(Some(vec![Some(child_required)]));};if plan.properties().eq_properties.requirements_compatible(parent_required.first().clone(),ordering.clone().into(),) {- Ok(Some(vec![Some(parent_required)]))+ Ok(Some(vec![Some(child_required)]))} else {Ok(None)}@@ -453,7 +473,10 @@ fn pushdown_requirement_to_children(|| !maintains_input_order.iter().any(|o| *o)|| plan.is::<RepartitionExec>()|| plan.is::<FilterExec>()- // TODO: Add support for Projection push down+ // A `ProjectionExec` with a fetch is handled in the fetch branch above+ // (it remaps the requirement through the projection's column mapping).+ // Without a fetch we do not push a sort requirement through a+ // projection (the sort is placed above it).|| plan.is::<ProjectionExec>()|| pushdown_would_violate_requirements(&parent_required, plan.as_ref()){@@ -481,7 +504,51 @@ fn pushdown_requirement_to_children(} else {handle_custom_pushdown(plan, parent_required, &maintains_input_order)}- // TODO: Add support for Projection push down+}++/// Remap an ordering requirement expressed in a [`ProjectionExec`]'s output+/// schema into its child (input) schema.+///+/// Every alternative requirement is remapped independently, and the+/// hard/soft-ness of the original [`OrderingRequirements`] is preserved. An+/// alternative that references a computed (non-[`Column`]) projection+/// expression cannot be expressed in the child schema and is dropped; if every+/// alternative drops out, this returns `None` (and pushdown is declined, i.e.+/// the sort is kept above the projection).+fn remap_requirement_through_projection(+ projection: &ProjectionExec,+ parent_required: &OrderingRequirements,+) -> Option<OrderingRequirements> {+ let exprs = projection.expr();+ let (alternatives, soft) = parent_required.clone().into_alternatives();+ let remapped = alternatives+ .iter()+ .filter_map(|req| remap_lex_requirement_through_projection(exprs, req));+ OrderingRequirements::new_alternatives(remapped, soft)+}++/// Remap a single [`LexRequirement`] expressed in a [`ProjectionExec`]'s output+/// schema into its child (input) schema.+///+/// Each requirement column at output index `i` is rewritten to the column the+/// projection produces at that index (`projection.expr()[i]`). Returns `None`+/// if any required column maps to a computed (non-[`Column`]) projection+/// expression, since that ordering cannot be expressed in the child schema.+fn remap_lex_requirement_through_projection(+ exprs: &[ProjectionExpr],+ req: &LexRequirement,+) -> Option<LexRequirement> {+ let mut child_reqs = Vec::with_capacity(req.len());+ for sort_req in req.iter() {+ let col = sort_req.expr.downcast_ref::<Column>()?;+ let proj_expr = exprs.get(col.index())?;+ let child_col = proj_expr.expr.downcast_ref::<Column>()?;+ child_reqs.push(PhysicalSortRequirement::new(+ Arc::new(child_col.clone()),+ sort_req.options,+ ));+ }+ LexRequirement::new(child_reqs)}/// Try to push sorting through [`AggregateExec`]@@ -974,3 +1041,185 @@ enum RequirementsCompatibility {/// Requirements not compatibleNonCompatible,}++#[cfg(test)]+mod tests {+ use super::*;++ use arrow::compute::SortOptions;+ use arrow::datatypes::{DataType, Field, Schema};+ use datafusion_expr::Operator;+ use datafusion_physical_expr::PhysicalExpr;+ use datafusion_physical_expr::expressions::{BinaryExpr, col};+ use datafusion_physical_plan::empty::EmptyExec;++ const DESC: SortOptions = SortOptions {+ descending: true,+ nulls_first: false,+ };+ const ASC: SortOptions = SortOptions {+ descending: false,+ nulls_first: true,+ };++ /// Child (input) schema fed to the projections under test: `[a, b, c]`.+ fn child_schema() -> Arc<Schema> {+ Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int32, true),+ Field::new("b", DataType::Int32, true),+ Field::new("c", DataType::Int32, true),+ ]))+ }++ /// A projection over `[a, b, c]` whose output is `[a@0, c@2 as score,+ /// b@1 as value]` — i.e. it *reorders* (`c` moves index 2 -> 1) and renames.+ fn reordering_projection() -> Arc<ProjectionExec> {+ let schema = child_schema();+ let input = Arc::new(EmptyExec::new(Arc::clone(&schema)));+ Arc::new(+ ProjectionExec::try_new(+ vec![+ (col("a", &schema).unwrap(), "a".to_string()),+ (col("c", &schema).unwrap(), "score".to_string()),+ (col("b", &schema).unwrap(), "value".to_string()),+ ],+ input,+ )+ .unwrap(),+ )+ }++ /// A projection over `[a, b, c]` whose output is `[a@0, b + c as computed]`,+ /// so output column index 1 maps to a *computed* (non-`Column`) expression.+ fn computed_projection() -> Arc<ProjectionExec> {+ let schema = child_schema();+ let input = Arc::new(EmptyExec::new(Arc::clone(&schema)));+ let b_plus_c = Arc::new(BinaryExpr::new(+ col("b", &schema).unwrap(),+ Operator::Plus,+ col("c", &schema).unwrap(),+ )) as Arc<dyn PhysicalExpr>;+ Arc::new(+ ProjectionExec::try_new(+ vec![+ (col("a", &schema).unwrap(), "a".to_string()),+ (b_plus_c, "computed".to_string()),+ ],+ input,+ )+ .unwrap(),+ )+ }++ /// `PhysicalSortRequirement` for `<name>@<index> <options>` in `schema`.+ fn req(name: &str, schema: &Schema, options: SortOptions) -> PhysicalSortRequirement {+ PhysicalSortRequirement::new(col(name, schema).unwrap(), Some(options))+ }++ fn lex(reqs: impl IntoIterator<Item = PhysicalSortRequirement>) -> LexRequirement {+ LexRequirement::new(reqs).unwrap()+ }++ #[test]+ fn remap_single_hard_requirement_through_reordering_projection() {+ let projection = reordering_projection();+ let out = projection.schema();+ let child = child_schema();++ // `score@1 DESC, a@0 ASC` in the output schema.+ let required = OrderingRequirements::new(lex([+ req("score", &out, DESC),+ req("a", &out, ASC),+ ]));++ let remapped =+ remap_requirement_through_projection(&projection, &required).unwrap();++ // `score@1` -> `c@2`, `a@0` -> `a@0`; still a single hard requirement.+ let expected = OrderingRequirements::new(lex([+ req("c", &child, DESC),+ req("a", &child, ASC),+ ]));+ assert_eq!(remapped, expected);+ }++ #[test]+ fn remap_preserves_softness() {+ let projection = reordering_projection();+ let out = projection.schema();+ let child = child_schema();++ let required = OrderingRequirements::new_soft(lex([req("score", &out, DESC)]));+ let remapped =+ remap_requirement_through_projection(&projection, &required).unwrap();++ let expected = OrderingRequirements::new_soft(lex([req("c", &child, DESC)]));+ assert_eq!(remapped, expected);+ // Hardness/softness is preserved through the remap.+ assert!(matches!(remapped, OrderingRequirements::Soft(_)));+ }++ #[test]+ fn remap_preserves_all_hard_alternatives() {+ let projection = reordering_projection();+ let out = projection.schema();+ let child = child_schema();++ // Two alternatives: `score@1 DESC` or `a@0 ASC, value@2 ASC`.+ let mut required = OrderingRequirements::new(lex([req("score", &out, DESC)]));+ required.add_alternative(lex([req("a", &out, ASC), req("value", &out, ASC)]));++ let remapped =+ remap_requirement_through_projection(&projection, &required).unwrap();++ // Both alternatives survive and are remapped; hardness preserved.+ let (alts, soft) = remapped.into_alternatives();+ assert!(!soft);+ assert_eq!(alts.len(), 2);+ assert_eq!(alts[0], lex([req("c", &child, DESC)]));+ // `value@2` -> `b@1`, `a@0` -> `a@0`.+ assert_eq!(alts[1], lex([req("a", &child, ASC), req("b", &child, ASC)]));+ }++ #[test]+ fn remap_drops_unsatisfiable_alternative_but_keeps_others() {+ let projection = computed_projection();+ let out = projection.schema();+ let child = child_schema();++ // Alt 1 (`a@0 ASC`) is expressible below the projection; alt 2+ // (`computed@1 DESC`) maps to `b + c` and is not.+ let mut required = OrderingRequirements::new(lex([req("a", &out, ASC)]));+ required.add_alternative(lex([req("computed", &out, DESC)]));++ let remapped =+ remap_requirement_through_projection(&projection, &required).unwrap();++ // Only the satisfiable alternative is kept; hardness preserved.+ let (alts, soft) = remapped.into_alternatives();+ assert!(!soft);+ assert_eq!(alts.len(), 1);+ assert_eq!(alts[0], lex([req("a", &child, ASC)]));+ }++ #[test]+ fn remap_declines_when_required_column_is_computed() {+ let projection = computed_projection();+ let out = projection.schema();++ // The only required column maps to a computed expression -> decline.+ let required = OrderingRequirements::new(lex([req("computed", &out, DESC)]));+ assert!(remap_requirement_through_projection(&projection, &required).is_none());+ }++ #[test]+ fn remap_declines_when_all_alternatives_are_computed() {+ let projection = computed_projection();+ let out = projection.schema();++ let mut required = OrderingRequirements::new(lex([req("computed", &out, DESC)]));+ required.add_alternative(lex([req("computed", &out, ASC)]));++ assert!(remap_requirement_through_projection(&projection, &required).is_none());+ }+}