Skip to content

Apache DataFusion SQL Query Engine

fix(`EnsureRequirements`): remap sort requirement through `ProjectionExec` on pushdown (#23199)

Jack Eadie31198bace873
diff --git a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs
index 9a459f2049..ecff2edbbe 100644
--- a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs
+++ b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs
@@ -42,7 +42,7 @@ use datafusion_execution::object_store::ObjectStoreUrl;
use datafusion_physical_expr_common::sort_expr::{
LexOrdering, PhysicalSortExpr, PhysicalSortRequirement, OrderingRequirements
};
-use datafusion_physical_expr::{Distribution, Partitioning};
+use datafusion_physical_expr::{Distribution, Partitioning, PhysicalExpr};
use datafusion_physical_expr::expressions::{col, BinaryExpr, Column, NotExpr};
use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
use datafusion_physical_plan::repartition::RepartitionExec;
@@ -52,6 +52,7 @@ use datafusion_physical_plan::{displayable, get_plan_string, ExecutionPlan};
use datafusion::datasource::physical_plan::CsvSource;
use datafusion::datasource::listing::PartitionedFile;
use datafusion_physical_optimizer::enforce_sorting::{PlanWithCorrespondingCoalescePartitions, PlanWithCorrespondingSort, parallelize_sorts, ensure_sorting};
+use datafusion_physical_optimizer::sanity_checker::SanityCheckPlan;
use datafusion_physical_optimizer::enforce_sorting::replace_with_order_preserving_variants::{replace_with_order_preserving_variants, OrderPreservationContext};
use datafusion_physical_optimizer::enforce_sorting::sort_pushdown::{SortPushDown, assign_initial_requirements, pushdown_sorts};
use datafusion_physical_optimizer::ensure_requirements::EnsureRequirements;
@@ -2845,3 +2846,131 @@ async fn test_sort_with_streaming_table() -> Result<()> {
Ok(())
}
+
+/// Regression: `parallelize_sorts` must not relocate a per-partition `SortExec`
+/// below an order-preserving `ProjectionExec` that *reorders* columns without
+/// remapping the sort-key column indices.
+///
+/// Builds the minimal physical plan that reproduces the bug (as it looks after
+/// distribution enforcement, before sorting enforcement):
+///
+/// ```text
+/// SortExec(fetch=4) [score@1 DESC, a@0 ASC] (global, single-partition)
+/// CoalescePartitionsExec
+/// ProjectionExec [a@0, score@2 as score, b@1 as value] <- reorder: score 2 -> 1
+/// ProjectionExec [a@0, b@1, c+d as score] <- computes score at index 2
+/// SortExec(fetch=1000) [c+d DESC] preserve_partitioning=[true] <- inner ordering
+/// RepartitionExec(RoundRobinBatch) <- multi-partition
+/// DataSourceExec
+/// ```
+///
+/// `parallelize_sorts` turns the `CoalescePartitionsExec` + global `SortExec`
+/// into a `SortPreservingMergeExec` + per-partition `SortExec`, sinking the
+/// per-partition sort *below* the reordering projection. The sort key
+/// `score@1` is valid in the projection's output schema, but in the child
+/// schema `[a, b, score]` index 1 is `b` and `score` is at index 2. If the
+/// index is not remapped, the relocated `SortExec` references the wrong column
+/// and `SanityCheckPlan` rejects the plan with
+/// "does not satisfy order requirements ... Child-0 order: []".
+fn reorder_projection_physical_plan() -> Result<Arc<dyn ExecutionPlan>> {
+ let schema = create_test_schema3()?; // [a, b, c, d, e]
+ let source = parquet_exec(schema.clone());
+ let repartitioned = repartition_exec(source); // RoundRobinBatch -> multi-partition
+
+ // The score-source expression `c + d`. The inner sort below orders by this
+ // expression, and the lower projection aliases the *same* expression to
+ // `score`, so the projection output is already ordered by `score`. This
+ // existing ordering is what drives sort enforcement to relocate the outer
+ // sort below the reorder projection.
+ let score_expr = Arc::new(BinaryExpr::new(
+ col("c", &schema)?,
+ Operator::Plus,
+ col("d", &schema)?,
+ )) as Arc<dyn PhysicalExpr>;
+
+ // Inner per-partition, fetch-bearing sort on `c + d`.
+ let inner_ordering: LexOrdering = [PhysicalSortExpr::new(
+ Arc::clone(&score_expr),
+ SortOptions {
+ descending: true,
+ nulls_first: false,
+ },
+ )]
+ .into();
+ let inner_sort = Arc::new(
+ SortExec::new(inner_ordering, repartitioned)
+ .with_fetch(Some(1000))
+ .with_preserve_partitioning(true),
+ );
+
+ // Lower projection: compute `score` (= c + d) as the last column. Output
+ // schema: [a, b, score]; output is ordered by `score`.
+ let lower = projection_exec(
+ vec![
+ (col("a", &schema)?, "a".to_string()),
+ (col("b", &schema)?, "b".to_string()),
+ (Arc::clone(&score_expr), "score".to_string()),
+ ],
+ inner_sort,
+ )?;
+
+ // Upper projection: reorder so `score` moves from input index 2 to output
+ // index 1, and rename `b` to `value`. Output schema: [a, score, value].
+ let lower_schema = lower.schema();
+ let upper = projection_exec(
+ vec![
+ (col("a", &lower_schema)?, "a".to_string()),
+ (col("score", &lower_schema)?, "score".to_string()),
+ (col("b", &lower_schema)?, "value".to_string()),
+ ],
+ lower,
+ )?;
+
+ // Global, fetch-bearing sort on the renamed column + a tiebreaker, expressed
+ // in the upper projection's output schema (score@1 DESC NULLS LAST, a@0 ASC).
+ let upper_schema = upper.schema();
+ let ordering: LexOrdering = [
+ sort_expr_options(
+ "score",
+ &upper_schema,
+ SortOptions {
+ descending: true,
+ nulls_first: false,
+ },
+ ),
+ sort_expr("a", &upper_schema),
+ ]
+ .into();
+ let coalesced = coalesce_partitions_exec(upper);
+ Ok(sort_exec_with_fetch(ordering, Some(4), coalesced))
+}
+
+#[tokio::test]
+async fn test_parallelize_sorts_remaps_index_through_reordering_projection() -> Result<()>
+{
+ let physical_plan = reorder_projection_physical_plan()?;
+
+ // `EnsureRequirements` (with sort repartitioning enabled) runs the sort
+ // enforcement pass, including `parallelize_sorts`.
+ let mut config = ConfigOptions::new();
+ config.optimizer.repartition_sorts = true;
+ let optimized = EnsureRequirements::new().optimize(physical_plan, &config)?;
+
+ // The optimized plan must be physically valid. Before the fix this fails:
+ // the per-partition `SortExec` was relocated below the reordering projection
+ // but kept the key `score@1` (valid only in the projection output), while its
+ // child schema `[a, b, score]` has `score` at index 2 — so `SanityCheckPlan`
+ // reports `does not satisfy order requirements: [...]. Child-0 order: []`.
+ SanityCheckPlan::new()
+ .optimize(Arc::clone(&optimized), &ConfigOptions::default())
+ .unwrap_or_else(|e| {
+ panic!(
+ "Sort enforcement produced a plan that fails SanityCheckPlan \
+ (stale sort-key index after relocating the SortExec below a \
+ reordering ProjectionExec): {e}\n\nPlan:\n{}",
+ displayable(optimized.as_ref()).indent(true)
+ )
+ });
+
+ Ok(())
+}
diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
index 43ec3eabbf..159c5a9e50 100644
--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
@@ -43,7 +43,7 @@ use datafusion_physical_plan::joins::utils::{
ColumnIndex, calculate_join_output_ordering,
};
use datafusion_physical_plan::joins::{HashJoinExec, SortMergeJoinExec};
-use datafusion_physical_plan::projection::ProjectionExec;
+use datafusion_physical_plan::projection::{ProjectionExec, ProjectionExpr};
use datafusion_physical_plan::repartition::RepartitionExec;
use datafusion_physical_plan::sorts::sort::SortExec;
use datafusion_physical_plan::tree_node::PlanContext;
@@ -397,14 +397,34 @@ fn pushdown_requirement_to_children(
// Push down through operator with fetch when:
// - requirement is aligned with output ordering
// - it preserves ordering during execution
+ //
+ // A `ProjectionExec` reports a `fetch()` forwarded from its input and
+ // can renumber/reorder columns, so the requirement (expressed in the
+ // projection's output schema) must be remapped into the child schema
+ // before being pushed down — forwarding it unchanged would let a key
+ // such as `score@1` (valid in the output schema) refer to a different
+ // column in the child schema, producing a `SortExec` whose key points
+ // at the wrong column ("does not satisfy order requirements ...
+ // Child-0 order: []"). If a required column maps to a computed
+ // (non-`Column`) projection expression it cannot be expressed below the
+ // projection, so the sort is kept above it.
+ let child_required =
+ if let Some(projection) = plan.downcast_ref::<ProjectionExec>() {
+ match remap_requirement_through_projection(projection, &parent_required) {
+ Some(remapped) => remapped,
+ None => return Ok(None),
+ }
+ } else {
+ parent_required.clone()
+ };
let Some(ordering) = plan.properties().output_ordering() else {
- return Ok(Some(vec![Some(parent_required)]));
+ return Ok(Some(vec![Some(child_required)]));
};
if plan.properties().eq_properties.requirements_compatible(
parent_required.first().clone(),
ordering.clone().into(),
) {
- Ok(Some(vec![Some(parent_required)]))
+ Ok(Some(vec![Some(child_required)]))
} else {
Ok(None)
}
@@ -453,7 +473,10 @@ fn pushdown_requirement_to_children(
|| !maintains_input_order.iter().any(|o| *o)
|| plan.is::<RepartitionExec>()
|| plan.is::<FilterExec>()
- // TODO: Add support for Projection push down
+ // A `ProjectionExec` with a fetch is handled in the fetch branch above
+ // (it remaps the requirement through the projection's column mapping).
+ // Without a fetch we do not push a sort requirement through a
+ // projection (the sort is placed above it).
|| plan.is::<ProjectionExec>()
|| pushdown_would_violate_requirements(&parent_required, plan.as_ref())
{
@@ -481,7 +504,51 @@ fn pushdown_requirement_to_children(
} else {
handle_custom_pushdown(plan, parent_required, &maintains_input_order)
}
- // TODO: Add support for Projection push down
+}
+
+/// Remap an ordering requirement expressed in a [`ProjectionExec`]'s output
+/// schema into its child (input) schema.
+///
+/// Every alternative requirement is remapped independently, and the
+/// hard/soft-ness of the original [`OrderingRequirements`] is preserved. An
+/// alternative that references a computed (non-[`Column`]) projection
+/// expression cannot be expressed in the child schema and is dropped; if every
+/// alternative drops out, this returns `None` (and pushdown is declined, i.e.
+/// the sort is kept above the projection).
+fn remap_requirement_through_projection(
+ projection: &ProjectionExec,
+ parent_required: &OrderingRequirements,
+) -> Option<OrderingRequirements> {
+ let exprs = projection.expr();
+ let (alternatives, soft) = parent_required.clone().into_alternatives();
+ let remapped = alternatives
+ .iter()
+ .filter_map(|req| remap_lex_requirement_through_projection(exprs, req));
+ OrderingRequirements::new_alternatives(remapped, soft)
+}
+
+/// Remap a single [`LexRequirement`] expressed in a [`ProjectionExec`]'s output
+/// schema into its child (input) schema.
+///
+/// Each requirement column at output index `i` is rewritten to the column the
+/// projection produces at that index (`projection.expr()[i]`). Returns `None`
+/// if any required column maps to a computed (non-[`Column`]) projection
+/// expression, since that ordering cannot be expressed in the child schema.
+fn remap_lex_requirement_through_projection(
+ exprs: &[ProjectionExpr],
+ req: &LexRequirement,
+) -> Option<LexRequirement> {
+ let mut child_reqs = Vec::with_capacity(req.len());
+ for sort_req in req.iter() {
+ let col = sort_req.expr.downcast_ref::<Column>()?;
+ let proj_expr = exprs.get(col.index())?;
+ let child_col = proj_expr.expr.downcast_ref::<Column>()?;
+ child_reqs.push(PhysicalSortRequirement::new(
+ Arc::new(child_col.clone()),
+ sort_req.options,
+ ));
+ }
+ LexRequirement::new(child_reqs)
}
/// Try to push sorting through [`AggregateExec`]
@@ -974,3 +1041,185 @@ enum RequirementsCompatibility {
/// Requirements not compatible
NonCompatible,
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ use arrow::compute::SortOptions;
+ use arrow::datatypes::{DataType, Field, Schema};
+ use datafusion_expr::Operator;
+ use datafusion_physical_expr::PhysicalExpr;
+ use datafusion_physical_expr::expressions::{BinaryExpr, col};
+ use datafusion_physical_plan::empty::EmptyExec;
+
+ const DESC: SortOptions = SortOptions {
+ descending: true,
+ nulls_first: false,
+ };
+ const ASC: SortOptions = SortOptions {
+ descending: false,
+ nulls_first: true,
+ };
+
+ /// Child (input) schema fed to the projections under test: `[a, b, c]`.
+ fn child_schema() -> Arc<Schema> {
+ Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Int32, true),
+ Field::new("b", DataType::Int32, true),
+ Field::new("c", DataType::Int32, true),
+ ]))
+ }
+
+ /// A projection over `[a, b, c]` whose output is `[a@0, c@2 as score,
+ /// b@1 as value]` — i.e. it *reorders* (`c` moves index 2 -> 1) and renames.
+ fn reordering_projection() -> Arc<ProjectionExec> {
+ let schema = child_schema();
+ let input = Arc::new(EmptyExec::new(Arc::clone(&schema)));
+ Arc::new(
+ ProjectionExec::try_new(
+ vec![
+ (col("a", &schema).unwrap(), "a".to_string()),
+ (col("c", &schema).unwrap(), "score".to_string()),
+ (col("b", &schema).unwrap(), "value".to_string()),
+ ],
+ input,
+ )
+ .unwrap(),
+ )
+ }
+
+ /// A projection over `[a, b, c]` whose output is `[a@0, b + c as computed]`,
+ /// so output column index 1 maps to a *computed* (non-`Column`) expression.
+ fn computed_projection() -> Arc<ProjectionExec> {
+ let schema = child_schema();
+ let input = Arc::new(EmptyExec::new(Arc::clone(&schema)));
+ let b_plus_c = Arc::new(BinaryExpr::new(
+ col("b", &schema).unwrap(),
+ Operator::Plus,
+ col("c", &schema).unwrap(),
+ )) as Arc<dyn PhysicalExpr>;
+ Arc::new(
+ ProjectionExec::try_new(
+ vec![
+ (col("a", &schema).unwrap(), "a".to_string()),
+ (b_plus_c, "computed".to_string()),
+ ],
+ input,
+ )
+ .unwrap(),
+ )
+ }
+
+ /// `PhysicalSortRequirement` for `<name>@<index> <options>` in `schema`.
+ fn req(name: &str, schema: &Schema, options: SortOptions) -> PhysicalSortRequirement {
+ PhysicalSortRequirement::new(col(name, schema).unwrap(), Some(options))
+ }
+
+ fn lex(reqs: impl IntoIterator<Item = PhysicalSortRequirement>) -> LexRequirement {
+ LexRequirement::new(reqs).unwrap()
+ }
+
+ #[test]
+ fn remap_single_hard_requirement_through_reordering_projection() {
+ let projection = reordering_projection();
+ let out = projection.schema();
+ let child = child_schema();
+
+ // `score@1 DESC, a@0 ASC` in the output schema.
+ let required = OrderingRequirements::new(lex([
+ req("score", &out, DESC),
+ req("a", &out, ASC),
+ ]));
+
+ let remapped =
+ remap_requirement_through_projection(&projection, &required).unwrap();
+
+ // `score@1` -> `c@2`, `a@0` -> `a@0`; still a single hard requirement.
+ let expected = OrderingRequirements::new(lex([
+ req("c", &child, DESC),
+ req("a", &child, ASC),
+ ]));
+ assert_eq!(remapped, expected);
+ }
+
+ #[test]
+ fn remap_preserves_softness() {
+ let projection = reordering_projection();
+ let out = projection.schema();
+ let child = child_schema();
+
+ let required = OrderingRequirements::new_soft(lex([req("score", &out, DESC)]));
+ let remapped =
+ remap_requirement_through_projection(&projection, &required).unwrap();
+
+ let expected = OrderingRequirements::new_soft(lex([req("c", &child, DESC)]));
+ assert_eq!(remapped, expected);
+ // Hardness/softness is preserved through the remap.
+ assert!(matches!(remapped, OrderingRequirements::Soft(_)));
+ }
+
+ #[test]
+ fn remap_preserves_all_hard_alternatives() {
+ let projection = reordering_projection();
+ let out = projection.schema();
+ let child = child_schema();
+
+ // Two alternatives: `score@1 DESC` or `a@0 ASC, value@2 ASC`.
+ let mut required = OrderingRequirements::new(lex([req("score", &out, DESC)]));
+ required.add_alternative(lex([req("a", &out, ASC), req("value", &out, ASC)]));
+
+ let remapped =
+ remap_requirement_through_projection(&projection, &required).unwrap();
+
+ // Both alternatives survive and are remapped; hardness preserved.
+ let (alts, soft) = remapped.into_alternatives();
+ assert!(!soft);
+ assert_eq!(alts.len(), 2);
+ assert_eq!(alts[0], lex([req("c", &child, DESC)]));
+ // `value@2` -> `b@1`, `a@0` -> `a@0`.
+ assert_eq!(alts[1], lex([req("a", &child, ASC), req("b", &child, ASC)]));
+ }
+
+ #[test]
+ fn remap_drops_unsatisfiable_alternative_but_keeps_others() {
+ let projection = computed_projection();
+ let out = projection.schema();
+ let child = child_schema();
+
+ // Alt 1 (`a@0 ASC`) is expressible below the projection; alt 2
+ // (`computed@1 DESC`) maps to `b + c` and is not.
+ let mut required = OrderingRequirements::new(lex([req("a", &out, ASC)]));
+ required.add_alternative(lex([req("computed", &out, DESC)]));
+
+ let remapped =
+ remap_requirement_through_projection(&projection, &required).unwrap();
+
+ // Only the satisfiable alternative is kept; hardness preserved.
+ let (alts, soft) = remapped.into_alternatives();
+ assert!(!soft);
+ assert_eq!(alts.len(), 1);
+ assert_eq!(alts[0], lex([req("a", &child, ASC)]));
+ }
+
+ #[test]
+ fn remap_declines_when_required_column_is_computed() {
+ let projection = computed_projection();
+ let out = projection.schema();
+
+ // The only required column maps to a computed expression -> decline.
+ let required = OrderingRequirements::new(lex([req("computed", &out, DESC)]));
+ assert!(remap_requirement_through_projection(&projection, &required).is_none());
+ }
+
+ #[test]
+ fn remap_declines_when_all_alternatives_are_computed() {
+ let projection = computed_projection();
+ let out = projection.schema();
+
+ let mut required = OrderingRequirements::new(lex([req("computed", &out, DESC)]));
+ required.add_alternative(lex([req("computed", &out, ASC)]));
+
+ assert!(remap_requirement_through_projection(&projection, &required).is_none());
+ }
+}