Skip to content

Apache DataFusion SQL Query Engine

fix: validate sort pushdown through custom execution plans (#26118)

Geoffrey Claude390622a030de
diff --git a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs
index 49ac9c9dc4..05959bc62d 100644
--- a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs
+++ b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs
@@ -13,7 +13,7 @@ use crate::physical_optimizer::test_utils::{
parquet_exec_with_sort, projection_exec, repartition_exec, sort_exec,
sort_exec_with_fetch, sort_expr, sort_expr_options, sort_merge_join_exec,
sort_preserving_merge_exec, sort_preserving_merge_exec_with_fetch,
- spr_repartition_exec, stream_exec_ordered, union_exec,
+ spr_repartition_exec, stream_exec, stream_exec_ordered, union_exec,
};
use arrow::compute::{SortOptions};
@@ -2775,6 +2775,68 @@ async fn test_push_with_required_input_ordering_allowed() -> Result<()> {
Ok(())
}
+#[tokio::test]
+async fn test_push_with_required_input_ordering_conflicting_suffix() -> Result<()> {
+ let schema = create_test_schema3()?;
+ let required: LexOrdering = [sort_expr("a", &schema), sort_expr("b", &schema)].into();
+ let requested = [sort_expr("a", &schema), sort_expr("c", &schema)].into();
+ let input = sort_exec(required.clone(), memory_exec(&schema));
+ let plan = RequirementsTestExec::new(input)
+ .with_required_input_ordering(Some(required))
+ .into_arc();
+ let plan = sort_exec(requested, plan);
+ let test = EnforceSortingTest::new(plan).with_repartition_sorts(true);
+
+ assert_snapshot!(test.run(), @r"
+ Input / Optimized Plan:
+ SortExec: expr=[a@0 ASC, c@2 ASC], preserve_partitioning=[false]
+ RequiredInputOrderingExec
+ SortExec: expr=[a@0 ASC, b@1 ASC], preserve_partitioning=[false]
+ DataSourceExec: partitions=1, partition_sizes=[0]
+ ");
+ Ok(())
+}
+
+#[test]
+fn test_custom_sort_pushdown_keeps_sort_above_unbounded_input() -> Result<()> {
+ let schema = create_test_schema3()?;
+ // Keep the custom node directly below the sort. It bounds the stream
+ // without advertising a fetch, so this reaches the unbounded-input guard.
+ let custom = RequirementsTestExec::new(stream_exec(&schema))
+ .with_required_input_distribution(Distribution::SinglePartition)
+ .with_bounded_output()
+ .into_arc();
+ let plan = sort_exec([sort_expr("a", &schema)].into(), custom);
+
+ assert_snapshot!(EnforceSortingTest::new(plan).run(), @r"
+ Input / Optimized Plan:
+ SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
+ RequiredInputOrderingExec
+ StreamingTableExec: partition_sizes=1, projection=[a, b, c, d, e], infinite_source=true
+ ");
+ Ok(())
+}
+
+#[test]
+fn test_custom_sort_pushdown_rejects_narrower_order_maintaining_input() -> Result<()> {
+ let schema = create_test_schema3()?;
+ let narrow_schema = Arc::new(schema.project(&[0])?);
+ // b@1 is valid in the first input and output, but not the second input.
+ let custom = RequirementsTestExec::new(memory_exec(&schema))
+ .with_additional_input(memory_exec(&narrow_schema))
+ .into_arc();
+ let plan = sort_exec([sort_expr("b", &schema)].into(), custom);
+
+ assert_snapshot!(EnforceSortingTest::new(plan).run(), @r"
+ Input / Optimized Plan:
+ SortExec: expr=[b@1 ASC], preserve_partitioning=[false]
+ RequiredInputOrderingExec
+ DataSourceExec: partitions=1, partition_sizes=[0]
+ DataSourceExec: partitions=1, partition_sizes=[0]
+ ");
+ Ok(())
+}
+
#[tokio::test]
async fn test_replace_with_partial_sort() -> Result<()> {
let schema = create_test_schema3()?;
diff --git a/datafusion/core/tests/physical_optimizer/pushdown_sort.rs b/datafusion/core/tests/physical_optimizer/pushdown_sort.rs
index aa1aebfb23..4a81d11949 100644
--- a/datafusion/core/tests/physical_optimizer/pushdown_sort.rs
+++ b/datafusion/core/tests/physical_optimizer/pushdown_sort.rs
@@ -21,8 +21,10 @@ use datafusion_physical_expr::expressions;
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
use datafusion_physical_expr_common::sort_expr::LexOrdering;
use datafusion_physical_optimizer::PhysicalOptimizerRule;
+use datafusion_physical_optimizer::ensure_requirements::EnsureRequirements;
use datafusion_physical_optimizer::pushdown_sort::PushdownSort;
-use datafusion_physical_plan::collect;
+use datafusion_physical_plan::sorts::sort::SortExec;
+use datafusion_physical_plan::{ExecutionPlan, collect};
use std::sync::Arc;
use crate::physical_optimizer::test_utils::{
@@ -1124,6 +1126,29 @@ fn test_sort_pushdown_exact_no_fetch_no_limit() {
);
}
+#[test]
+fn test_custom_plan_with_fetch_keeps_outer_sort() -> Result<()> {
+ let schema = schema();
+ let a = sort_expr("a", &schema);
+ let source = TestScan::new(schema.clone(), vec![[a.clone()].into()])
+ .with_supports_fetch(true)
+ .with_fetch(Some(10))
+ .unwrap();
+ let ordering = LexOrdering::new(vec![a, sort_expr("b", &schema)]).unwrap();
+ let plan = sort_exec(ordering.clone(), source);
+
+ let mut config = ConfigOptions::new();
+ config.execution.target_partitions = 1;
+ let optimized = EnsureRequirements::new().optimize(plan, &config)?;
+ let sort = optimized
+ .downcast_ref::<SortExec>()
+ .expect("a fetched custom plan must retain the requested outer sort");
+ assert_eq!(sort.expr(), &ordering);
+ assert!(sort.input().is::<TestScan>());
+ assert_eq!(sort.input().fetch(), Some(10));
+ Ok(())
+}
+
#[test]
fn test_sort_pushdown_exact_preserves_fetch_with_global_limit() {
// When a source returns Exact but does NOT support with_fetch(),
diff --git a/datafusion/core/tests/physical_optimizer/test_utils.rs b/datafusion/core/tests/physical_optimizer/test_utils.rs
index a5c750c79e..b93281def2 100644
--- a/datafusion/core/tests/physical_optimizer/test_utils.rs
+++ b/datafusion/core/tests/physical_optimizer/test_utils.rs
@@ -421,25 +421,44 @@ pub fn projection_exec(
Ok(Arc::new(ProjectionExec::try_new(proj_exprs, input)?))
}
-/// A test [`ExecutionPlan`] whose requirements can be configured.
+/// A test [`ExecutionPlan`] with configurable input requirements.
+/// Its output properties follow its first input unless boundedness is overridden.
#[derive(Debug)]
pub struct RequirementsTestExec {
required_input_ordering: Option<LexOrdering>,
required_input_distribution: Distribution,
maintains_input_order: bool,
- input: Arc<dyn ExecutionPlan>,
+ inputs: Vec<Arc<dyn ExecutionPlan>>,
+ bounded_output: bool,
+ cache: Arc<PlanProperties>,
}
impl RequirementsTestExec {
pub fn new(input: Arc<dyn ExecutionPlan>) -> Self {
+ let cache = Arc::clone(input.properties());
Self {
required_input_ordering: None,
required_input_distribution: Distribution::UnspecifiedDistribution,
maintains_input_order: true,
- input,
+ inputs: vec![input],
+ bounded_output: false,
+ cache,
}
}
+ /// Add another input with the same requirements and order-preservation flag.
+ pub fn with_additional_input(mut self, input: Arc<dyn ExecutionPlan>) -> Self {
+ self.inputs.push(input);
+ self
+ }
+
+ /// Model a custom operator that stops reading an unbounded input.
+ pub fn with_bounded_output(mut self) -> Self {
+ self.bounded_output = true;
+ Arc::make_mut(&mut self.cache).boundedness = Boundedness::Bounded;
+ self
+ }
+
/// sets the required input ordering
pub fn with_required_input_ordering(
mut self,
@@ -490,27 +509,31 @@ impl ExecutionPlan for RequirementsTestExec {
}
fn properties(&self) -> &Arc<PlanProperties> {
- self.input.properties()
+ &self.cache
}
fn required_input_ordering(&self) -> Vec<Option<OrderingRequirements>> {
vec![
self.required_input_ordering
.as_ref()
- .map(|ordering| OrderingRequirements::from(ordering.clone())),
+ .map(|ordering| OrderingRequirements::from(ordering.clone()));
+ self.inputs.len()
]
}
fn input_distribution_requirements(&self) -> InputDistributionRequirements {
- InputDistributionRequirements::new(vec![self.required_input_distribution.clone()])
+ InputDistributionRequirements::new(vec![
+ self.required_input_distribution.clone();
+ self.inputs.len()
+ ])
}
fn maintains_input_order(&self) -> Vec<bool> {
- vec![self.maintains_input_order]
+ vec![self.maintains_input_order; self.inputs.len()]
}
fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
- vec![&self.input]
+ self.inputs.iter().collect()
}
fn replace_children(
@@ -518,12 +541,16 @@ impl ExecutionPlan for RequirementsTestExec {
children: Vec<Arc<dyn ExecutionPlan>>,
_: ReplaceChildrenOptions,
) -> Result<Arc<dyn ExecutionPlan>> {
- assert_eq!(children.len(), 1);
- Ok(RequirementsTestExec::new(Arc::clone(&children[0]))
+ assert_eq!(children.len(), self.inputs.len());
+ let mut plan = RequirementsTestExec::new(Arc::clone(&children[0]))
.with_required_input_ordering(self.required_input_ordering.clone())
.with_required_input_distribution(self.required_input_distribution.clone())
- .with_maintains_input_order(self.maintains_input_order)
- .into_arc())
+ .with_maintains_input_order(self.maintains_input_order);
+ plan.inputs.extend(children.into_iter().skip(1));
+ if self.bounded_output {
+ plan = plan.with_bounded_output();
+ }
+ Ok(plan.into_arc())
}
fn with_new_children(
@@ -1061,6 +1088,10 @@ impl ExecutionPlan for TestScan {
Ok(Arc::new(Statistics::new_unknown(&self.schema)))
}
+ fn supports_limit_pushdown(&self) -> bool {
+ self.supports_fetch
+ }
+
fn with_fetch(&self, fetch: Option<usize>) -> Option<Arc<dyn ExecutionPlan>> {
if self.supports_fetch {
let mut new_scan = self.clone();
diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
index 4f8670c15b..65a71510c9 100644
--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
@@ -36,7 +36,10 @@ use datafusion_physical_plan::repartition::RepartitionExec;
use datafusion_physical_plan::sorts::sort::SortExec;
use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;
use datafusion_physical_plan::tree_node::PlanContext;
-use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties};
+use datafusion_physical_plan::{
+ ChildSatisfactionOptions, ChildrenPropertiesMode, ExecutionPlan,
+ ExecutionPlanProperties, ReplaceChildrenOptions,
+};
/// "Data class" used by sort pushdown (now driven from `EnsureRequirements`)
/// to push down [`SortExec`] in the plan. In some cases the total
@@ -66,15 +69,14 @@ impl Default for ParentRequirements {
pub type SortPushDown = PlanContext<ParentRequirements>;
-/// Number of input rows `plan` needs from its children in order to produce
-/// the rows its parent will consume.
+/// Input row bound used by the specialized sort-pushdown handlers.
///
/// `parent_fetch` is the fetch a parent imposes on `plan`'s *output*
/// (`ParentRequirements::fetch`). It bounds `plan.fetch()` directly because
/// both count output rows, so the effective output fetch is
/// `min(plan.fetch(), parent_fetch)`.
///
-/// The input fetch is that output fetch for every operator except
+/// For these operators, the input fetch is that output fetch except for
/// [`GlobalLimitExec`], which discards `skip` rows first and therefore needs
/// `skip` more input rows. `skip` must be added *after* taking the minimum:
/// `min(fetch + skip, parent_fetch)` would be too small whenever the parent
@@ -371,7 +373,7 @@ fn pushdown_sorts_helper(
.unwrap_or(&Distribution::UnspecifiedDistribution),
);
}
- } else if let Some(adjusted) = pushdown_requirement_to_children(
+ } else if let Some(proposal) = pushdown_requirement_to_children(
&sort_push_down.plan,
parent_requirement.clone(),
parent_fetch,
@@ -379,7 +381,6 @@ fn pushdown_sorts_helper(
// For operators that can take a sort pushdown, continue with updated
// requirements. If this node already outputs single partition (e.g. SPM),
// don't push SinglePartition to children.
- let current_fetch = input_fetch(&sort_push_down.plan, parent_fetch);
let dists = sort_push_down
.plan
.input_distribution_requirements()
@@ -390,11 +391,14 @@ fn pushdown_sorts_helper(
} else {
parent_distribution.clone()
};
- for (idx, (child, order)) in
- sort_push_down.children.iter_mut().zip(adjusted).enumerate()
+ for (idx, (child, order)) in sort_push_down
+ .children
+ .iter_mut()
+ .zip(proposal.child_orderings)
+ .enumerate()
{
child.data.ordering_requirement = order;
- child.data.fetch = current_fetch;
+ child.data.fetch = proposal.child_fetch;
child.data.distribution_requirement = stronger_distribution(
&effective_dist,
dists
@@ -432,13 +436,19 @@ fn pushdown_sorts_helper(
Ok(Transformed::yes(sort_push_down))
}
+/// Child orderings and the input row bound that can accompany sort pushdown.
+struct SortPushdownProposal {
+ child_orderings: Vec<Option<OrderingRequirements>>,
+ child_fetch: Option<usize>,
+}
+
/// Calculate the pushdown ordering requirements for children.
/// If sort cannot be pushed down, return None.
fn pushdown_requirement_to_children(
plan: &Arc<dyn ExecutionPlan>,
parent_required: OrderingRequirements,
parent_fetch: Option<usize>,
-) -> Result<Option<Vec<Option<OrderingRequirements>>>> {
+) -> Result<Option<SortPushdownProposal>> {
// A parent fetch can be pushed through a plan only if the plan explicitly
// supports limit pushdown and does not change cardinality. For example, if
// LIMIT 2 is applied below a FilterExec that removes half the rows, only one
@@ -467,7 +477,7 @@ fn pushdown_requirement_to_children(
}
let maintains_input_order = plan.maintains_input_order();
- if is_window(plan) {
+ let child_orderings = if is_window(plan) {
let mut required_input_ordering = plan.required_input_ordering();
let maybe_child_requirement = required_input_ordering.swap_remove(0);
let child_plan = plan.children().swap_remove(0);
@@ -535,49 +545,21 @@ fn pushdown_requirement_to_children(
} else {
Ok(None)
}
- } else if plan.fetch().is_some()
- && plan.supports_limit_pushdown()
- && plan
- .maintains_input_order()
- .into_iter()
- .all(|maintain| maintain)
- {
- // Push down through operator with fetch when:
+ } else if plan.is::<SortPreservingMergeExec>() && plan.fetch().is_some() {
+ // Push down through a fetched merge when:
// - requirement is aligned with output ordering
// - it preserves ordering during execution
- //
- // A `ProjectionExec` reports a `fetch()` forwarded from its input and
- // can renumber/reorder columns, so the requirement (expressed in the
- // projection's output schema) must be remapped into the child schema
- // before being pushed down — forwarding it unchanged would let a key
- // such as `score@1` (valid in the output schema) refer to a different
- // column in the child schema, producing a `SortExec` whose key points
- // at the wrong column ("does not satisfy order requirements ...
- // Child-0 order: []"). If a required column maps to a computed
- // (non-`Column`) projection expression it cannot be expressed below the
- // projection, so the sort is kept above it.
- let child_required =
- if let Some(projection) = plan.downcast_ref::<ProjectionExec>() {
- match remap_requirement_through_projection(projection, &parent_required) {
- Some(remapped) => remapped,
- None => return Ok(None),
- }
- } else {
- parent_required.clone()
- };
-
- // Pushing a sort below an unordered fetch changes which rows it keeps.
- let Some(ordering) = plan.properties().output_ordering() else {
+ // Refining a fetched merge must preserve the ordering that selected its rows.
+ let Some(ordering) = plan.output_ordering() else {
return Ok(None);
};
- if plan.properties().eq_properties.requirements_compatible(
- parent_required.first().clone(),
- ordering.clone().into(),
- ) {
- Ok(Some(vec![Some(child_required)]))
- } else {
- Ok(None)
- }
+ Ok(plan
+ .equivalence_properties()
+ .requirements_compatible(
+ parent_required.first().clone(),
+ ordering.clone().into(),
+ )
+ .then(|| vec![Some(parent_required)]))
} else if is_union(plan) {
// `UnionExec` does not have real sort requirements for its input, we
// just propagate the sort requirements down:
@@ -622,14 +604,15 @@ fn pushdown_requirement_to_children(
} else if let Some(projection_exec) = plan.downcast_ref::<ProjectionExec>() {
handle_projection_pushdown(projection_exec, &parent_required)
} else if maintains_input_order.is_empty()
- || !maintains_input_order.iter().any(|o| *o)
+ || !maintains_input_order.iter().any(|maintains| *maintains)
|| plan.is::<RepartitionExec>()
|| plan.is::<FilterExec>()
|| pushdown_would_violate_requirements(&parent_required, plan.as_ref())
{
- // If the current plan is a leaf node or can not maintain any of the input ordering, can not pushed down requirements.
- // For RepartitionExec, we always choose to not push down the sort requirements even the RepartitionExec(input_partition=1) could maintain input ordering.
- // Pushing down is not beneficial
+ // Leaf nodes and nodes that preserve no input ordering cannot push
+ // requirements down to their children.
+ // Keep sorts above RepartitionExec even when a single input partition
+ // lets it preserve order; pushing the sort below it provides no benefit.
Ok(None)
} else if is_sort_preserving_merge(plan) {
let new_ordering = LexOrdering::from(parent_required.first().clone());
@@ -649,8 +632,25 @@ fn pushdown_requirement_to_children(
} else if let Some(hash_join) = plan.downcast_ref::<HashJoinExec>() {
handle_hash_join(hash_join, parent_required)
} else {
- handle_custom_pushdown(plan, parent_required, &maintains_input_order)
- }
+ // An offset or filter may need more input rows than its output fetch.
+ // Forward only the safe parent fetch; the child TopK retains its own bound.
+ return Ok(handle_custom_pushdown(
+ plan,
+ parent_required,
+ &maintains_input_order,
+ )?
+ .map(|child_orderings| SortPushdownProposal {
+ child_orderings,
+ child_fetch: parent_fetch,
+ }));
+ };
+
+ Ok(
+ child_orderings?.map(|child_orderings| SortPushdownProposal {
+ child_orderings,
+ child_fetch: input_fetch(plan, parent_fetch),
+ }),
+ )
}
/// Remap an ordering requirement expressed in a [`ProjectionExec`]'s output
@@ -966,20 +966,18 @@ fn expr_source_side(
}
}
-/// Handles the custom pushdown of parent-required sorting requirements down to
-/// the child execution plans, considering whether the input order is maintained.
+/// [`handle_projection_pushdown`] remaps columns and rejects computed projections.
+/// This generic fallback must prove its proposed input orderings are safe.
///
/// # Arguments
///
/// * `plan` - A reference to an `ExecutionPlan` for which the pushdown will be applied.
/// * `parent_required` - The sorting requirements expected by the parent node.
-/// * `maintains_input_order` - A vector of booleans indicating whether each child
-/// maintains the input order.
+/// * `maintains_input_order` - Whether the node preserves each input's row order.
///
/// # Returns
///
-/// Returns `Ok(Some(Vec<Option<LexRequirement>>))` if the sorting requirements can be
-/// pushed down, `Ok(None)` if not. On error, returns a `Result::Err`.
+/// Returns validated child orderings, or `None` to decline pushdown. Errors propagate.
fn handle_custom_pushdown(
plan: &Arc<dyn ExecutionPlan>,
parent_required: OrderingRequirements,
@@ -992,18 +990,15 @@ fn handle_custom_pushdown(
return Ok(None);
}
- // Collect all unique column indices used in the parent-required sorting
- // expression:
- let requirement = parent_required.into_single();
- let all_indices: HashSet<usize> = requirement
+ // Use the preferred alternative. Computed keys may fail on discarded rows.
+ let output_requirement = parent_required.into_single();
+ let Some(columns) = output_requirement
.iter()
- .flat_map(|order| {
- collect_columns(&order.expr)
- .iter()
- .map(|col| col.index())
- .collect::<HashSet<_>>()
- })
- .collect();
+ .map(|sort| sort.expr.downcast_ref::<Column>())
+ .collect::<Option<Vec<_>>>()
+ else {
+ return Ok(None);
+ };
// Get the number of fields in each child's schema:
let children_schema_lengths: Vec<usize> = plan_children
@@ -1011,7 +1006,7 @@ fn handle_custom_pushdown(
.map(|c| c.schema().fields().len())
.collect();
- // Find the index of the order-maintaining child:
+ // Use the first order-maintaining input for this candidate mapping.
let Some(maintained_child_idx) = maintains_input_order
.iter()
.enumerate()
@@ -1021,61 +1016,178 @@ fn handle_custom_pushdown(
return Ok(None);
};
- // Check if all required columns come from the order-maintaining child:
+ // Matching column positions is only a candidate; prove output ordering below.
let start_idx = children_schema_lengths[..maintained_child_idx]
.iter()
.sum::<usize>();
let end_idx = start_idx + children_schema_lengths[maintained_child_idx];
- let all_from_maintained_child =
- all_indices.iter().all(|i| i >= &start_idx && i < &end_idx);
+ let fits_maintained_input = columns
+ .iter()
+ .all(|col| (start_idx..end_idx).contains(&col.index()));
- // If all columns are from the maintained child, update the parent requirements:
- if all_from_maintained_child {
- let sub_offset = children_schema_lengths
- .iter()
- .take(maintained_child_idx)
- .sum::<usize>();
- // Transform the parent-required expression for the child schema by
- // adjusting columns:
- let updated_parent_req = requirement
- .into_iter()
- .map(|req| {
- let child_schema = plan_children[maintained_child_idx].schema();
- let updated_columns = req
- .expr
- .transform_up(|expr| {
- if let Some(col) = expr.downcast_ref::<Column>() {
- let new_index = col.index() - sub_offset;
- Ok(Transformed::yes(Arc::new(Column::new(
- child_schema.field(new_index).name(),
- new_index,
- ))))
- } else {
- Ok(Transformed::no(expr))
- }
- })?
- .data;
- Ok(PhysicalSortRequirement::new(updated_columns, req.options))
- })
- .collect::<Result<Vec<_>>>()?;
+ if !fits_maintained_input {
+ return Ok(None);
+ }
- // Prepare the result, populating with the updated requirements for children that maintain order
- let result = maintains_input_order
- .iter()
- .map(|&maintains_order| {
- if maintains_order {
- LexRequirement::new(updated_parent_req.clone())
- .map(OrderingRequirements::new)
- } else {
- None
- }
- })
- .collect();
+ // Remap the columns into the maintained child's schema.
+ let child_schema = plan_children[maintained_child_idx].schema();
+ let mapped_requirement = columns
+ .iter()
+ .zip(output_requirement.iter())
+ .map(|(col, req)| {
+ let index = col.index() - start_idx;
+ let col = Column::new(child_schema.field(index).name(), index);
+ PhysicalSortRequirement::new(Arc::new(col), req.options)
+ })
+ .collect::<Vec<_>>();
- Ok(Some(result))
+ // Propose the same ordering for every order-maintaining input.
+ let child_requirements = maintains_input_order
+ .iter()
+ .map(|&maintains_order| {
+ if maintains_order {
+ LexRequirement::new(mapped_requirement.clone())
+ .map(OrderingRequirements::new)
+ } else {
+ None
+ }
+ })
+ .collect::<Vec<_>>();
+
+ // Row positions can stay unchanged while value order reverses (`-a AS a`).
+ // Prove that the candidate satisfies both input and output requirements.
+ if !is_custom_sort_pushdown_valid(plan, &child_requirements, output_requirement)? {
+ return Ok(None);
+ }
+ Ok(Some(child_requirements))
+}
+
+/// Rebuild with temporary [`SortExec`] inputs to recompute the node's properties.
+/// Validate input requirements and output ordering without executing these plans.
+fn is_custom_sort_pushdown_valid(
+ plan: &Arc<dyn ExecutionPlan>,
+ child_requirements: &[Option<OrderingRequirements>],
+ output_requirement: LexRequirement,
+) -> Result<bool> {
+ let plan_children = plan.children();
+
+ let selecting_sort = if plan.fetch().is_some() {
+ // Sorting below a fetch can change which rows it selects. Only refine
+ // a SortExec directly below this boundary, as for native limits.
+ let ([child], [Some(_)]) = (plan_children.as_slice(), child_requirements) else {
+ return Ok(false);
+ };
+ let Some(sort) = child.downcast_ref::<SortExec>() else {
+ return Ok(false);
+ };
+ Some(sort)
} else {
- Ok(None)
+ None
+ };
+
+ let mut probe_children = Vec::with_capacity(plan_children.len());
+ for (child, required) in plan_children.iter().zip(child_requirements) {
+ let Some(required) = required else {
+ probe_children.push(Arc::clone(child));
+ continue;
+ };
+ // Other order-maintaining inputs may have narrower schemas.
+ let input_width = child.schema().fields().len();
+ if required.first().iter().any(|sort| {
+ collect_columns(&sort.expr)
+ .iter()
+ .any(|column| column.index() >= input_width)
+ }) {
+ return Ok(false);
+ }
+ let ordering: LexOrdering = required.first().clone().into();
+ let child_properties = child.equivalence_properties();
+ if let Some(sort) = selecting_sort {
+ // Preserve every key that selected the fetched rows, including hidden keys.
+ // Clear old ordering metadata so the proposal must prove this on its own.
+ let mut proposed_properties = child_properties.clone();
+ proposed_properties.clear_orderings();
+ proposed_properties.add_ordering(ordering.clone());
+ if !proposed_properties.ordering_satisfy(sort.expr().clone())? {
+ return Ok(false);
+ }
+ }
+ // A new blocking sort would prevent a bounded node from stopping an
+ // unbounded input. Already ordered inputs need no new sort.
+ if child.boundedness().is_unbounded()
+ && !child_properties.ordering_satisfy_requirement(required.first().clone())?
+ {
+ return Ok(false);
+ }
+ probe_children.push(Arc::new(
+ SortExec::new(ordering, Arc::clone(child)).with_preserve_partitioning(true),
+ ));
+ }
+
+ // The input plan already satisfies distribution, which these sorts preserve.
+ // Check ordering before rebuilding: the node may reject incompatible children.
+ let children = probe_children.iter().map(Arc::as_ref).collect::<Vec<_>>();
+ if !sort_pushdown_input_orderings_satisfied(plan.as_ref(), &children)? {
+ return Ok(false);
+ }
+
+ // Rebuild with fresh properties to see how the proposed orderings pass through.
+ let probe = Arc::clone(plan).replace_children(
+ probe_children,
+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),
+ )?;
+
+ // Rebuilding may change input requirements. Prove those and the output ordering.
+ Ok(sort_pushdown_inputs_satisfied(probe.as_ref())?
+ && probe
+ .equivalence_properties()
+ .ordering_satisfy_requirement(output_requirement)?)
+}
+
+/// Validate ordering separately so it can be checked before rebuilding a node.
+fn sort_pushdown_input_orderings_satisfied(
+ plan: &dyn ExecutionPlan,
+ children: &[&dyn ExecutionPlan],
+) -> Result<bool> {
+ let ordering = plan.required_input_ordering();
+ if ordering.len() != children.len() {
+ return Ok(false);
+ }
+ for (child, required) in children.iter().zip(ordering) {
+ if let Some(required) = required
+ && !child
+ .equivalence_properties()
+ .ordering_satisfy_requirement(required.first().clone())?
+ {
+ return Ok(false);
+ }
+ }
+ Ok(true)
+}
+
+/// Checks input ordering, per-child distribution, and co-partitioning.
+fn sort_pushdown_inputs_satisfied(plan: &dyn ExecutionPlan) -> Result<bool> {
+ let children = plan
+ .children()
+ .into_iter()
+ .map(Arc::as_ref)
+ .collect::<Vec<_>>();
+ let distribution = plan.input_distribution_requirements();
+ if distribution.per_child_distributions().len() != children.len()
+ || !sort_pushdown_input_orderings_satisfied(plan, &children)?
+ {
+ return Ok(false);
}
+ let options = ChildSatisfactionOptions::new().with_allow_subset(true);
+ for (index, child) in children.iter().enumerate() {
+ let satisfaction = distribution.child_satisfaction(index, *child, options)?;
+ if !satisfaction.is_satisfied() {
+ return Ok(false);
+ }
+ }
+ Ok(distribution
+ .unsatisfied_co_partitioned_children(plan.name(), &children)?
+ .is_empty())
}
// For hash join we only maintain the input order for the right child
@@ -1218,9 +1330,15 @@ fn handle_projection_pushdown(
.iter()
.any(|expr| !expr.expr.is::<Column>())
{
+ // Computed projection expressions are not supported by this
+ // remapping, so keep the sort above the projection.
return Ok(None);
}
+ // A `ProjectionExec` can renumber/reorder columns, so the requirement
+ // (expressed in the projection's output schema) must be remapped into the
+ // child schema before being pushed down. Forwarding `score@1` unchanged
+ // could refer to a different input column and sort on the wrong key.
Ok(
remap_requirement_through_projection(projection_exec, parent_required)
.map(|requirements| vec![Some(requirements)]),
@@ -1234,10 +1352,16 @@ mod tests {
use arrow::compute::SortOptions;
use arrow::datatypes::{DataType, Field, Schema};
use datafusion_expr::Operator;
+ use datafusion_expr::physical_planning_context::{
+ ScalarSubqueryResults, SubqueryIndex,
+ };
use datafusion_physical_expr::PhysicalExpr;
use datafusion_physical_expr::expressions::{BinaryExpr, col};
use datafusion_physical_plan::empty::EmptyExec;
use datafusion_physical_plan::limit::GlobalLimitExec;
+ use datafusion_physical_plan::scalar_subquery::{
+ ScalarSubqueryExec, ScalarSubqueryLink,
+ };
const DESC: SortOptions = SortOptions {
descending: true,
@@ -1353,6 +1477,30 @@ mod tests {
LexRequirement::new(reqs).unwrap()
}
+ #[test]
+ fn scalar_subquery_sort_applies_only_to_the_main_input() -> Result<()> {
+ let schema = child_schema();
+ let input: Arc<dyn ExecutionPlan> = Arc::new(EmptyExec::new(Arc::clone(&schema)));
+ let requirement = OrderingRequirements::new(lex([
+ req("c", &schema, DESC),
+ req("a", &schema, ASC),
+ ]));
+ let plan: Arc<dyn ExecutionPlan> = Arc::new(ScalarSubqueryExec::new(
+ Arc::clone(&input),
+ vec![ScalarSubqueryLink {
+ plan: Arc::clone(&input),
+ index: SubqueryIndex::new(0),
+ }],
+ ScalarSubqueryResults::new(1),
+ ));
+ let proposal =
+ pushdown_requirement_to_children(&plan, requirement.clone(), None)?
+ .expect("the main input supports sort pushdown");
+ assert_eq!(proposal.child_orderings, vec![Some(requirement), None]);
+ assert_eq!(proposal.child_fetch, None);
+ Ok(())
+ }
+
#[test]
fn remap_single_hard_requirement_through_reordering_projection() {
let projection = reordering_projection();
diff --git a/datafusion/sqllogictest/src/test_context.rs b/datafusion/sqllogictest/src/test_context.rs
index 9196a92243..c85642bb0a 100644
--- a/datafusion/sqllogictest/src/test_context.rs
+++ b/datafusion/sqllogictest/src/test_context.rs
@@ -33,6 +33,7 @@ use datafusion::logical_expr::{
ColumnarValue, Expr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature,
Volatility, create_udf,
};
+use datafusion::physical_optimizer::optimizer::PhysicalOptimizer;
use datafusion::physical_plan::ExecutionPlan;
use datafusion::prelude::*;
use datafusion::{
@@ -60,6 +61,7 @@ use log::info;
use sqlparser::ast;
use tempfile::TempDir;
+mod custom_sort_pushdown;
mod range_partitioning;
/// Context for running tests
@@ -100,6 +102,7 @@ impl TestContext {
/// If `None` is returned (e.g. because some needed feature is not
/// enabled), the file should be skipped
pub async fn try_new_for_test_file(relative_path: &Path) -> Option<Self> {
+ let file_name = relative_path.file_name().unwrap().to_str().unwrap();
let config = SessionConfig::new()
// hardcode target partitions so plans are deterministic
.with_target_partitions(4);
@@ -163,12 +166,30 @@ impl TestContext {
state_builder = state_builder.with_statistics_registry(registry);
}
+ if file_name == "sort_pushdown.slt" {
+ let mut rules = PhysicalOptimizer::new().rules;
+ let ensure_requirements = rules
+ .iter()
+ .position(|rule| rule.name() == "EnsureRequirements")
+ .expect("default optimizer includes EnsureRequirements");
+ rules.insert(
+ ensure_requirements,
+ Arc::new(custom_sort_pushdown::CustomLimitRule),
+ );
+ state_builder = state_builder.with_physical_optimizer_rules(rules);
+ }
+
let state = state_builder.build();
let mut test_ctx = TestContext::new(SessionContext::new_with_state(state));
- let file_name = relative_path.file_name().unwrap().to_str().unwrap();
match file_name {
+ "sort_pushdown.slt" => {
+ custom_sort_pushdown::register_custom_sort_pushdown(
+ test_ctx.session_ctx(),
+ )
+ .expect("register custom projection table");
+ }
"parquet_missing_bounds.slt" => {
register_parquet_missing_bounds(&mut test_ctx).await;
}
diff --git a/datafusion/sqllogictest/src/test_context/custom_sort_pushdown.rs b/datafusion/sqllogictest/src/test_context/custom_sort_pushdown.rs
new file mode 100644
index 0000000000..cc50db4974
--- /dev/null
+++ b/datafusion/sqllogictest/src/test_context/custom_sort_pushdown.rs
@@ -0,0 +1,266 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
+
+use std::fmt::{self, Formatter};
+use std::sync::Arc;
+
+use arrow::array::record_batch;
+use arrow::datatypes::{self as arrow_schema, SchemaRef};
+use async_trait::async_trait;
+use datafusion::catalog::{Session, TableProvider};
+use datafusion::common::Result;
+use datafusion::common::config::ConfigOptions;
+use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};
+use datafusion::execution::{SendableRecordBatchStream, TaskContext};
+use datafusion::logical_expr::{Expr, Operator, TableType};
+use datafusion::physical_expr::expressions::{binary, col, lit, negative};
+use datafusion::physical_expr::projection::ProjectionExprs;
+use datafusion::physical_expr::{PhysicalExpr, PhysicalSortExpr};
+use datafusion::physical_optimizer::PhysicalOptimizerRule;
+use datafusion::physical_plan::filter::{FilterExec, FilterExecBuilder};
+use datafusion::physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
+use datafusion::physical_plan::projection::ProjectionExec;
+use datafusion::physical_plan::sorts::sort::SortExec;
+use datafusion::physical_plan::test::TestMemoryExec;
+use datafusion::physical_plan::{
+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan,
+ ExecutionPlanProperties, InputDistributionRequirements, PlanProperties,
+ ReplaceChildrenOptions,
+};
+use datafusion::prelude::SessionContext;
+
+/// Register custom plans whose native implementations are hidden from the optimizer.
+pub(super) fn register_custom_sort_pushdown(ctx: &SessionContext) -> Result<()> {
+ let batch = record_batch!(("a", Int32, [2, 1, 3]))?;
+ let schema = batch.schema();
+ let input = TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?;
+ let projection = ProjectionExec::try_new(
+ [(negative(col("a", &schema)?, &schema)?, "a".to_string())],
+ input,
+ )?;
+ ctx.register_table(
+ "custom_negated_projection",
+ Arc::new(CustomExec(Arc::new(projection))),
+ )?;
+
+ let batch = record_batch!(("a", Int32, [2, 1, 3]), ("b", Int32, [10, 30, 20]))?;
+ let schema = batch.schema();
+ let input = TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?;
+ let projection = ProjectionExec::try_new(
+ [
+ (col("b", &schema)?, "a".to_string()),
+ (col("a", &schema)?, "b".to_string()),
+ ],
+ input,
+ )?;
+ ctx.register_table(
+ "custom_reordered_projection",
+ Arc::new(CustomExec(Arc::new(projection))),
+ )?;
+
+ let batch = record_batch!(("t", Int32, [1, 2, 3]), ("x", Int32, [0, 2, 4]))?;
+ let schema = batch.schema();
+ let input = TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?;
+ let predicate = binary(col("x", &schema)?, Operator::NotEq, lit(0i32), &schema)?;
+ ctx.register_table(
+ "custom_filtered_input",
+ Arc::new(CustomExec(Arc::new(FilterExec::try_new(predicate, input)?))),
+ )?;
+
+ let batch = record_batch!(("a", Int32, [1, 2, 3]), ("b", Int32, [30, 20, 10]))?;
+ let schema = batch.schema();
+ let ordering =
+ [PhysicalSortExpr::new_default(col("a", &schema)?).nulls_last()].into();
+ let input = Arc::new(
+ TestMemoryExec::try_new(&[vec![batch]], Arc::clone(&schema), None)?
+ .try_with_sort_information(vec![ordering])?,
+ );
+ ctx.register_table("custom_ordered_input", Arc::new(CustomExec(input)))?;
+
+ // The custom node hides the final sort key while selecting its first row.
+ // Refining its visible ordering must still preserve that hidden key.
+ let batch = record_batch!(
+ ("a", Int32, [1, 1]),
+ ("c", Int32, [10, 20]),
+ ("b", Int32, [2, 1])
+ )?;
+ let schema = batch.schema();
+ let input = TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?;
+ let ordering = [
+ PhysicalSortExpr::new_default(col("a", &schema)?).nulls_last(),
+ PhysicalSortExpr::new_default(col("b", &schema)?).nulls_last(),
+ ]
+ .into();
+ // Keep the selection ordering through earlier sort-removal passes.
+ let input = Arc::new(SortExec::new(ordering, input).with_fetch(Some(2)));
+ let projected_limit = FilterExecBuilder::new(lit(true), input)
+ .apply_projection(Some(vec![0, 1]))?
+ .with_fetch(Some(1))
+ .build()?;
+ ctx.register_table(
+ "custom_projected_limit",
+ Arc::new(CustomExec(Arc::new(projected_limit))),
+ )?;
+
+ // The output fetch excludes the skipped row, so it cannot also bound input.
+ let batch = record_batch!(("a", Int32, [2, 1, 3]), ("b", Int32, [20, 30, 10]))?;
+ let schema = batch.schema();
+ let input = TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?;
+ let ordering =
+ [PhysicalSortExpr::new_default(col("a", &schema)?).nulls_last()].into();
+ let input = Arc::new(SortExec::new(ordering, input).with_fetch(Some(3)));
+ let limit = GlobalLimitExec::new(input, 1, Some(1));
+ ctx.register_table("custom_offset_limit", Arc::new(CustomExec(Arc::new(limit))))?;
+ Ok(())
+}
+
+/// Replace a redundant sort on the ordered test input with a custom limit, so SQL
+/// `ORDER BY ... LIMIT` exercises the generic sort-pushdown path.
+#[derive(Debug)]
+pub(super) struct CustomLimitRule;
+
+impl PhysicalOptimizerRule for CustomLimitRule {
+ fn optimize(
+ &self,
+ plan: Arc<dyn ExecutionPlan>,
+ _: &ConfigOptions,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ plan.transform_up(|plan| {
+ let Some(sort) = plan.downcast_ref::<SortExec>() else {
+ return Ok(Transformed::no(plan));
+ };
+ let Some(fetch) = sort.fetch() else {
+ return Ok(Transformed::no(plan));
+ };
+ let input = sort.input();
+ let source = input
+ .downcast_ref::<ProjectionExec>()
+ .map_or(input, |projection| projection.input());
+ let Some(source) = source.downcast_ref::<CustomExec>() else {
+ return Ok(Transformed::no(plan));
+ };
+ // Only the dedicated ordered source is a wrapped memory scan.
+ if !source.0.is::<TestMemoryExec>()
+ || input.output_partitioning().partition_count() != 1
+ || !input
+ .equivalence_properties()
+ .ordering_satisfy(sort.expr().clone())?
+ {
+ return Ok(Transformed::no(plan));
+ }
+ let limit = LocalLimitExec::new(Arc::clone(input), fetch);
+ let plan: Arc<dyn ExecutionPlan> = Arc::new(CustomExec(Arc::new(limit)));
+ Ok(Transformed::yes(plan))
+ })
+ .map(|transformed| transformed.data)
+ }
+
+ fn name(&self) -> &str {
+ "CustomLimitRule"
+ }
+
+ fn schema_check(&self) -> bool {
+ true
+ }
+}
+
+/// Exposes a native plan's children and properties under a distinct plan type,
+/// exercising the generic custom-plan sort pushdown path.
+#[derive(Debug, Clone)]
+struct CustomExec(Arc<dyn ExecutionPlan>);
+
+#[async_trait]
+impl TableProvider for CustomExec {
+ fn schema(&self) -> SchemaRef {
+ self.0.schema()
+ }
+
+ fn table_type(&self) -> TableType {
+ TableType::Base
+ }
+
+ async fn scan(
+ &self,
+ _: &dyn Session,
+ projection: Option<&[usize]>,
+ _: &[Expr],
+ _: Option<usize>,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ let plan: Arc<dyn ExecutionPlan> = Arc::new(self.clone());
+ // Keep identity projections out of the plan so sort pushdown reaches
+ // this custom node before native projection-specific handling.
+ if let Some(projection) = projection
+ && !projection
+ .iter()
+ .copied()
+ .eq(0..self.0.schema().fields().len())
+ {
+ let expressions = ProjectionExprs::from_indices(projection, &self.0.schema());
+ Ok(Arc::new(ProjectionExec::try_new(
+ expressions.iter().cloned(),
+ plan,
+ )?))
+ } else {
+ Ok(plan)
+ }
+ }
+}
+
+impl DisplayAs for CustomExec {
+ fn fmt_as(&self, display_type: DisplayFormatType, f: &mut Formatter) -> fmt::Result {
+ write!(f, "Custom")?;
+ self.0.fmt_as(display_type, f)
+ }
+}
+
+impl ExecutionPlan for CustomExec {
+ fn name(&self) -> &str {
+ "CustomExec"
+ }
+
+ fn properties(&self) -> &Arc<PlanProperties> {
+ self.0.properties()
+ }
+
+ fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
+ self.0.children()
+ }
+
+ fn maintains_input_order(&self) -> Vec<bool> {
+ self.0.maintains_input_order()
+ }
+
+ fn input_distribution_requirements(&self) -> InputDistributionRequirements {
+ self.0.input_distribution_requirements()
+ }
+
+ fn fetch(&self) -> Option<usize> {
+ self.0.fetch()
+ }
+
+ fn with_new_children(
+ self: Arc<Self>,
+ children: Vec<Arc<dyn ExecutionPlan>>,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ Ok(Arc::new(Self(Arc::clone(&self.0).replace_children(
+ children,
+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),
+ )?)))
+ }
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ self.0.apply_expressions(f)
+ }
+
+ fn execute(
+ &self,
+ partition: usize,
+ context: Arc<TaskContext>,
+ ) -> Result<SendableRecordBatchStream> {
+ self.0.execute(partition, context)
+ }
+}
diff --git a/datafusion/sqllogictest/test_files/sort_pushdown.slt b/datafusion/sqllogictest/test_files/sort_pushdown.slt
index 6b6ab2cca1..33b9f45e5b 100644
--- a/datafusion/sqllogictest/test_files/sort_pushdown.slt
+++ b/datafusion/sqllogictest/test_files/sort_pushdown.slt
@@ -2,6 +2,153 @@
# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
+# The custom operator evaluates -a AS a, preserving row positions while
+# reversing the ordering of values. Its input is [2, 1, 3].
+statement ok
+SET datafusion.execution.target_partitions = 1;
+
+query I
+SELECT a FROM custom_negated_projection ORDER BY a;
+----
+-3
+-2
+-1
+
+# The sort must stay above the custom projection.
+statement ok
+SET datafusion.explain.physical_plan_only = true;
+
+query TT
+EXPLAIN SELECT a FROM custom_negated_projection ORDER BY a;
+----
+physical_plan
+01)SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false]
+02)--CustomProjectionExec: expr=[(- a@0) as a]
+03)----CooperativeExec
+04)------DataSourceExec: partitions=1, partition_sizes=[1]
+
+# Reordering columns must not make the generic path sort the wrong input column.
+query II
+SELECT a, b FROM custom_reordered_projection ORDER BY a;
+----
+10 2
+20 3
+30 1
+
+query TT
+EXPLAIN SELECT a, b FROM custom_reordered_projection ORDER BY a;
+----
+physical_plan
+01)SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false]
+02)--CustomProjectionExec: expr=[b@1 as a, a@0 as b]
+03)----CooperativeExec
+04)------DataSourceExec: partitions=1, partition_sizes=[1]
+
+# Sorting before the custom filter would evaluate 1 / x on the rejected row
+# where x = 0, failing the query with an integer division-by-zero error.
+query II
+SELECT t, x FROM custom_filtered_input ORDER BY t, 1 / x;
+----
+2 2
+3 4
+
+query TT
+EXPLAIN SELECT t, x FROM custom_filtered_input ORDER BY t, 1 / x;
+----
+physical_plan
+01)SortExec: expr=[t@0 ASC NULLS LAST, 1 / CAST(x@1 AS Int64) ASC NULLS LAST], preserve_partitioning=[false]
+02)--CustomFilterExec: x@1 != 0
+03)----CooperativeExec
+04)------DataSourceExec: partitions=1, partition_sizes=[1]
+
+# The inner ORDER BY determines which rows LIMIT selects. The outer sort
+# must keep those rows, even when LIMIT is implemented by a custom node.
+query II
+SELECT a, b
+FROM (SELECT a, b FROM custom_ordered_input ORDER BY a LIMIT 2) s
+ORDER BY b;
+----
+2 20
+1 30
+
+query TT
+EXPLAIN SELECT a, b
+FROM (SELECT a, b FROM custom_ordered_input ORDER BY a LIMIT 2) s
+ORDER BY b;
+----
+physical_plan
+01)SortExec: expr=[b@1 ASC NULLS LAST], preserve_partitioning=[false]
+02)--CustomLocalLimitExec: fetch=2
+03)----CooperativeExec
+04)------CustomDataSourceExec: partitions=1, partition_sizes=[1], output_ordering=a@0 ASC NULLS LAST
+
+# The custom node sorts by (a, b), keeps one row, and hides b. Ordering its
+# output by (a, c) must not replace the hidden b tie-breaker that selected c=20.
+query II
+SELECT a, c FROM custom_projected_limit ORDER BY a, c;
+----
+1 20
+
+query TT
+EXPLAIN SELECT a, c FROM custom_projected_limit ORDER BY a, c;
+----
+physical_plan
+01)SortExec: expr=[a@0 ASC NULLS LAST, c@1 ASC NULLS LAST], preserve_partitioning=[false]
+02)--CustomFilterExec: true, projection=[a@0, c@1], fetch=1
+03)----SortExec: TopK(fetch=2), expr=[a@0 ASC NULLS LAST, b@2 ASC NULLS LAST], preserve_partitioning=[false]
+04)------CooperativeExec
+05)--------DataSourceExec: partitions=1, partition_sizes=[1]
+
+# Refining (a) to (a, b) below the custom OFFSET 1 LIMIT 1 is safe, but its
+# output fetch=1 must not tighten TopK(3) to TopK(1): the offset needs a second row.
+query II
+SELECT a, b FROM custom_offset_limit ORDER BY a, b;
+----
+2 20
+
+query TT
+EXPLAIN SELECT a, b FROM custom_offset_limit ORDER BY a, b;
+----
+physical_plan
+01)CustomGlobalLimitExec: skip=1, fetch=1
+02)--SortExec: TopK(fetch=3), expr=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST], preserve_partitioning=[false]
+03)----CooperativeExec
+04)------DataSourceExec: partitions=1, partition_sizes=[1]
+
+# The aggregate's amount ordering can refine the inner TopK's country
+# ordering. Keep one TopK with both keys, preserving its original fetch=3.
+statement ok
+CREATE TABLE sort_pushdown_sales(country VARCHAR, amount INT) AS VALUES
+('B', 20), ('A', 10), ('C', 40), ('A', 30);
+
+query T? rowsort
+SELECT s.country, ARRAY_AGG(s.amount ORDER BY s.amount DESC) AS amounts
+FROM (SELECT * FROM sort_pushdown_sales ORDER BY country LIMIT 3) AS s
+GROUP BY s.country;
+----
+A [30, 10]
+B [20]
+
+query TT
+EXPLAIN SELECT s.country, ARRAY_AGG(s.amount ORDER BY s.amount DESC) AS amounts
+FROM (SELECT * FROM sort_pushdown_sales ORDER BY country LIMIT 3) AS s
+GROUP BY s.country;
+----
+physical_plan
+01)ProjectionExec: expr=[country@0 as country, array_agg(s.amount) ORDER BY [s.amount DESC NULLS FIRST]@1 as amounts]
+02)--AggregateExec: mode=Single, gby=[country@0 as country], aggr=[array_agg(s.amount) ORDER BY [s.amount DESC NULLS FIRST]], group_clustering_mode=Full
+03)----SortExec: TopK(fetch=3), expr=[country@0 ASC NULLS LAST, amount@1 DESC], preserve_partitioning=[false]
+04)------DataSourceExec: partitions=1, partition_sizes=[1]
+
+statement ok
+DROP TABLE sort_pushdown_sales;
+
+statement ok
+SET datafusion.explain.physical_plan_only = false;
+
+statement ok
+SET datafusion.execution.target_partitions = 4;
+
#Sort Pushdown for ordered Parquet files
statement ok
SET datafusion.execution.parquet.pushdown_filters = true;