Apache DataFusion SQL Query Engine
fix: validate sort pushdown through custom execution plans (#26118)
Geoffrey Claude390622a030de
diff --git a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs b/datafusion/core/tests/physical_optimizer/enforce_sorting.rsindex 49ac9c9dc4..05959bc62d 100644--- a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs+++ b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs@@ -13,7 +13,7 @@ use crate::physical_optimizer::test_utils::{parquet_exec_with_sort, projection_exec, repartition_exec, sort_exec,sort_exec_with_fetch, sort_expr, sort_expr_options, sort_merge_join_exec,sort_preserving_merge_exec, sort_preserving_merge_exec_with_fetch,- spr_repartition_exec, stream_exec_ordered, union_exec,+ spr_repartition_exec, stream_exec, stream_exec_ordered, union_exec,};use arrow::compute::{SortOptions};@@ -2775,6 +2775,68 @@ async fn test_push_with_required_input_ordering_allowed() -> Result<()> {Ok(())}+#[tokio::test]+async fn test_push_with_required_input_ordering_conflicting_suffix() -> Result<()> {+ let schema = create_test_schema3()?;+ let required: LexOrdering = [sort_expr("a", &schema), sort_expr("b", &schema)].into();+ let requested = [sort_expr("a", &schema), sort_expr("c", &schema)].into();+ let input = sort_exec(required.clone(), memory_exec(&schema));+ let plan = RequirementsTestExec::new(input)+ .with_required_input_ordering(Some(required))+ .into_arc();+ let plan = sort_exec(requested, plan);+ let test = EnforceSortingTest::new(plan).with_repartition_sorts(true);++ assert_snapshot!(test.run(), @r"+ Input / Optimized Plan:+ SortExec: expr=[a@0 ASC, c@2 ASC], preserve_partitioning=[false]+ RequiredInputOrderingExec+ SortExec: expr=[a@0 ASC, b@1 ASC], preserve_partitioning=[false]+ DataSourceExec: partitions=1, partition_sizes=[0]+ ");+ Ok(())+}++#[test]+fn test_custom_sort_pushdown_keeps_sort_above_unbounded_input() -> Result<()> {+ let schema = create_test_schema3()?;+ // Keep the custom node directly below the sort. It bounds the stream+ // without advertising a fetch, so this reaches the unbounded-input guard.+ let custom = RequirementsTestExec::new(stream_exec(&schema))+ .with_required_input_distribution(Distribution::SinglePartition)+ .with_bounded_output()+ .into_arc();+ let plan = sort_exec([sort_expr("a", &schema)].into(), custom);++ assert_snapshot!(EnforceSortingTest::new(plan).run(), @r"+ Input / Optimized Plan:+ SortExec: expr=[a@0 ASC], preserve_partitioning=[false]+ RequiredInputOrderingExec+ StreamingTableExec: partition_sizes=1, projection=[a, b, c, d, e], infinite_source=true+ ");+ Ok(())+}++#[test]+fn test_custom_sort_pushdown_rejects_narrower_order_maintaining_input() -> Result<()> {+ let schema = create_test_schema3()?;+ let narrow_schema = Arc::new(schema.project(&[0])?);+ // b@1 is valid in the first input and output, but not the second input.+ let custom = RequirementsTestExec::new(memory_exec(&schema))+ .with_additional_input(memory_exec(&narrow_schema))+ .into_arc();+ let plan = sort_exec([sort_expr("b", &schema)].into(), custom);++ assert_snapshot!(EnforceSortingTest::new(plan).run(), @r"+ Input / Optimized Plan:+ SortExec: expr=[b@1 ASC], preserve_partitioning=[false]+ RequiredInputOrderingExec+ DataSourceExec: partitions=1, partition_sizes=[0]+ DataSourceExec: partitions=1, partition_sizes=[0]+ ");+ Ok(())+}+#[tokio::test]async fn test_replace_with_partial_sort() -> Result<()> {let schema = create_test_schema3()?;diff --git a/datafusion/core/tests/physical_optimizer/pushdown_sort.rs b/datafusion/core/tests/physical_optimizer/pushdown_sort.rsindex aa1aebfb23..4a81d11949 100644--- a/datafusion/core/tests/physical_optimizer/pushdown_sort.rs+++ b/datafusion/core/tests/physical_optimizer/pushdown_sort.rs@@ -21,8 +21,10 @@ use datafusion_physical_expr::expressions;use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use datafusion_physical_expr_common::sort_expr::LexOrdering;use datafusion_physical_optimizer::PhysicalOptimizerRule;+use datafusion_physical_optimizer::ensure_requirements::EnsureRequirements;use datafusion_physical_optimizer::pushdown_sort::PushdownSort;-use datafusion_physical_plan::collect;+use datafusion_physical_plan::sorts::sort::SortExec;+use datafusion_physical_plan::{ExecutionPlan, collect};use std::sync::Arc;use crate::physical_optimizer::test_utils::{@@ -1124,6 +1126,29 @@ fn test_sort_pushdown_exact_no_fetch_no_limit() {);}+#[test]+fn test_custom_plan_with_fetch_keeps_outer_sort() -> Result<()> {+ let schema = schema();+ let a = sort_expr("a", &schema);+ let source = TestScan::new(schema.clone(), vec![[a.clone()].into()])+ .with_supports_fetch(true)+ .with_fetch(Some(10))+ .unwrap();+ let ordering = LexOrdering::new(vec![a, sort_expr("b", &schema)]).unwrap();+ let plan = sort_exec(ordering.clone(), source);++ let mut config = ConfigOptions::new();+ config.execution.target_partitions = 1;+ let optimized = EnsureRequirements::new().optimize(plan, &config)?;+ let sort = optimized+ .downcast_ref::<SortExec>()+ .expect("a fetched custom plan must retain the requested outer sort");+ assert_eq!(sort.expr(), &ordering);+ assert!(sort.input().is::<TestScan>());+ assert_eq!(sort.input().fetch(), Some(10));+ Ok(())+}+#[test]fn test_sort_pushdown_exact_preserves_fetch_with_global_limit() {// When a source returns Exact but does NOT support with_fetch(),diff --git a/datafusion/core/tests/physical_optimizer/test_utils.rs b/datafusion/core/tests/physical_optimizer/test_utils.rsindex a5c750c79e..b93281def2 100644--- a/datafusion/core/tests/physical_optimizer/test_utils.rs+++ b/datafusion/core/tests/physical_optimizer/test_utils.rs@@ -421,25 +421,44 @@ pub fn projection_exec(Ok(Arc::new(ProjectionExec::try_new(proj_exprs, input)?))}-/// A test [`ExecutionPlan`] whose requirements can be configured.+/// A test [`ExecutionPlan`] with configurable input requirements.+/// Its output properties follow its first input unless boundedness is overridden.#[derive(Debug)]pub struct RequirementsTestExec {required_input_ordering: Option<LexOrdering>,required_input_distribution: Distribution,maintains_input_order: bool,- input: Arc<dyn ExecutionPlan>,+ inputs: Vec<Arc<dyn ExecutionPlan>>,+ bounded_output: bool,+ cache: Arc<PlanProperties>,}impl RequirementsTestExec {pub fn new(input: Arc<dyn ExecutionPlan>) -> Self {+ let cache = Arc::clone(input.properties());Self {required_input_ordering: None,required_input_distribution: Distribution::UnspecifiedDistribution,maintains_input_order: true,- input,+ inputs: vec![input],+ bounded_output: false,+ cache,}}+ /// Add another input with the same requirements and order-preservation flag.+ pub fn with_additional_input(mut self, input: Arc<dyn ExecutionPlan>) -> Self {+ self.inputs.push(input);+ self+ }++ /// Model a custom operator that stops reading an unbounded input.+ pub fn with_bounded_output(mut self) -> Self {+ self.bounded_output = true;+ Arc::make_mut(&mut self.cache).boundedness = Boundedness::Bounded;+ self+ }+/// sets the required input orderingpub fn with_required_input_ordering(mut self,@@ -490,27 +509,31 @@ impl ExecutionPlan for RequirementsTestExec {}fn properties(&self) -> &Arc<PlanProperties> {- self.input.properties()+ &self.cache}fn required_input_ordering(&self) -> Vec<Option<OrderingRequirements>> {vec![self.required_input_ordering.as_ref()- .map(|ordering| OrderingRequirements::from(ordering.clone())),+ .map(|ordering| OrderingRequirements::from(ordering.clone()));+ self.inputs.len()]}fn input_distribution_requirements(&self) -> InputDistributionRequirements {- InputDistributionRequirements::new(vec![self.required_input_distribution.clone()])+ InputDistributionRequirements::new(vec![+ self.required_input_distribution.clone();+ self.inputs.len()+ ])}fn maintains_input_order(&self) -> Vec<bool> {- vec![self.maintains_input_order]+ vec![self.maintains_input_order; self.inputs.len()]}fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {- vec![&self.input]+ self.inputs.iter().collect()}fn replace_children(@@ -518,12 +541,16 @@ impl ExecutionPlan for RequirementsTestExec {children: Vec<Arc<dyn ExecutionPlan>>,_: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {- assert_eq!(children.len(), 1);- Ok(RequirementsTestExec::new(Arc::clone(&children[0]))+ assert_eq!(children.len(), self.inputs.len());+ let mut plan = RequirementsTestExec::new(Arc::clone(&children[0])).with_required_input_ordering(self.required_input_ordering.clone()).with_required_input_distribution(self.required_input_distribution.clone())- .with_maintains_input_order(self.maintains_input_order)- .into_arc())+ .with_maintains_input_order(self.maintains_input_order);+ plan.inputs.extend(children.into_iter().skip(1));+ if self.bounded_output {+ plan = plan.with_bounded_output();+ }+ Ok(plan.into_arc())}fn with_new_children(@@ -1061,6 +1088,10 @@ impl ExecutionPlan for TestScan {Ok(Arc::new(Statistics::new_unknown(&self.schema)))}+ fn supports_limit_pushdown(&self) -> bool {+ self.supports_fetch+ }+fn with_fetch(&self, fetch: Option<usize>) -> Option<Arc<dyn ExecutionPlan>> {if self.supports_fetch {let mut new_scan = self.clone();diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rsindex 4f8670c15b..65a71510c9 100644--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs@@ -36,7 +36,10 @@ use datafusion_physical_plan::repartition::RepartitionExec;use datafusion_physical_plan::sorts::sort::SortExec;use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;use datafusion_physical_plan::tree_node::PlanContext;-use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties};+use datafusion_physical_plan::{+ ChildSatisfactionOptions, ChildrenPropertiesMode, ExecutionPlan,+ ExecutionPlanProperties, ReplaceChildrenOptions,+};/// "Data class" used by sort pushdown (now driven from `EnsureRequirements`)/// to push down [`SortExec`] in the plan. In some cases the total@@ -66,15 +69,14 @@ impl Default for ParentRequirements {pub type SortPushDown = PlanContext<ParentRequirements>;-/// Number of input rows `plan` needs from its children in order to produce-/// the rows its parent will consume.+/// Input row bound used by the specialized sort-pushdown handlers.////// `parent_fetch` is the fetch a parent imposes on `plan`'s *output*/// (`ParentRequirements::fetch`). It bounds `plan.fetch()` directly because/// both count output rows, so the effective output fetch is/// `min(plan.fetch(), parent_fetch)`.///-/// The input fetch is that output fetch for every operator except+/// For these operators, the input fetch is that output fetch except for/// [`GlobalLimitExec`], which discards `skip` rows first and therefore needs/// `skip` more input rows. `skip` must be added *after* taking the minimum:/// `min(fetch + skip, parent_fetch)` would be too small whenever the parent@@ -371,7 +373,7 @@ fn pushdown_sorts_helper(.unwrap_or(&Distribution::UnspecifiedDistribution),);}- } else if let Some(adjusted) = pushdown_requirement_to_children(+ } else if let Some(proposal) = pushdown_requirement_to_children(&sort_push_down.plan,parent_requirement.clone(),parent_fetch,@@ -379,7 +381,6 @@ fn pushdown_sorts_helper(// For operators that can take a sort pushdown, continue with updated// requirements. If this node already outputs single partition (e.g. SPM),// don't push SinglePartition to children.- let current_fetch = input_fetch(&sort_push_down.plan, parent_fetch);let dists = sort_push_down.plan.input_distribution_requirements()@@ -390,11 +391,14 @@ fn pushdown_sorts_helper(} else {parent_distribution.clone()};- for (idx, (child, order)) in- sort_push_down.children.iter_mut().zip(adjusted).enumerate()+ for (idx, (child, order)) in sort_push_down+ .children+ .iter_mut()+ .zip(proposal.child_orderings)+ .enumerate(){child.data.ordering_requirement = order;- child.data.fetch = current_fetch;+ child.data.fetch = proposal.child_fetch;child.data.distribution_requirement = stronger_distribution(&effective_dist,dists@@ -432,13 +436,19 @@ fn pushdown_sorts_helper(Ok(Transformed::yes(sort_push_down))}+/// Child orderings and the input row bound that can accompany sort pushdown.+struct SortPushdownProposal {+ child_orderings: Vec<Option<OrderingRequirements>>,+ child_fetch: Option<usize>,+}+/// Calculate the pushdown ordering requirements for children./// If sort cannot be pushed down, return None.fn pushdown_requirement_to_children(plan: &Arc<dyn ExecutionPlan>,parent_required: OrderingRequirements,parent_fetch: Option<usize>,-) -> Result<Option<Vec<Option<OrderingRequirements>>>> {+) -> Result<Option<SortPushdownProposal>> {// A parent fetch can be pushed through a plan only if the plan explicitly// supports limit pushdown and does not change cardinality. For example, if// LIMIT 2 is applied below a FilterExec that removes half the rows, only one@@ -467,7 +477,7 @@ fn pushdown_requirement_to_children(}let maintains_input_order = plan.maintains_input_order();- if is_window(plan) {+ let child_orderings = if is_window(plan) {let mut required_input_ordering = plan.required_input_ordering();let maybe_child_requirement = required_input_ordering.swap_remove(0);let child_plan = plan.children().swap_remove(0);@@ -535,49 +545,21 @@ fn pushdown_requirement_to_children(} else {Ok(None)}- } else if plan.fetch().is_some()- && plan.supports_limit_pushdown()- && plan- .maintains_input_order()- .into_iter()- .all(|maintain| maintain)- {- // Push down through operator with fetch when:+ } else if plan.is::<SortPreservingMergeExec>() && plan.fetch().is_some() {+ // Push down through a fetched merge when:// - requirement is aligned with output ordering// - it preserves ordering during execution- //- // A `ProjectionExec` reports a `fetch()` forwarded from its input and- // can renumber/reorder columns, so the requirement (expressed in the- // projection's output schema) must be remapped into the child schema- // before being pushed down — forwarding it unchanged would let a key- // such as `score@1` (valid in the output schema) refer to a different- // column in the child schema, producing a `SortExec` whose key points- // at the wrong column ("does not satisfy order requirements ...- // Child-0 order: []"). If a required column maps to a computed- // (non-`Column`) projection expression it cannot be expressed below the- // projection, so the sort is kept above it.- let child_required =- if let Some(projection) = plan.downcast_ref::<ProjectionExec>() {- match remap_requirement_through_projection(projection, &parent_required) {- Some(remapped) => remapped,- None => return Ok(None),- }- } else {- parent_required.clone()- };-- // Pushing a sort below an unordered fetch changes which rows it keeps.- let Some(ordering) = plan.properties().output_ordering() else {+ // Refining a fetched merge must preserve the ordering that selected its rows.+ let Some(ordering) = plan.output_ordering() else {return Ok(None);};- if plan.properties().eq_properties.requirements_compatible(- parent_required.first().clone(),- ordering.clone().into(),- ) {- Ok(Some(vec![Some(child_required)]))- } else {- Ok(None)- }+ Ok(plan+ .equivalence_properties()+ .requirements_compatible(+ parent_required.first().clone(),+ ordering.clone().into(),+ )+ .then(|| vec![Some(parent_required)]))} else if is_union(plan) {// `UnionExec` does not have real sort requirements for its input, we// just propagate the sort requirements down:@@ -622,14 +604,15 @@ fn pushdown_requirement_to_children(} else if let Some(projection_exec) = plan.downcast_ref::<ProjectionExec>() {handle_projection_pushdown(projection_exec, &parent_required)} else if maintains_input_order.is_empty()- || !maintains_input_order.iter().any(|o| *o)+ || !maintains_input_order.iter().any(|maintains| *maintains)|| plan.is::<RepartitionExec>()|| plan.is::<FilterExec>()|| pushdown_would_violate_requirements(&parent_required, plan.as_ref()){- // If the current plan is a leaf node or can not maintain any of the input ordering, can not pushed down requirements.- // For RepartitionExec, we always choose to not push down the sort requirements even the RepartitionExec(input_partition=1) could maintain input ordering.- // Pushing down is not beneficial+ // Leaf nodes and nodes that preserve no input ordering cannot push+ // requirements down to their children.+ // Keep sorts above RepartitionExec even when a single input partition+ // lets it preserve order; pushing the sort below it provides no benefit.Ok(None)} else if is_sort_preserving_merge(plan) {let new_ordering = LexOrdering::from(parent_required.first().clone());@@ -649,8 +632,25 @@ fn pushdown_requirement_to_children(} else if let Some(hash_join) = plan.downcast_ref::<HashJoinExec>() {handle_hash_join(hash_join, parent_required)} else {- handle_custom_pushdown(plan, parent_required, &maintains_input_order)- }+ // An offset or filter may need more input rows than its output fetch.+ // Forward only the safe parent fetch; the child TopK retains its own bound.+ return Ok(handle_custom_pushdown(+ plan,+ parent_required,+ &maintains_input_order,+ )?+ .map(|child_orderings| SortPushdownProposal {+ child_orderings,+ child_fetch: parent_fetch,+ }));+ };++ Ok(+ child_orderings?.map(|child_orderings| SortPushdownProposal {+ child_orderings,+ child_fetch: input_fetch(plan, parent_fetch),+ }),+ )}/// Remap an ordering requirement expressed in a [`ProjectionExec`]'s output@@ -966,20 +966,18 @@ fn expr_source_side(}}-/// Handles the custom pushdown of parent-required sorting requirements down to-/// the child execution plans, considering whether the input order is maintained.+/// [`handle_projection_pushdown`] remaps columns and rejects computed projections.+/// This generic fallback must prove its proposed input orderings are safe.////// # Arguments////// * `plan` - A reference to an `ExecutionPlan` for which the pushdown will be applied./// * `parent_required` - The sorting requirements expected by the parent node.-/// * `maintains_input_order` - A vector of booleans indicating whether each child-/// maintains the input order.+/// * `maintains_input_order` - Whether the node preserves each input's row order.////// # Returns///-/// Returns `Ok(Some(Vec<Option<LexRequirement>>))` if the sorting requirements can be-/// pushed down, `Ok(None)` if not. On error, returns a `Result::Err`.+/// Returns validated child orderings, or `None` to decline pushdown. Errors propagate.fn handle_custom_pushdown(plan: &Arc<dyn ExecutionPlan>,parent_required: OrderingRequirements,@@ -992,18 +990,15 @@ fn handle_custom_pushdown(return Ok(None);}- // Collect all unique column indices used in the parent-required sorting- // expression:- let requirement = parent_required.into_single();- let all_indices: HashSet<usize> = requirement+ // Use the preferred alternative. Computed keys may fail on discarded rows.+ let output_requirement = parent_required.into_single();+ let Some(columns) = output_requirement.iter()- .flat_map(|order| {- collect_columns(&order.expr)- .iter()- .map(|col| col.index())- .collect::<HashSet<_>>()- })- .collect();+ .map(|sort| sort.expr.downcast_ref::<Column>())+ .collect::<Option<Vec<_>>>()+ else {+ return Ok(None);+ };// Get the number of fields in each child's schema:let children_schema_lengths: Vec<usize> = plan_children@@ -1011,7 +1006,7 @@ fn handle_custom_pushdown(.map(|c| c.schema().fields().len()).collect();- // Find the index of the order-maintaining child:+ // Use the first order-maintaining input for this candidate mapping.let Some(maintained_child_idx) = maintains_input_order.iter().enumerate()@@ -1021,61 +1016,178 @@ fn handle_custom_pushdown(return Ok(None);};- // Check if all required columns come from the order-maintaining child:+ // Matching column positions is only a candidate; prove output ordering below.let start_idx = children_schema_lengths[..maintained_child_idx].iter().sum::<usize>();let end_idx = start_idx + children_schema_lengths[maintained_child_idx];- let all_from_maintained_child =- all_indices.iter().all(|i| i >= &start_idx && i < &end_idx);+ let fits_maintained_input = columns+ .iter()+ .all(|col| (start_idx..end_idx).contains(&col.index()));- // If all columns are from the maintained child, update the parent requirements:- if all_from_maintained_child {- let sub_offset = children_schema_lengths- .iter()- .take(maintained_child_idx)- .sum::<usize>();- // Transform the parent-required expression for the child schema by- // adjusting columns:- let updated_parent_req = requirement- .into_iter()- .map(|req| {- let child_schema = plan_children[maintained_child_idx].schema();- let updated_columns = req- .expr- .transform_up(|expr| {- if let Some(col) = expr.downcast_ref::<Column>() {- let new_index = col.index() - sub_offset;- Ok(Transformed::yes(Arc::new(Column::new(- child_schema.field(new_index).name(),- new_index,- ))))- } else {- Ok(Transformed::no(expr))- }- })?- .data;- Ok(PhysicalSortRequirement::new(updated_columns, req.options))- })- .collect::<Result<Vec<_>>>()?;+ if !fits_maintained_input {+ return Ok(None);+ }- // Prepare the result, populating with the updated requirements for children that maintain order- let result = maintains_input_order- .iter()- .map(|&maintains_order| {- if maintains_order {- LexRequirement::new(updated_parent_req.clone())- .map(OrderingRequirements::new)- } else {- None- }- })- .collect();+ // Remap the columns into the maintained child's schema.+ let child_schema = plan_children[maintained_child_idx].schema();+ let mapped_requirement = columns+ .iter()+ .zip(output_requirement.iter())+ .map(|(col, req)| {+ let index = col.index() - start_idx;+ let col = Column::new(child_schema.field(index).name(), index);+ PhysicalSortRequirement::new(Arc::new(col), req.options)+ })+ .collect::<Vec<_>>();- Ok(Some(result))+ // Propose the same ordering for every order-maintaining input.+ let child_requirements = maintains_input_order+ .iter()+ .map(|&maintains_order| {+ if maintains_order {+ LexRequirement::new(mapped_requirement.clone())+ .map(OrderingRequirements::new)+ } else {+ None+ }+ })+ .collect::<Vec<_>>();++ // Row positions can stay unchanged while value order reverses (`-a AS a`).+ // Prove that the candidate satisfies both input and output requirements.+ if !is_custom_sort_pushdown_valid(plan, &child_requirements, output_requirement)? {+ return Ok(None);+ }+ Ok(Some(child_requirements))+}++/// Rebuild with temporary [`SortExec`] inputs to recompute the node's properties.+/// Validate input requirements and output ordering without executing these plans.+fn is_custom_sort_pushdown_valid(+ plan: &Arc<dyn ExecutionPlan>,+ child_requirements: &[Option<OrderingRequirements>],+ output_requirement: LexRequirement,+) -> Result<bool> {+ let plan_children = plan.children();++ let selecting_sort = if plan.fetch().is_some() {+ // Sorting below a fetch can change which rows it selects. Only refine+ // a SortExec directly below this boundary, as for native limits.+ let ([child], [Some(_)]) = (plan_children.as_slice(), child_requirements) else {+ return Ok(false);+ };+ let Some(sort) = child.downcast_ref::<SortExec>() else {+ return Ok(false);+ };+ Some(sort)} else {- Ok(None)+ None+ };++ let mut probe_children = Vec::with_capacity(plan_children.len());+ for (child, required) in plan_children.iter().zip(child_requirements) {+ let Some(required) = required else {+ probe_children.push(Arc::clone(child));+ continue;+ };+ // Other order-maintaining inputs may have narrower schemas.+ let input_width = child.schema().fields().len();+ if required.first().iter().any(|sort| {+ collect_columns(&sort.expr)+ .iter()+ .any(|column| column.index() >= input_width)+ }) {+ return Ok(false);+ }+ let ordering: LexOrdering = required.first().clone().into();+ let child_properties = child.equivalence_properties();+ if let Some(sort) = selecting_sort {+ // Preserve every key that selected the fetched rows, including hidden keys.+ // Clear old ordering metadata so the proposal must prove this on its own.+ let mut proposed_properties = child_properties.clone();+ proposed_properties.clear_orderings();+ proposed_properties.add_ordering(ordering.clone());+ if !proposed_properties.ordering_satisfy(sort.expr().clone())? {+ return Ok(false);+ }+ }+ // A new blocking sort would prevent a bounded node from stopping an+ // unbounded input. Already ordered inputs need no new sort.+ if child.boundedness().is_unbounded()+ && !child_properties.ordering_satisfy_requirement(required.first().clone())?+ {+ return Ok(false);+ }+ probe_children.push(Arc::new(+ SortExec::new(ordering, Arc::clone(child)).with_preserve_partitioning(true),+ ));+ }++ // The input plan already satisfies distribution, which these sorts preserve.+ // Check ordering before rebuilding: the node may reject incompatible children.+ let children = probe_children.iter().map(Arc::as_ref).collect::<Vec<_>>();+ if !sort_pushdown_input_orderings_satisfied(plan.as_ref(), &children)? {+ return Ok(false);+ }++ // Rebuild with fresh properties to see how the proposed orderings pass through.+ let probe = Arc::clone(plan).replace_children(+ probe_children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )?;++ // Rebuilding may change input requirements. Prove those and the output ordering.+ Ok(sort_pushdown_inputs_satisfied(probe.as_ref())?+ && probe+ .equivalence_properties()+ .ordering_satisfy_requirement(output_requirement)?)+}++/// Validate ordering separately so it can be checked before rebuilding a node.+fn sort_pushdown_input_orderings_satisfied(+ plan: &dyn ExecutionPlan,+ children: &[&dyn ExecutionPlan],+) -> Result<bool> {+ let ordering = plan.required_input_ordering();+ if ordering.len() != children.len() {+ return Ok(false);+ }+ for (child, required) in children.iter().zip(ordering) {+ if let Some(required) = required+ && !child+ .equivalence_properties()+ .ordering_satisfy_requirement(required.first().clone())?+ {+ return Ok(false);+ }+ }+ Ok(true)+}++/// Checks input ordering, per-child distribution, and co-partitioning.+fn sort_pushdown_inputs_satisfied(plan: &dyn ExecutionPlan) -> Result<bool> {+ let children = plan+ .children()+ .into_iter()+ .map(Arc::as_ref)+ .collect::<Vec<_>>();+ let distribution = plan.input_distribution_requirements();+ if distribution.per_child_distributions().len() != children.len()+ || !sort_pushdown_input_orderings_satisfied(plan, &children)?+ {+ return Ok(false);}+ let options = ChildSatisfactionOptions::new().with_allow_subset(true);+ for (index, child) in children.iter().enumerate() {+ let satisfaction = distribution.child_satisfaction(index, *child, options)?;+ if !satisfaction.is_satisfied() {+ return Ok(false);+ }+ }+ Ok(distribution+ .unsatisfied_co_partitioned_children(plan.name(), &children)?+ .is_empty())}// For hash join we only maintain the input order for the right child@@ -1218,9 +1330,15 @@ fn handle_projection_pushdown(.iter().any(|expr| !expr.expr.is::<Column>()){+ // Computed projection expressions are not supported by this+ // remapping, so keep the sort above the projection.return Ok(None);}+ // A `ProjectionExec` can renumber/reorder columns, so the requirement+ // (expressed in the projection's output schema) must be remapped into the+ // child schema before being pushed down. Forwarding `score@1` unchanged+ // could refer to a different input column and sort on the wrong key.Ok(remap_requirement_through_projection(projection_exec, parent_required).map(|requirements| vec![Some(requirements)]),@@ -1234,10 +1352,16 @@ mod tests {use arrow::compute::SortOptions;use arrow::datatypes::{DataType, Field, Schema};use datafusion_expr::Operator;+ use datafusion_expr::physical_planning_context::{+ ScalarSubqueryResults, SubqueryIndex,+ };use datafusion_physical_expr::PhysicalExpr;use datafusion_physical_expr::expressions::{BinaryExpr, col};use datafusion_physical_plan::empty::EmptyExec;use datafusion_physical_plan::limit::GlobalLimitExec;+ use datafusion_physical_plan::scalar_subquery::{+ ScalarSubqueryExec, ScalarSubqueryLink,+ };const DESC: SortOptions = SortOptions {descending: true,@@ -1353,6 +1477,30 @@ mod tests {LexRequirement::new(reqs).unwrap()}+ #[test]+ fn scalar_subquery_sort_applies_only_to_the_main_input() -> Result<()> {+ let schema = child_schema();+ let input: Arc<dyn ExecutionPlan> = Arc::new(EmptyExec::new(Arc::clone(&schema)));+ let requirement = OrderingRequirements::new(lex([+ req("c", &schema, DESC),+ req("a", &schema, ASC),+ ]));+ let plan: Arc<dyn ExecutionPlan> = Arc::new(ScalarSubqueryExec::new(+ Arc::clone(&input),+ vec![ScalarSubqueryLink {+ plan: Arc::clone(&input),+ index: SubqueryIndex::new(0),+ }],+ ScalarSubqueryResults::new(1),+ ));+ let proposal =+ pushdown_requirement_to_children(&plan, requirement.clone(), None)?+ .expect("the main input supports sort pushdown");+ assert_eq!(proposal.child_orderings, vec![Some(requirement), None]);+ assert_eq!(proposal.child_fetch, None);+ Ok(())+ }+#[test]fn remap_single_hard_requirement_through_reordering_projection() {let projection = reordering_projection();diff --git a/datafusion/sqllogictest/src/test_context.rs b/datafusion/sqllogictest/src/test_context.rsindex 9196a92243..c85642bb0a 100644--- a/datafusion/sqllogictest/src/test_context.rs+++ b/datafusion/sqllogictest/src/test_context.rs@@ -33,6 +33,7 @@ use datafusion::logical_expr::{ColumnarValue, Expr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature,Volatility, create_udf,};+use datafusion::physical_optimizer::optimizer::PhysicalOptimizer;use datafusion::physical_plan::ExecutionPlan;use datafusion::prelude::*;use datafusion::{@@ -60,6 +61,7 @@ use log::info;use sqlparser::ast;use tempfile::TempDir;+mod custom_sort_pushdown;mod range_partitioning;/// Context for running tests@@ -100,6 +102,7 @@ impl TestContext {/// If `None` is returned (e.g. because some needed feature is not/// enabled), the file should be skippedpub async fn try_new_for_test_file(relative_path: &Path) -> Option<Self> {+ let file_name = relative_path.file_name().unwrap().to_str().unwrap();let config = SessionConfig::new()// hardcode target partitions so plans are deterministic.with_target_partitions(4);@@ -163,12 +166,30 @@ impl TestContext {state_builder = state_builder.with_statistics_registry(registry);}+ if file_name == "sort_pushdown.slt" {+ let mut rules = PhysicalOptimizer::new().rules;+ let ensure_requirements = rules+ .iter()+ .position(|rule| rule.name() == "EnsureRequirements")+ .expect("default optimizer includes EnsureRequirements");+ rules.insert(+ ensure_requirements,+ Arc::new(custom_sort_pushdown::CustomLimitRule),+ );+ state_builder = state_builder.with_physical_optimizer_rules(rules);+ }+let state = state_builder.build();let mut test_ctx = TestContext::new(SessionContext::new_with_state(state));- let file_name = relative_path.file_name().unwrap().to_str().unwrap();match file_name {+ "sort_pushdown.slt" => {+ custom_sort_pushdown::register_custom_sort_pushdown(+ test_ctx.session_ctx(),+ )+ .expect("register custom projection table");+ }"parquet_missing_bounds.slt" => {register_parquet_missing_bounds(&mut test_ctx).await;}diff --git a/datafusion/sqllogictest/src/test_context/custom_sort_pushdown.rs b/datafusion/sqllogictest/src/test_context/custom_sort_pushdown.rsnew file mode 100644index 0000000000..cc50db4974--- /dev/null+++ b/datafusion/sqllogictest/src/test_context/custom_sort_pushdown.rs@@ -0,0 +1,266 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements++use std::fmt::{self, Formatter};+use std::sync::Arc;++use arrow::array::record_batch;+use arrow::datatypes::{self as arrow_schema, SchemaRef};+use async_trait::async_trait;+use datafusion::catalog::{Session, TableProvider};+use datafusion::common::Result;+use datafusion::common::config::ConfigOptions;+use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};+use datafusion::execution::{SendableRecordBatchStream, TaskContext};+use datafusion::logical_expr::{Expr, Operator, TableType};+use datafusion::physical_expr::expressions::{binary, col, lit, negative};+use datafusion::physical_expr::projection::ProjectionExprs;+use datafusion::physical_expr::{PhysicalExpr, PhysicalSortExpr};+use datafusion::physical_optimizer::PhysicalOptimizerRule;+use datafusion::physical_plan::filter::{FilterExec, FilterExecBuilder};+use datafusion::physical_plan::limit::{GlobalLimitExec, LocalLimitExec};+use datafusion::physical_plan::projection::ProjectionExec;+use datafusion::physical_plan::sorts::sort::SortExec;+use datafusion::physical_plan::test::TestMemoryExec;+use datafusion::physical_plan::{+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan,+ ExecutionPlanProperties, InputDistributionRequirements, PlanProperties,+ ReplaceChildrenOptions,+};+use datafusion::prelude::SessionContext;++/// Register custom plans whose native implementations are hidden from the optimizer.+pub(super) fn register_custom_sort_pushdown(ctx: &SessionContext) -> Result<()> {+ let batch = record_batch!(("a", Int32, [2, 1, 3]))?;+ let schema = batch.schema();+ let input = TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?;+ let projection = ProjectionExec::try_new(+ [(negative(col("a", &schema)?, &schema)?, "a".to_string())],+ input,+ )?;+ ctx.register_table(+ "custom_negated_projection",+ Arc::new(CustomExec(Arc::new(projection))),+ )?;++ let batch = record_batch!(("a", Int32, [2, 1, 3]), ("b", Int32, [10, 30, 20]))?;+ let schema = batch.schema();+ let input = TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?;+ let projection = ProjectionExec::try_new(+ [+ (col("b", &schema)?, "a".to_string()),+ (col("a", &schema)?, "b".to_string()),+ ],+ input,+ )?;+ ctx.register_table(+ "custom_reordered_projection",+ Arc::new(CustomExec(Arc::new(projection))),+ )?;++ let batch = record_batch!(("t", Int32, [1, 2, 3]), ("x", Int32, [0, 2, 4]))?;+ let schema = batch.schema();+ let input = TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?;+ let predicate = binary(col("x", &schema)?, Operator::NotEq, lit(0i32), &schema)?;+ ctx.register_table(+ "custom_filtered_input",+ Arc::new(CustomExec(Arc::new(FilterExec::try_new(predicate, input)?))),+ )?;++ let batch = record_batch!(("a", Int32, [1, 2, 3]), ("b", Int32, [30, 20, 10]))?;+ let schema = batch.schema();+ let ordering =+ [PhysicalSortExpr::new_default(col("a", &schema)?).nulls_last()].into();+ let input = Arc::new(+ TestMemoryExec::try_new(&[vec![batch]], Arc::clone(&schema), None)?+ .try_with_sort_information(vec![ordering])?,+ );+ ctx.register_table("custom_ordered_input", Arc::new(CustomExec(input)))?;++ // The custom node hides the final sort key while selecting its first row.+ // Refining its visible ordering must still preserve that hidden key.+ let batch = record_batch!(+ ("a", Int32, [1, 1]),+ ("c", Int32, [10, 20]),+ ("b", Int32, [2, 1])+ )?;+ let schema = batch.schema();+ let input = TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?;+ let ordering = [+ PhysicalSortExpr::new_default(col("a", &schema)?).nulls_last(),+ PhysicalSortExpr::new_default(col("b", &schema)?).nulls_last(),+ ]+ .into();+ // Keep the selection ordering through earlier sort-removal passes.+ let input = Arc::new(SortExec::new(ordering, input).with_fetch(Some(2)));+ let projected_limit = FilterExecBuilder::new(lit(true), input)+ .apply_projection(Some(vec![0, 1]))?+ .with_fetch(Some(1))+ .build()?;+ ctx.register_table(+ "custom_projected_limit",+ Arc::new(CustomExec(Arc::new(projected_limit))),+ )?;++ // The output fetch excludes the skipped row, so it cannot also bound input.+ let batch = record_batch!(("a", Int32, [2, 1, 3]), ("b", Int32, [20, 30, 10]))?;+ let schema = batch.schema();+ let input = TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?;+ let ordering =+ [PhysicalSortExpr::new_default(col("a", &schema)?).nulls_last()].into();+ let input = Arc::new(SortExec::new(ordering, input).with_fetch(Some(3)));+ let limit = GlobalLimitExec::new(input, 1, Some(1));+ ctx.register_table("custom_offset_limit", Arc::new(CustomExec(Arc::new(limit))))?;+ Ok(())+}++/// Replace a redundant sort on the ordered test input with a custom limit, so SQL+/// `ORDER BY ... LIMIT` exercises the generic sort-pushdown path.+#[derive(Debug)]+pub(super) struct CustomLimitRule;++impl PhysicalOptimizerRule for CustomLimitRule {+ fn optimize(+ &self,+ plan: Arc<dyn ExecutionPlan>,+ _: &ConfigOptions,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ plan.transform_up(|plan| {+ let Some(sort) = plan.downcast_ref::<SortExec>() else {+ return Ok(Transformed::no(plan));+ };+ let Some(fetch) = sort.fetch() else {+ return Ok(Transformed::no(plan));+ };+ let input = sort.input();+ let source = input+ .downcast_ref::<ProjectionExec>()+ .map_or(input, |projection| projection.input());+ let Some(source) = source.downcast_ref::<CustomExec>() else {+ return Ok(Transformed::no(plan));+ };+ // Only the dedicated ordered source is a wrapped memory scan.+ if !source.0.is::<TestMemoryExec>()+ || input.output_partitioning().partition_count() != 1+ || !input+ .equivalence_properties()+ .ordering_satisfy(sort.expr().clone())?+ {+ return Ok(Transformed::no(plan));+ }+ let limit = LocalLimitExec::new(Arc::clone(input), fetch);+ let plan: Arc<dyn ExecutionPlan> = Arc::new(CustomExec(Arc::new(limit)));+ Ok(Transformed::yes(plan))+ })+ .map(|transformed| transformed.data)+ }++ fn name(&self) -> &str {+ "CustomLimitRule"+ }++ fn schema_check(&self) -> bool {+ true+ }+}++/// Exposes a native plan's children and properties under a distinct plan type,+/// exercising the generic custom-plan sort pushdown path.+#[derive(Debug, Clone)]+struct CustomExec(Arc<dyn ExecutionPlan>);++#[async_trait]+impl TableProvider for CustomExec {+ fn schema(&self) -> SchemaRef {+ self.0.schema()+ }++ fn table_type(&self) -> TableType {+ TableType::Base+ }++ async fn scan(+ &self,+ _: &dyn Session,+ projection: Option<&[usize]>,+ _: &[Expr],+ _: Option<usize>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ let plan: Arc<dyn ExecutionPlan> = Arc::new(self.clone());+ // Keep identity projections out of the plan so sort pushdown reaches+ // this custom node before native projection-specific handling.+ if let Some(projection) = projection+ && !projection+ .iter()+ .copied()+ .eq(0..self.0.schema().fields().len())+ {+ let expressions = ProjectionExprs::from_indices(projection, &self.0.schema());+ Ok(Arc::new(ProjectionExec::try_new(+ expressions.iter().cloned(),+ plan,+ )?))+ } else {+ Ok(plan)+ }+ }+}++impl DisplayAs for CustomExec {+ fn fmt_as(&self, display_type: DisplayFormatType, f: &mut Formatter) -> fmt::Result {+ write!(f, "Custom")?;+ self.0.fmt_as(display_type, f)+ }+}++impl ExecutionPlan for CustomExec {+ fn name(&self) -> &str {+ "CustomExec"+ }++ fn properties(&self) -> &Arc<PlanProperties> {+ self.0.properties()+ }++ fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {+ self.0.children()+ }++ fn maintains_input_order(&self) -> Vec<bool> {+ self.0.maintains_input_order()+ }++ fn input_distribution_requirements(&self) -> InputDistributionRequirements {+ self.0.input_distribution_requirements()+ }++ fn fetch(&self) -> Option<usize> {+ self.0.fetch()+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ Ok(Arc::new(Self(Arc::clone(&self.0).replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )?)))+ }++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ self.0.apply_expressions(f)+ }++ fn execute(+ &self,+ partition: usize,+ context: Arc<TaskContext>,+ ) -> Result<SendableRecordBatchStream> {+ self.0.execute(partition, context)+ }+}diff --git a/datafusion/sqllogictest/test_files/sort_pushdown.slt b/datafusion/sqllogictest/test_files/sort_pushdown.sltindex 6b6ab2cca1..33b9f45e5b 100644--- a/datafusion/sqllogictest/test_files/sort_pushdown.slt+++ b/datafusion/sqllogictest/test_files/sort_pushdown.slt@@ -2,6 +2,153 @@# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements+# The custom operator evaluates -a AS a, preserving row positions while+# reversing the ordering of values. Its input is [2, 1, 3].+statement ok+SET datafusion.execution.target_partitions = 1;++query I+SELECT a FROM custom_negated_projection ORDER BY a;+----+-3+-2+-1++# The sort must stay above the custom projection.+statement ok+SET datafusion.explain.physical_plan_only = true;++query TT+EXPLAIN SELECT a FROM custom_negated_projection ORDER BY a;+----+physical_plan+01)SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false]+02)--CustomProjectionExec: expr=[(- a@0) as a]+03)----CooperativeExec+04)------DataSourceExec: partitions=1, partition_sizes=[1]++# Reordering columns must not make the generic path sort the wrong input column.+query II+SELECT a, b FROM custom_reordered_projection ORDER BY a;+----+10 2+20 3+30 1++query TT+EXPLAIN SELECT a, b FROM custom_reordered_projection ORDER BY a;+----+physical_plan+01)SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false]+02)--CustomProjectionExec: expr=[b@1 as a, a@0 as b]+03)----CooperativeExec+04)------DataSourceExec: partitions=1, partition_sizes=[1]++# Sorting before the custom filter would evaluate 1 / x on the rejected row+# where x = 0, failing the query with an integer division-by-zero error.+query II+SELECT t, x FROM custom_filtered_input ORDER BY t, 1 / x;+----+2 2+3 4++query TT+EXPLAIN SELECT t, x FROM custom_filtered_input ORDER BY t, 1 / x;+----+physical_plan+01)SortExec: expr=[t@0 ASC NULLS LAST, 1 / CAST(x@1 AS Int64) ASC NULLS LAST], preserve_partitioning=[false]+02)--CustomFilterExec: x@1 != 0+03)----CooperativeExec+04)------DataSourceExec: partitions=1, partition_sizes=[1]++# The inner ORDER BY determines which rows LIMIT selects. The outer sort+# must keep those rows, even when LIMIT is implemented by a custom node.+query II+SELECT a, b+FROM (SELECT a, b FROM custom_ordered_input ORDER BY a LIMIT 2) s+ORDER BY b;+----+2 20+1 30++query TT+EXPLAIN SELECT a, b+FROM (SELECT a, b FROM custom_ordered_input ORDER BY a LIMIT 2) s+ORDER BY b;+----+physical_plan+01)SortExec: expr=[b@1 ASC NULLS LAST], preserve_partitioning=[false]+02)--CustomLocalLimitExec: fetch=2+03)----CooperativeExec+04)------CustomDataSourceExec: partitions=1, partition_sizes=[1], output_ordering=a@0 ASC NULLS LAST++# The custom node sorts by (a, b), keeps one row, and hides b. Ordering its+# output by (a, c) must not replace the hidden b tie-breaker that selected c=20.+query II+SELECT a, c FROM custom_projected_limit ORDER BY a, c;+----+1 20++query TT+EXPLAIN SELECT a, c FROM custom_projected_limit ORDER BY a, c;+----+physical_plan+01)SortExec: expr=[a@0 ASC NULLS LAST, c@1 ASC NULLS LAST], preserve_partitioning=[false]+02)--CustomFilterExec: true, projection=[a@0, c@1], fetch=1+03)----SortExec: TopK(fetch=2), expr=[a@0 ASC NULLS LAST, b@2 ASC NULLS LAST], preserve_partitioning=[false]+04)------CooperativeExec+05)--------DataSourceExec: partitions=1, partition_sizes=[1]++# Refining (a) to (a, b) below the custom OFFSET 1 LIMIT 1 is safe, but its+# output fetch=1 must not tighten TopK(3) to TopK(1): the offset needs a second row.+query II+SELECT a, b FROM custom_offset_limit ORDER BY a, b;+----+2 20++query TT+EXPLAIN SELECT a, b FROM custom_offset_limit ORDER BY a, b;+----+physical_plan+01)CustomGlobalLimitExec: skip=1, fetch=1+02)--SortExec: TopK(fetch=3), expr=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST], preserve_partitioning=[false]+03)----CooperativeExec+04)------DataSourceExec: partitions=1, partition_sizes=[1]++# The aggregate's amount ordering can refine the inner TopK's country+# ordering. Keep one TopK with both keys, preserving its original fetch=3.+statement ok+CREATE TABLE sort_pushdown_sales(country VARCHAR, amount INT) AS VALUES+('B', 20), ('A', 10), ('C', 40), ('A', 30);++query T? rowsort+SELECT s.country, ARRAY_AGG(s.amount ORDER BY s.amount DESC) AS amounts+FROM (SELECT * FROM sort_pushdown_sales ORDER BY country LIMIT 3) AS s+GROUP BY s.country;+----+A [30, 10]+B [20]++query TT+EXPLAIN SELECT s.country, ARRAY_AGG(s.amount ORDER BY s.amount DESC) AS amounts+FROM (SELECT * FROM sort_pushdown_sales ORDER BY country LIMIT 3) AS s+GROUP BY s.country;+----+physical_plan+01)ProjectionExec: expr=[country@0 as country, array_agg(s.amount) ORDER BY [s.amount DESC NULLS FIRST]@1 as amounts]+02)--AggregateExec: mode=Single, gby=[country@0 as country], aggr=[array_agg(s.amount) ORDER BY [s.amount DESC NULLS FIRST]], group_clustering_mode=Full+03)----SortExec: TopK(fetch=3), expr=[country@0 ASC NULLS LAST, amount@1 DESC], preserve_partitioning=[false]+04)------DataSourceExec: partitions=1, partition_sizes=[1]++statement ok+DROP TABLE sort_pushdown_sales;++statement ok+SET datafusion.explain.physical_plan_only = false;++statement ok+SET datafusion.execution.target_partitions = 4;+#Sort Pushdown for ordered Parquet filesstatement okSET datafusion.execution.parquet.pushdown_filters = true;