Apache DataFusion SQL Query Engine
fix: preserve fetch across distribution reoptimization (#24809)
xudong.w40488988ad59
diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rsindex 4aeee42239..6b8e38d150 100644--- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs+++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs@@ -26,7 +26,7 @@ use crate::physical_optimizer::test_utils::{sort_merge_join_exec, sort_preserving_merge_exec, union_exec,};-use arrow::array::{RecordBatch, UInt8Array, UInt64Array};+use arrow::array::{Int64Array, RecordBatch, UInt8Array, UInt64Array};use arrow::compute::SortOptions;use arrow_schema::{DataType, Field, Schema, SchemaRef};use datafusion::config::ConfigOptions;@@ -47,6 +47,7 @@ use datafusion_common::tree_node::{};use datafusion_datasource::file_groups::FileGroup;use datafusion_datasource::file_scan_config::FileScanConfigBuilder;+use datafusion_datasource::memory::MemorySourceConfig;use datafusion_expr::{JoinType, Operator};use datafusion_functions_aggregate::count::count_udaf;use datafusion_physical_expr::aggregate::AggregateExprBuilder;@@ -76,11 +77,12 @@ use datafusion_physical_plan::joins::utils::JoinOn;use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};use datafusion_physical_plan::projection::{ProjectionExec, ProjectionExpr};use datafusion_physical_plan::repartition::RepartitionExec;+use datafusion_physical_plan::sorts::sort::SortExec;use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;use datafusion_physical_plan::union::{InterleaveExec, UnionExec};use datafusion_physical_plan::{ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlanProperties,- PlanProperties, ReplaceChildrenOptions, displayable,+ PlanProperties, ReplaceChildrenOptions, collect, displayable,};use insta::Settings;@@ -4844,11 +4846,12 @@ fn test_replace_order_preserving_variants_with_fetch() -> Result<()> {// Apply the functionlet result = replace_order_preserving_variants(dist_context)?;- // Verify the plan was transformed to CoalescePartitionsExec+ // A fetched ordered merge must still select the TopK rows.+ let result = check_integrity(result)?;result.plan- .downcast_ref::<CoalescePartitionsExec>()- .expect("Expected CoalescePartitionsExec");+ .downcast_ref::<SortExec>()+ .expect("Expected a TopK SortExec");// Verify fetch was preservedassert_eq!(@@ -4860,6 +4863,318 @@ fn test_replace_order_preserving_variants_with_fetch() -> Result<()> {Ok(())}+#[test]+fn preserve_fetch_when_reoptimizing_ordered_merge() -> Result<()> {+ let schema = schema();+ let sort_key: LexOrdering =+ [PhysicalSortExpr::new_default(col("c", &schema)?)].into();+ let input = parquet_exec_multiple_sorted(vec![sort_key.clone()]);+ let plan: Arc<dyn ExecutionPlan> =+ Arc::new(SortPreservingMergeExec::new(sort_key, input).with_fetch(Some(5)));++ let optimized =+ EnsureRequirements::new().optimize(plan, &test_suite_default_config_options())?;+ let plan = displayable(optimized.as_ref()).indent(true).to_string();++ assert!(+ plan.contains("SortPreservingMergeExec: [c@2 ASC], fetch=5"),+ "expected the optimizer to preserve fetch:\n{plan}"+ );++ Ok(())+}++#[test]+fn preserve_fetch_when_reoptimizing_coalesce_partitions() -> Result<()> {+ let input = parquet_exec_multiple();+ let plan: Arc<dyn ExecutionPlan> =+ Arc::new(CoalescePartitionsExec::new(input).with_fetch(Some(5)));++ let optimized =+ EnsureRequirements::new().optimize(plan, &test_suite_default_config_options())?;++ assert_eq!(optimized.fetch(), Some(5));+ optimized+ .downcast_ref::<CoalescePartitionsExec>()+ .expect("expected CoalescePartitionsExec");++ Ok(())+}++#[tokio::test]+async fn move_fetch_to_replacement_sort() -> Result<()> {+ for (options, partitions, expected) in [+ (+ SortOptions::default(),+ [+ vec![None, Some(1), Some(1), Some(6)],+ vec![None, Some(1), Some(2), Some(7)],+ ],+ vec![None, None, Some(1), Some(1), Some(1)],+ ),+ (+ SortOptions {+ descending: true,+ nulls_first: false,+ },+ [vec![Some(7), Some(1), None], vec![Some(6), Some(1), None]],+ vec![Some(7), Some(6), Some(1), Some(1), None],+ ),+ ] {+ let (input, sort_key) = sorted_memory_input(partitions, options)?;+ let merge: Arc<dyn ExecutionPlan> = Arc::new(+ SortPreservingMergeExec::new(sort_key.clone(), input).with_fetch(Some(5)),+ );+ assert_eq!(fetch_test_values(Arc::clone(&merge)).await?, expected);+ let plan = sort_required_exec_with_req(merge, sort_key);+ let optimized = ensure_distribution_helper(plan, 10, false)?;+ let replacement = Arc::clone(optimized.children()[0]);+ let sort = replacement+ .downcast_ref::<SortExec>()+ .expect("expected a replacement sort");+ assert_eq!(sort.fetch(), Some(5));+ assert_eq!(fetch_test_values(replacement).await?, expected);+ }+ Ok(())+}++#[tokio::test]+async fn preserve_fetch_in_nested_distribution_operators() -> Result<()> {+ for outer_fetch in [0, 3, 10] {+ let (input, sort_key) = sorted_memory_input(+ [0, 1].map(|start| (start..10).step_by(2).map(Some).collect()),+ SortOptions::default(),+ )?;+ let merge: Arc<dyn ExecutionPlan> =+ Arc::new(SortPreservingMergeExec::new(sort_key, input).with_fetch(Some(5)));+ let plan: Arc<dyn ExecutionPlan> =+ Arc::new(CoalescePartitionsExec::new(merge).with_fetch(Some(outer_fetch)));+ let expected = (0..outer_fetch.min(5))+ .map(|value| Some(value as i64))+ .collect::<Vec<_>>();+ assert_reoptimized_fetch_values(plan, &expected).await?;+ }+ Ok(())+}++#[tokio::test]+async fn preserve_topk_when_parent_changes_ordering() -> Result<()> {+ let (input, sort_key) = sorted_memory_input(+ [0, 1].map(|start| (start..10).step_by(2).map(Some).collect()),+ SortOptions::default(),+ )?;+ let descending = [PhysicalSortExpr::new(+ col("c", &input.schema())?,+ SortOptions {+ descending: true,+ nulls_first: false,+ },+ )]+ .into();+ let merge: Arc<dyn ExecutionPlan> =+ Arc::new(SortPreservingMergeExec::new(sort_key, input).with_fetch(Some(5)));+ let plan: Arc<dyn ExecutionPlan> = Arc::new(SortExec::new(descending, merge));+ assert_reoptimized_fetch_values(plan, &[Some(4), Some(3), Some(2), Some(1), Some(0)])+ .await+}++#[tokio::test]+async fn preserve_fetch_when_parallelizing_sort_above_filter() -> Result<()> {+ let (input, sort_key) = sorted_memory_input(+ [+ vec![Some(-4), Some(-2), Some(2), Some(4), Some(6)],+ vec![Some(-3), Some(-1), Some(3), Some(5), Some(7)],+ ],+ SortOptions::default(),+ )?;+ let predicate = Arc::new(BinaryExpr::new(+ col("c", &input.schema())?,+ Operator::Gt,+ lit(0_i64),+ ));+ let coalesce: Arc<dyn ExecutionPlan> =+ Arc::new(CoalescePartitionsExec::new(input).with_fetch(Some(5)));+ let filter: Arc<dyn ExecutionPlan> =+ Arc::new(FilterExec::try_new(predicate, coalesce)?);+ let mut plan: Arc<dyn ExecutionPlan> = Arc::new(SortExec::new(sort_key, filter));+ let mut config = test_suite_default_config_options();+ config.optimizer.enable_round_robin_repartition = false;+ config.optimizer.repartition_sorts = true;+ for iteration in 0..3 {+ if iteration > 0 {+ plan = EnsureRequirements::new().optimize(plan, &config)?;+ }+ // Either input batch can arrive first. Both contain three positive+ // rows, so keeping the limit below the filter always returns three.+ assert_eq!(+ fetch_test_values(Arc::clone(&plan)).await?.len(),+ 3,+ "iteration {iteration}:\n{}",+ displayable(plan.as_ref()).indent(true)+ );+ }+ Ok(())+}++fn sorted_memory_input(+ partitions: [Vec<Option<i64>>; 2],+ options: SortOptions,+) -> Result<(Arc<dyn ExecutionPlan>, LexOrdering)> {+ let schema = Arc::new(Schema::new(vec![Field::new("c", DataType::Int64, true)]));+ let order: LexOrdering = [PhysicalSortExpr::new(col("c", &schema)?, options)].into();+ let partitions = partitions+ .into_iter()+ .map(|values| {+ RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(Int64Array::from(values))],+ )+ .map(|batch| vec![batch])+ })+ .collect::<std::result::Result<Vec<_>, _>>()?;+ let source = MemorySourceConfig::try_new(&partitions, schema, None)?+ .try_with_sort_information(vec![order.clone()])?;+ Ok((DataSourceExec::from_data_source(source), order))+}++async fn fetch_test_values(plan: Arc<dyn ExecutionPlan>) -> Result<Vec<Option<i64>>> {+ let batches = collect(plan, SessionContext::new().task_ctx()).await?;+ Ok(batches+ .iter()+ .flat_map(|batch| {+ batch+ .column(0)+ .as_any()+ .downcast_ref::<Int64Array>()+ .unwrap()+ .iter()+ })+ .collect())+}++async fn assert_reoptimized_fetch_values(+ plan: Arc<dyn ExecutionPlan>,+ expected: &[Option<i64>],+) -> Result<()> {+ for repartition_sorts in [false, true] {+ let mut optimized = Arc::clone(&plan);+ let mut config = test_suite_default_config_options();+ config.optimizer.enable_round_robin_repartition = false;+ config.optimizer.repartition_sorts = repartition_sorts;+ for iteration in 0..3 {+ if iteration > 0 {+ let distribution =+ DistributionContext::new_default(Arc::clone(&optimized))+ .transform_up(|context| ensure_distribution(context, &config))?+ .data;+ check_integrity(distribution)?;+ optimized = EnsureRequirements::new().optimize(optimized, &config)?;+ }+ assert_eq!(+ fetch_test_values(Arc::clone(&optimized)).await?,+ expected,+ "iteration {iteration}, repartition_sorts={repartition_sorts}:\n{}",+ displayable(optimized.as_ref()).indent(true)+ );+ }+ }+ Ok(())+}++#[tokio::test]+async fn preserve_fetch_below_filter_when_reoptimizing() -> Result<()> {+ check_fetch_below_filter(+ Operator::Gt,+ [vec![-2, 0, 2, 4], vec![-1, 1, 3, 5]],+ &[1, 2],+ )+ .await+}++#[tokio::test]+async fn preserve_fetch_below_filter_with_constant_ordering() -> Result<()> {+ check_fetch_below_filter(+ Operator::Eq,+ [vec![-2, 0, 0, 0], vec![-1, 0, 0, 0]],+ &[0, 0, 0],+ )+ .await+}++async fn check_fetch_below_filter(+ op: Operator,+ partitions: [Vec<i64>; 2],+ expected: &[i64],+) -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("c", DataType::Int64, false)]));+ let sort_key: LexOrdering =+ [PhysicalSortExpr::new_default(col("c", &schema)?)].into();+ let partitions = partitions+ .into_iter()+ .map(|values| {+ RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(Int64Array::from(values))],+ )+ .map(|batch| vec![batch])+ })+ .collect::<std::result::Result<Vec<_>, _>>()?;+ let source = MemorySourceConfig::try_new(&partitions, Arc::clone(&schema), None)?+ .try_with_sort_information(vec![sort_key.clone()])?;+ let merge: Arc<dyn ExecutionPlan> = Arc::new(+ SortPreservingMergeExec::new(+ sort_key.clone(),+ DataSourceExec::from_data_source(source),+ )+ .with_fetch(Some(5)),+ );+ let predicate = Arc::new(BinaryExpr::new(col("c", &schema)?, op, lit(0_i64)));+ let filter: Arc<dyn ExecutionPlan> = Arc::new(FilterExec::try_new(predicate, merge)?);+ let mut plan = sort_required_exec_with_req(filter, sort_key);+ let mut config = test_suite_default_config_options();+ config.optimizer.enable_round_robin_repartition = false;+ let task_context = SessionContext::new().task_ctx();++ // The test operator only declares ordering requirements. Execute its child+ // to compare query results before optimization and after repeated passes.+ for iteration in 0..3 {+ if iteration > 0 {+ let distribution = DistributionContext::new_default(Arc::clone(&plan))+ .transform_up(|context| ensure_distribution(context, &config))?+ .data;+ check_integrity(distribution)?;+ plan = EnsureRequirements::new().optimize(plan, &config)?;+ }+ let input = Arc::clone(plan.children()[0]);+ let batches = collect(input, Arc::clone(&task_context)).await?;+ let values = batches+ .iter()+ .flat_map(|batch| {+ batch+ .column(0)+ .as_any()+ .downcast_ref::<Int64Array>()+ .unwrap()+ .values()+ .iter()+ .copied()+ })+ .collect::<Vec<_>>();+ assert_eq!(+ values,+ expected,+ "iteration {iteration}:\n{}",+ displayable(plan.as_ref()).indent(true)+ );+ assert!(+ plan.children()[0].is::<FilterExec>(),+ "fetch must stay below the filter:\n{}",+ displayable(plan.as_ref()).indent(true)+ );+ }+ Ok(())+}+/// When a parent requires SinglePartition and maintains input order, order-preserving/// variants (e.g. SortPreservingMergeExec) should be kept so that ordering can/// propagate to ancestors. Replacing them with CoalescePartitionsExec would destroydiff --git a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs b/datafusion/core/tests/physical_optimizer/enforce_sorting.rsindex f298fa1e47..04019feb66 100644--- a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs+++ b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs@@ -2327,7 +2327,9 @@ async fn test_remove_unnecessary_spm2() -> Result<()> {DataSourceExec: partitions=1, partition_sizes=[0]Optimized Plan:- DataSourceExec: partitions=1, partition_sizes=[0]+ LocalLimitExec: fetch=100+ SortExec: expr=[non_nullable_col@1 ASC], preserve_partitioning=[false]+ DataSourceExec: partitions=1, partition_sizes=[0]");Ok(())diff --git a/datafusion/core/tests/physical_optimizer/replace_with_order_preserving_variants.rs b/datafusion/core/tests/physical_optimizer/replace_with_order_preserving_variants.rsindex 601667ea02..b8adb283c2 100644--- a/datafusion/core/tests/physical_optimizer/replace_with_order_preserving_variants.rs+++ b/datafusion/core/tests/physical_optimizer/replace_with_order_preserving_variants.rs@@ -31,7 +31,7 @@ use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use arrow::record_batch::RecordBatch;use insta::{allow_duplicates, assert_snapshot};use datafusion_common::tree_node::{TransformedResult, TreeNode};-use datafusion_common::{assert_contains, NullEquality, Result};+use datafusion_common::{NullEquality, Result};use datafusion_common::config::ConfigOptions;use datafusion_datasource::source::DataSourceExec;use datafusion_execution::TaskContext;@@ -1199,43 +1199,22 @@ fn test_plan_with_order_preserving_variants_preserves_fetch() -> Result<()> {.with_fetch(Some(10)).unwrap();- // Test sort's fetch is greater than coalesce fetch, return error because it's not reasonable- let requirements = OrderPreservationContext::new(- coalesced.clone(),- false,- vec![OrderPreservationContext::new(- parquet_exec.clone(),- false,- vec![],- )],- );- let res = plan_with_order_preserving_variants(requirements, false, true, Some(15));- assert_contains!(- res.unwrap_err().to_string(),- "CoalescePartitionsExec fetch [10] should be greater than or equal to SortExec fetch [15]"- );-- // Test sort is without fetch, expected to get the fetch value from the coalesced- let requirements = OrderPreservationContext::new(- coalesced.clone(),- false,- vec![OrderPreservationContext::new(- parquet_exec.clone(),+ // Keep the coalesce's row selection independently of an ancestor's fetch.+ for sort_fetch in [Some(15), None, Some(5)] {+ let requirements = OrderPreservationContext::new(+ Arc::clone(&coalesced),false,- vec![],- )],- );- let res = plan_with_order_preserving_variants(requirements, false, true, None)?;- assert_eq!(res.plan.fetch(), Some(10),);-- // Test sort's fetch is less than coalesces fetch, expected to get the fetch value from the sort- let requirements = OrderPreservationContext::new(- coalesced,- false,- vec![OrderPreservationContext::new(parquet_exec, false, vec![])],- );- let res = plan_with_order_preserving_variants(requirements, false, true, Some(5))?;- assert_eq!(res.plan.fetch(), Some(5),);+ vec![OrderPreservationContext::new(+ parquet_exec.clone(),+ false,+ vec![],+ )],+ );+ let res =+ plan_with_order_preserving_variants(requirements, false, true, sort_fetch)?;+ assert!(Arc::ptr_eq(&res.plan, &coalesced));+ assert_eq!(res.plan.fetch(), Some(10));+ }Ok(())}diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rsindex ae9774c9f8..0d850d4113 100644--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs@@ -63,6 +63,7 @@ use datafusion_physical_plan::joins::{};use datafusion_physical_plan::projection::{ProjectionExec, ProjectionExpr};use datafusion_physical_plan::repartition::RepartitionExec;+use datafusion_physical_plan::sorts::sort::SortExec;use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext};use datafusion_physical_plan::tree_node::PlanContext;@@ -813,10 +814,7 @@ fn preserving_order_enables_streaming(////// Updated node with an execution plan, where the desired single distribution/// requirement is satisfied.-fn add_merge_on_top(- input: DistributionContext,- fetch: Option<usize>,-) -> DistributionContext {+fn add_merge_on_top(input: DistributionContext) -> DistributionContext {// Apply only when the partition count is larger than one.if input.plan.output_partitioning().partition_count() > 1 {// When there is an existing ordering, we preserve ordering@@ -825,21 +823,16 @@ fn add_merge_on_top(// - Preserving ordering is not helpful in terms of satisfying ordering requirements// - Usage of order preserving variants is not desirable// (determined by flag `config.optimizer.prefer_existing_sort`)- let new_plan: Arc<dyn ExecutionPlan> = if let Some(req) =- input.plan.output_ordering()- {- let mut spm =- SortPreservingMergeExec::new(req.clone(), Arc::clone(&input.plan));- if let Some(f) = fetch {- spm = spm.with_fetch(Some(f));- }- Arc::new(spm)- } else {- // If there is no input order, we can simply coalesce partitions:- Arc::new(- CoalescePartitionsExec::new(Arc::clone(&input.plan)).with_fetch(fetch),- )- };+ let new_plan: Arc<dyn ExecutionPlan> =+ if let Some(req) = input.plan.output_ordering() {+ Arc::new(SortPreservingMergeExec::new(+ req.clone(),+ Arc::clone(&input.plan),+ ))+ } else {+ // If there is no input order, we can simply coalesce partitions:+ Arc::new(CoalescePartitionsExec::new(Arc::clone(&input.plan)))+ };DistributionContext::new(new_plan, true, vec![input])} else {@@ -864,41 +857,21 @@ fn add_merge_on_top(/// ```text/// "DataSourceExec: file_groups={2 groups: \[\[x], \[y]]}, projection=\[a, b, c, d, e], output_ordering=\[a@0 ASC], file_type=parquet",/// ```-/// Returned by [`remove_dist_changing_operators`] to carry the fetch value-/// that may have been on a removed `SortPreservingMergeExec` or `CoalescePartitionsExec`.-struct RemovedDistOps {- context: DistributionContext,- /// The fetch value from the removed SPM/Coalesce, if any.- /// Must be re-applied when distribution operators are re-inserted.- removed_fetch: Option<usize>,-}-+/// A distribution operator with a fetch also selects rows. Stop at that+/// boundary so neither its limit nor an ordered merge's TopK selection is+/// moved across another operator.fn remove_dist_changing_operators(mut distribution_context: DistributionContext,-) -> Result<RemovedDistOps> {- let mut removed_fetch = None;- while is_repartition(&distribution_context.plan)- || is_coalesce_partitions(&distribution_context.plan)- || is_sort_preserving_merge(&distribution_context.plan)+) -> DistributionContext {+ while distribution_context.plan.fetch().is_none()+ && (is_repartition(&distribution_context.plan)+ || is_coalesce_partitions(&distribution_context.plan)+ || is_sort_preserving_merge(&distribution_context.plan)){- // Preserve fetch from SPM or CoalescePartitions before removing (#14150).- if let Some(fetch) = distribution_context.plan.fetch() {- removed_fetch = Some(- removed_fetch- .map(|existing: usize| existing.min(fetch))- .unwrap_or(fetch),- );- }- // All of above operators have a single child. First child is only child.- // Remove any distribution changing operators at the beginning:+ // All of the above operators have a single child.distribution_context = distribution_context.children.swap_remove(0);- // Note that they will be re-inserted later on if necessary or helpful.}-- Ok(RemovedDistOps {- context: distribution_context,- removed_fetch,- })+ distribution_context}/// Updates the [`DistributionContext`] if preserving ordering while changing partitioning is not helpful or desirable.@@ -934,11 +907,21 @@ pub fn replace_order_preserving_variants(}).collect::<Result<Vec<_>>>()?;- if is_sort_preserving_merge(&context.plan) {+ if let Some(spm) = context.plan.downcast_ref::<SortPreservingMergeExec>() {let child_plan = Arc::clone(&context.children[0].plan);- context.plan = Arc::new(- CoalescePartitionsExec::new(child_plan).with_fetch(context.plan.fetch()),- );+ let fetch = spm.fetch();+ if fetch.is_some() {+ // A fetched merge selects the first rows in its sort order. Moving+ // fetch to an ancestor's sort can cross a filter, or lose the limit+ // entirely if the ancestor needs no additional sort.+ let ordering = spm.expr().clone();+ context.plan = Arc::new(CoalescePartitionsExec::new(child_plan));+ let sort = Arc::new(+ SortExec::new(ordering, Arc::clone(&context.plan)).with_fetch(fetch),+ );+ return Ok(DistributionContext::new(sort, false, vec![context]));+ }+ context.plan = Arc::new(CoalescePartitionsExec::new(child_plan));return Ok(context);} else if let Some(repartition) = context.plan.downcast_ref::<RepartitionExec>()&& repartition.preserve_order()@@ -1383,17 +1366,13 @@ pub fn ensure_distribution(let order_preserving_variants_desirable =unbounded_and_pipeline_friendly || config.optimizer.prefer_existing_sort;- // Remove unnecessary repartition from the physical plan if any.- // Preserve fetch from removed SPM/Coalesce (#14150).- let RemovedDistOps {- context:- DistributionContext {- mut plan,- data,- children,- },- removed_fetch,- } = remove_dist_changing_operators(dist_context)?;+ // Remove distribution-only operators, retaining any fetched operator as+ // a row-selection boundary.+ let DistributionContext {+ mut plan,+ data,+ children,+ } = remove_dist_changing_operators(dist_context);if let Some(exec) = plan.downcast_ref::<WindowAggExec>() {if let Some(updated_window) = get_best_fitting_window(@@ -1548,7 +1527,7 @@ pub fn ensure_distribution(// Satisfy the distribution requirement if it is unmet.match &requirement {Distribution::SinglePartition => {- child = add_merge_on_top(child, removed_fetch);+ child = add_merge_on_top(child);}Distribution::HashPartitioned(exprs)| Distribution::KeyPartitioned(exprs) => {@@ -1666,12 +1645,13 @@ pub fn ensure_distribution(// make sure ordering requirements are still satisfied after.if ordering_satisfied {// Make sure to satisfy ordering requirement:+ let output_fetch = plan+ .downcast_ref::<OutputRequirementExec>()+ .and_then(|output| output.fetch());context = add_sort_above_with_check(context,sort_req,- plan.downcast_ref::<OutputRequirementExec>()- .map(|output| output.fetch())- .unwrap_or(None),+ output_fetch,)?;}}diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/mod.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/mod.rsindex e9467d508a..cd2b5bb2f4 100644--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/mod.rs+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/mod.rs@@ -170,8 +170,10 @@ fn update_coalesce_ctx_children(// Plan has no children, it cannot be a `CoalescePartitionsExec`.false} else if is_coalesce_partitions(&coalesce_context.plan) {- // Initiate a connection:- true+ // A fetched coalesce selects rows before the sort. Removing it would+ // lose that limit, and moving its fetch onto the sort changes which+ // rows are selected.+ coalesce_context.plan.fetch().is_none()} else {children.iter().enumerate().any(|(idx, node)| {// Only consider operators that don't require a single partition,@@ -668,11 +670,9 @@ fn remove_bottleneck_in_subplan_impl(Some(Distribution::SinglePartition))};- let remove_from_first_child = requirements- .children- .first()- .is_some_and(|child| is_coalesce_partitions(&child.plan))- && removable(0);+ let remove_from_first_child = requirements.children.first().is_some_and(|child| {+ is_coalesce_partitions(&child.plan) && child.plan.fetch().is_none()+ }) && removable(0);let children = &mut requirements.children;if remove_from_first_child {// We can safely use the 0th index since we have a `CoalescePartitionsExec`.diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/replace_with_order_preserving_variants.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/replace_with_order_preserving_variants.rsindex d7b3f16a8f..2f9d19569b 100644--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/replace_with_order_preserving_variants.rs+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/replace_with_order_preserving_variants.rs@@ -25,9 +25,9 @@ use crate::utils::{is_coalesce_partitions, is_repartition, is_sort, is_sort_preserving_merge,};+use datafusion_common::Result;use datafusion_common::config::ConfigOptions;use datafusion_common::tree_node::Transformed;-use datafusion_common::{Result, assert_or_internal_err};use datafusion_physical_plan::ExecutionPlanProperties;use datafusion_physical_plan::coalesce_partitions::CoalescePartitionsExec;use datafusion_physical_plan::execution_plan::EmissionType;@@ -64,7 +64,9 @@ pub fn update_order_preservation_ctx_children_data(opc: &mut OrderPreservationCo}let plan_children = plan.children();- *data = if plan_children.is_empty() {+ *data = if plan_children.is_empty()+ || (is_coalesce_partitions(plan) && plan.fetch().is_some())+ {false} else if !children[0].data&& ((is_repartition(plan) && !maintains_input_order[0])@@ -102,6 +104,12 @@ pub fn plan_with_order_preserving_variants(is_spm_better: bool,fetch: Option<usize>,) -> Result<OrderPreservationContext> {+ if is_coalesce_partitions(&sort_input.plan) && sort_input.plan.fetch().is_some() {+ // A fetched coalesce selects rows in arrival order. An ordered merge+ // would select different rows, even with the same fetch value.+ sort_input.data = false;+ return Ok(sort_input);+ }sort_input.children = sort_input.children.into_iter()@@ -137,24 +145,6 @@ pub fn plan_with_order_preserving_variants(} else if is_coalesce_partitions(&sort_input.plan) && is_spm_better {let child = &sort_input.children[0].plan;if let Some(ordering) = child.output_ordering() {- let mut fetch = fetch;- if let Some(coalesce_fetch) = sort_input.plan.fetch() {- fetch = match fetch {- Some(sort_fetch) => {- assert_or_internal_err!(- coalesce_fetch >= sort_fetch,- "CoalescePartitionsExec fetch [{:?}] should be greater than or equal to SortExec fetch [{:?}]",- coalesce_fetch,- sort_fetch- );- Some(sort_fetch)- }- None => {- // If the sort node does not have a fetch, we need to keep the coalesce node's fetch.- Some(coalesce_fetch)- }- };- }// When the input of a `CoalescePartitionsExec` has an ordering,// replace it with a `SortPreservingMergeExec` if appropriate:let spm = SortPreservingMergeExec::new(ordering.clone(), Arc::clone(child))