Apache DataFusion SQL Query Engine
fix: keep file order for scans that advertise an output ordering (#26434)
Jay Zhan63af08da1b74
diff --git a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs b/datafusion/core/tests/physical_optimizer/enforce_sorting.rsindex b5a2a720cb..084ae59a76 100644--- a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs+++ b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs@@ -2360,14 +2360,12 @@ fn test_spm_fetch_preserves_ordering_through_child_rewrite() -> Result<()> {.downcast_ref::<LocalLimitExec>().expect("SPM fetch should become a local limit");assert_eq!(limit.fetch(), 100);- assert_eq!(limit.required_ordering().as_ref(), Some(&ordering));let config = ConfigOptions::new();let optimized = ProjectionPushdown::new().optimize(optimized, &config)?;let limit = optimized.downcast_ref::<LocalLimitExec>().expect("projection rewrite should retain the local limit");- assert_eq!(limit.required_ordering().as_ref(), Some(&ordering));assert!(limit.input().is::<DataSourceExec>());let optimized = LimitPushdown::new().optimize(optimized, &config)?;@@ -2379,7 +2377,7 @@ fn test_spm_fetch_preserves_ordering_through_child_rewrite() -> Result<()> {.downcast_ref::<FileScanConfig>().expect("parquet scan should use FileScanConfig");assert_eq!(config.limit, Some(100));- assert!(config.preserve_order);+ assert!(config.preserves_file_order());Ok(())}diff --git a/datafusion/core/tests/physical_optimizer/ensure_requirements.rs b/datafusion/core/tests/physical_optimizer/ensure_requirements.rsindex fe73518bf1..8e2c026dcd 100644--- a/datafusion/core/tests/physical_optimizer/ensure_requirements.rs+++ b/datafusion/core/tests/physical_optimizer/ensure_requirements.rs@@ -877,10 +877,10 @@ fn test_fetch_fallback_is_stable() {let output_children = pushed_down.children();let merge_children = output_children[0].children();- let limit = merge_children[0]- .downcast_ref::<LocalLimitExec>()- .expect("the fetch should be carried by a local limit");- assert_eq!(limit.required_ordering(), &Some(ordering));+ assert!(+ merge_children[0].is::<LocalLimitExec>(),+ "the fetch should be carried by a local limit"+ );let pushed_down = plan_string(&pushed_down);let next_pass = plan_string(&next_pass);diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rsindex 2a8b5d428f..0c7c1ee280 100644--- a/datafusion/datasource-parquet/src/source.rs+++ b/datafusion/datasource-parquet/src/source.rs@@ -636,7 +636,7 @@ impl FileSource for ParquetSource {.batch_size.expect("Batch size must set before creating ParquetMorselizer"),limit: base_config.limit,- preserve_order: base_config.preserve_order,+ preserve_order: base_config.preserves_file_order(),predicate: self.predicate.clone(),table_schema: self.table_schema.clone(),metadata_size_hint: self.metadata_size_hint,diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rsindex 18f952172a..3aa68826cb 100644--- a/datafusion/datasource/src/file_scan_config/mod.rs+++ b/datafusion/datasource/src/file_scan_config/mod.rs@@ -184,10 +184,12 @@ pub struct FileScanConfig {/// The maximum number of records to read from this plan. If `None`,/// all records after filtering are returned.pub limit: Option<usize>,- /// Whether the scan's limit is order sensitive- /// When `true`, files must be read in the exact order specified to produce- /// correct results (e.g., for `ORDER BY ... LIMIT` queries). When `false`,- /// DataFusion may reorder file processing for optimization without affecting correctness.+ /// Read files in the order of [`Self::file_groups`].+ ///+ /// [`FileScanConfigBuilder::build`] sets this when the scan has an output+ /// ordering. Readers use [`Self::preserves_file_order`], which also checks+ /// the current [`Self::output_ordering`], so clearing this flag never+ /// reorders a scan that advertises an ordering.pub preserve_order: bool,/// All equivalent lexicographical output orderings of this file scan, in terms of/// [`FileSource::table_schema`]. See [`FileScanConfigBuilder::with_output_ordering`] for more@@ -333,14 +335,10 @@ impl FileScanConfigBuilder {self}- /// Set whether the limit should be order-sensitive.- ///- /// When `true`, files must be read in the exact order specified to produce- /// correct results (e.g., for `ORDER BY ... LIMIT` queries). When `false`,- /// DataFusion may reorder file processing for optimization without- /// affecting correctness.- pub fn with_preserve_order(mut self, order_sensitive: bool) -> Self {- self.preserve_order = order_sensitive;+ /// Set whether files must be read in the order of the file groups even+ /// when the scan has no output ordering. See [`FileScanConfig::preserve_order`].+ pub fn with_preserve_order(mut self, preserve_order: bool) -> Self {+ self.preserve_order = preserve_order;self}@@ -1188,18 +1186,6 @@ impl DataSource for FileScanConfig {}}- fn with_preserve_order(&self, preserve_order: bool) -> Option<Arc<dyn DataSource>> {- if self.preserve_order == preserve_order {- return Some(Arc::new(self.clone()));- }-- let new_config = FileScanConfig {- preserve_order,- ..self.clone()- };- Some(Arc::new(new_config))- }-fn apply_expressions(&self,f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,@@ -1220,7 +1206,7 @@ impl DataSource for FileScanConfig {&self,config: &ConfigOptions,) -> Option<Arc<dyn Any + Send + Sync>> {- if self.preserve_order+ if self.preserves_file_order()|| self.output_partitioning.is_some()|| !config.execution.enable_file_stream_work_stealing{@@ -1269,6 +1255,16 @@ fn append_known_empty_files(}impl FileScanConfig {+ /// Returns true if files must be read in the order of the file groups.+ ///+ /// Parents such as `SortPreservingMergeExec` rely on the output ordering+ /// this scan advertises, so a scan with an output ordering never shares+ /// files between partitions (work stealing) and never skips rows based on+ /// its limit (limit pruning).+ pub fn preserves_file_order(&self) -> bool {+ self.preserve_order || !self.output_ordering.is_empty()+ }+/// Returns only the output orderings that are validated against actual/// file group statistics.///@@ -3036,6 +3032,39 @@ mod tests {}}+ /// A scan that advertises an output ordering never shares files between+ /// partitions, even if `preserve_order` is cleared after it is built.+ #[test]+ fn output_ordering_keeps_files_in_order() {+ let schema =+ Arc::new(Schema::new(vec![Field::new("a", DataType::Float64, false)]));+ let sort_expr = PhysicalSortExpr::new_default(Arc::new(Column::new("a", 0)));+ let mut config = FileScanConfigBuilder::new(+ ObjectStoreUrl::local_filesystem(),+ Arc::new(MockSource::new(schema)),+ )+ .with_file_groups(vec![+ FileGroup::new(vec![make_file_with_stats("file1", 0.0, 9.0)]),+ FileGroup::new(vec![make_file_with_stats("file2", 10.0, 19.0)]),+ ])+ .with_output_ordering(vec![LexOrdering::new(vec![sort_expr]).unwrap()])+ .build();+ let options = ConfigOptions::default();++ config.preserve_order = false;+ assert!(config.preserves_file_order());+ assert!(config.create_sibling_state(&options).is_none());++ config.output_ordering = vec![];+ assert!(!config.preserves_file_order());+ assert!(config.create_sibling_state(&options).is_some());++ // An explicit `preserve_order` keeps file order without an output ordering+ config.preserve_order = true;+ assert!(config.preserves_file_order());+ assert!(config.create_sibling_state(&options).is_none());+ }+#[test]fn test_output_partitioning_not_partitioned_by_file_group() {let file_schema = aggr_test_schema();diff --git a/datafusion/datasource/src/source.rs b/datafusion/datasource/src/source.rsindex 741010c595..32a300ff31 100644--- a/datafusion/datasource/src/source.rs+++ b/datafusion/datasource/src/source.rs@@ -223,6 +223,10 @@ pub trait DataSource: Any + Send + Sync + Debug {}/// Returns a variant of this `DataSource` that is aware of order-sensitivity.+ #[deprecated(+ since = "56.0.0",+ note = "no longer called: a scan that advertises an output ordering always preserves it"+ )]fn with_preserve_order(&self, _preserve_order: bool) -> Option<Arc<dyn DataSource>> {None}@@ -585,18 +589,6 @@ impl ExecutionPlan for DataSourceExec {})}- fn with_preserve_order(- &self,- preserve_order: bool,- ) -> Option<Arc<dyn ExecutionPlan>> {- self.data_source- .with_preserve_order(preserve_order)- .map(|new_data_source| {- Arc::new(self.clone().with_data_source(new_data_source))- as Arc<dyn ExecutionPlan>- })- }-fn with_new_state(&self,state: Arc<dyn Any + Send + Sync>,diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/mod.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/mod.rsindex 414a1d0938..8a4905365b 100644--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/mod.rs+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/mod.rs@@ -467,8 +467,7 @@ pub fn ensure_sorting(// single partition.let child_node = requirements.children.swap_remove(0);if let Some(fetch) = requirements.plan.fetch() {- let mut limit = LocalLimitExec::new(Arc::clone(&child_node.plan), fetch);- limit.set_required_ordering(requirements.plan.output_ordering().cloned());+ let limit = LocalLimitExec::new(Arc::clone(&child_node.plan), fetch);return Ok(Transformed::yes(PlanContext::new(Arc::new(limit),false,@@ -516,17 +515,11 @@ fn analyze_immediate_sort_removal(// Remove the sort:node.children = node.children.swap_remove(0).children;if let Some(fetch) = sort_exec.fetch() {- let required_ordering = sort_exec.properties().output_ordering().cloned();// If the sort has a fetch, we need to add a limit:if properties.output_partitioning().partition_count() == 1 {- let mut global_limit =- GlobalLimitExec::new(Arc::clone(sort_input), 0, Some(fetch));- global_limit.set_required_ordering(required_ordering);- Arc::new(global_limit)+ Arc::new(GlobalLimitExec::new(Arc::clone(sort_input), 0, Some(fetch)))} else {- let mut local_limit = LocalLimitExec::new(Arc::clone(sort_input), fetch);- local_limit.set_required_ordering(required_ordering);- Arc::new(local_limit)+ Arc::new(LocalLimitExec::new(Arc::clone(sort_input), fetch))}} else {Arc::clone(sort_input)diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rsindex d7f556b90d..867dcb7525 100644--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs@@ -189,13 +189,9 @@ fn preserve_fetch_above_ordered_plan(let input = Arc::clone(&node.plan);let limit: Arc<dyn ExecutionPlan> = if input_has_multiple_partitions {- let mut limit = LocalLimitExec::new(input, fetch);- limit.set_required_ordering(Some(ordering.clone()));- Arc::new(limit)+ Arc::new(LocalLimitExec::new(input, fetch))} else {- let mut limit = GlobalLimitExec::new(input, 0, Some(fetch));- limit.set_required_ordering(Some(ordering.clone()));- Arc::new(limit)+ Arc::new(GlobalLimitExec::new(input, 0, Some(fetch)))};let limit_node = PlanContext::new(limit, ParentRequirements::default(), vec![node]);diff --git a/datafusion/physical-optimizer/src/limit_pushdown.rs b/datafusion/physical-optimizer/src/limit_pushdown.rsindex 226c41cf8d..55e2a71038 100644--- a/datafusion/physical-optimizer/src/limit_pushdown.rs+++ b/datafusion/physical-optimizer/src/limit_pushdown.rs@@ -100,7 +100,6 @@ pub struct GlobalRequirements {fetch: Option<usize>,skip: usize,satisfied: bool,- preserve_order: bool,}impl LimitPushdown {@@ -128,7 +127,6 @@ impl PhysicalOptimizerRule for LimitPushdown {fetch: None,skip: 0,satisfied: false,- preserve_order: false,};with_statistics_context(context, |stats_ctx| {pushdown_limits(plan, global_state, stats_ctx)@@ -148,7 +146,6 @@ struct LimitInfo {input: Arc<dyn ExecutionPlan>,fetch: Option<usize>,skip: usize,- preserve_order: bool,}/// This function is the main helper function of the `LimitPushDown` rule.@@ -198,7 +195,6 @@ pub fn pushdown_limit_helper_with_stats();global_state.skip = skip;global_state.fetch = fetch;- global_state.preserve_order = limit_info.preserve_order;global_state.satisfied = false;if let Some(fetch) = fetch@@ -315,28 +311,17 @@ pub fn pushdown_limit_helper_with_stats(let maybe_fetchable = pushdown_plan.with_fetch(skip_and_fetch);if global_state.satisfied {if let Some(plan_with_fetch) = maybe_fetchable {- let plan_with_preserve_order = plan_with_fetch- .with_preserve_order(global_state.preserve_order)- .unwrap_or(plan_with_fetch);- Ok((Transformed::yes(plan_with_preserve_order), global_state))+ Ok((Transformed::yes(plan_with_fetch), global_state))} else {Ok((Transformed::no(pushdown_plan), global_state))}} else {global_state.satisfied = true;pushdown_plan = if let Some(plan_with_fetch) = maybe_fetchable {- let plan_with_preserve_order = plan_with_fetch- .with_preserve_order(global_state.preserve_order)- .unwrap_or(plan_with_fetch);-if global_skip > 0 {- add_global_limit(- plan_with_preserve_order,- global_skip,- Some(global_fetch),- )+ add_global_limit(plan_with_fetch, global_skip, Some(global_fetch))} else {- plan_with_preserve_order+ plan_with_fetch}} else {add_limit(pushdown_plan, global_skip, global_fetch)@@ -460,7 +445,6 @@ fn extract_limit(plan: &Arc<dyn ExecutionPlan>) -> Option<LimitInfo> {input: Arc::clone(global_limit.input()),fetch: global_limit.fetch(),skip: global_limit.skip(),- preserve_order: global_limit.required_ordering().is_some(),})} else {plan.downcast_ref::<LocalLimitExec>()@@ -468,7 +452,6 @@ fn extract_limit(plan: &Arc<dyn ExecutionPlan>) -> Option<LimitInfo> {input: Arc::clone(local_limit.input()),fetch: Some(local_limit.fetch()),skip: 0,- preserve_order: local_limit.required_ordering().is_some(),})}}diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rsindex d3de70a3d9..7097cc780d 100644--- a/datafusion/physical-plan/src/coalesce_partitions.rs+++ b/datafusion/physical-plan/src/coalesce_partitions.rs@@ -28,9 +28,7 @@ use super::{DisplayAs, ExecutionPlanProperties, PlanProperties, SendableRecordBatchStream,Statistics,};-use crate::execution_plan::{- CardinalityEffect, EvaluationType, SchedulingType, replace_children_if_necessary,-};+use crate::execution_plan::{CardinalityEffect, EvaluationType, SchedulingType};use crate::filter_pushdown::{FilterDescription, FilterPushdownPhase};use crate::projection::{ProjectionExec, make_with_child};use crate::sort_pushdown::SortOrderPushdownResult;@@ -347,18 +345,6 @@ impl ExecutionPlan for CoalescePartitionsExec {}))}- fn with_preserve_order(- &self,- preserve_order: bool,- ) -> Option<Arc<dyn ExecutionPlan>> {- self.input- .with_preserve_order(preserve_order)- .and_then(|new_input| {- replace_children_if_necessary(Arc::new(self.clone()), vec![new_input])- .ok()- })- }-fn gather_filters_for_pushdown(&self,_phase: FilterPushdownPhase,diff --git a/datafusion/physical-plan/src/execution_plan.rs b/datafusion/physical-plan/src/execution_plan.rsindex 5d6afc2b9b..ee15653f7d 100644--- a/datafusion/physical-plan/src/execution_plan.rs+++ b/datafusion/physical-plan/src/execution_plan.rs@@ -996,10 +996,12 @@ pub trait ExecutionPlan: Any + Debug + DisplayAs + Send + Sync {/// Returns a variant of this `ExecutionPlan` that is aware of order-sensitivity.///- /// This is used to signal to data sources that the output ordering must be- /// preserved, even if it might be more efficient to ignore it (e.g. by- /// skipping some row groups in Parquet).- ///+ /// No longer called by DataFusion: a data source that advertises an output+ /// ordering always preserves it, so no optimizer rule needs to request it.+ #[deprecated(+ since = "56.0.0",+ note = "no longer called: a scan that advertises an output ordering always preserves it"+ )]fn with_preserve_order(&self,_preserve_order: bool,diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rsindex a9546864c9..2500822c5d 100644--- a/datafusion/physical-plan/src/filter.rs+++ b/datafusion/physical-plan/src/filter.rs@@ -30,7 +30,7 @@ use super::{};use crate::coalesce::{LimitedBatchCoalescer, PushBatchStatus};use crate::common::can_project;-use crate::execution_plan::{CardinalityEffect, replace_children_if_necessary};+use crate::execution_plan::CardinalityEffect;use crate::filter_pushdown::{ChildFilterDescription, ChildPushdownResult, FilterDescription, FilterPushdownPhase,FilterPushdownPropagation, FilterRemapper, PushedDown,@@ -962,18 +962,6 @@ impl ExecutionPlan for FilterExec {}))}- fn with_preserve_order(- &self,- preserve_order: bool,- ) -> Option<Arc<dyn ExecutionPlan>> {- self.input- .with_preserve_order(preserve_order)- .and_then(|new_input| {- replace_children_if_necessary(Arc::new(self.clone()), vec![new_input])- .ok()- })- }-#[cfg(feature = "proto")]fn try_to_proto(&self,diff --git a/datafusion/physical-plan/src/limit.rs b/datafusion/physical-plan/src/limit.rsindex ef504a40ea..e2bfefb13e 100644--- a/datafusion/physical-plan/src/limit.rs+++ b/datafusion/physical-plan/src/limit.rs@@ -70,9 +70,6 @@ pub struct GlobalLimitExec {fetch: Option<usize>,/// Execution metricsmetrics: ExecutionPlanMetricsSet,- /// Input ordering that must be preserved so limit pushdown does not change- /// which rows are returned.- required_ordering: Option<LexOrdering>,cache: Arc<PlanProperties>,}@@ -85,7 +82,6 @@ impl GlobalLimitExec {skip,fetch,metrics: ExecutionPlanMetricsSet::new(),- required_ordering: None,cache: Arc::new(cache),}}@@ -116,15 +112,21 @@ impl GlobalLimitExec {)}- /// Get the required ordering from limit+ /// Always `None`: limits no longer record an ordering.+ #[deprecated(+ since = "56.0.0",+ note = "no longer used: a scan that advertises an output ordering always preserves it"+ )]pub fn required_ordering(&self) -> &Option<LexOrdering> {- &self.required_ordering+ &None}- /// Set the required ordering for limit- pub fn set_required_ordering(&mut self, required_ordering: Option<LexOrdering>) {- self.required_ordering = required_ordering;- }+ /// Does nothing: limits no longer record an ordering.+ #[deprecated(+ since = "56.0.0",+ note = "no longer used: a scan that advertises an output ordering always preserves it"+ )]+ pub fn set_required_ordering(&mut self, _required_ordering: Option<LexOrdering>) {}}impl DisplayAs for GlobalLimitExec {@@ -202,12 +204,11 @@ impl ExecutionPlan for GlobalLimitExec {metrics: ExecutionPlanMetricsSet::new(),..Self::clone(&*self)})),- ChildrenPropertiesMode::Recompute => {- let mut new_limit =- GlobalLimitExec::new(children.swap_remove(0), self.skip, self.fetch);- new_limit.set_required_ordering(self.required_ordering.clone());- Ok(Arc::new(new_limit))- }+ ChildrenPropertiesMode::Recompute => Ok(Arc::new(GlobalLimitExec::new(+ children.swap_remove(0),+ self.skip,+ self.fetch,+ ))),}}@@ -291,7 +292,6 @@ impl ExecutionPlan for GlobalLimitExec {&self,ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {- use datafusion_physical_expr_common::sort_expr::optional_ordering_try_to_proto;use datafusion_proto_models::protobuf;// Destructure exhaustively (no `..`) so that adding a field to// `GlobalLimitExec` is a compile error here until it is either@@ -300,15 +300,12 @@ impl ExecutionPlan for GlobalLimitExec {input,skip,fetch,- required_ordering,// Runtime metrics, not part of the plan shape.metrics: _,// Derived plan properties, recomputed on decode.cache: _,} = self;let input = ctx.encode_child(input)?;- let required_ordering =- optional_ordering_try_to_proto(required_ordering.as_ref(), &ctx.expr_ctx())?;Ok(Some(protobuf::PhysicalPlanNode {physical_plan_type: Some(protobuf::physical_plan_node::PhysicalPlanType::GlobalLimit(Box::new(@@ -319,7 +316,8 @@ impl ExecutionPlan for GlobalLimitExec {Some(n) => usize_to_wire(*n, "GlobalLimitExec", "fetch")?,None => -1, // no limit},- required_ordering,+ // Deprecated, no longer written.+ required_ordering: vec![],},)),),@@ -333,7 +331,6 @@ impl GlobalLimitExec {node: &datafusion_proto_models::protobuf::PhysicalPlanNode,ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,) -> Result<Arc<dyn ExecutionPlan>> {- use datafusion_physical_expr_common::sort_expr::optional_ordering_try_from_proto;use datafusion_proto_models::protobuf;let limit = crate::expect_plan_variant!(node,@@ -347,21 +344,16 @@ impl GlobalLimitExec {input,skip,fetch,- required_ordering,+ // Deprecated, ignored.+ required_ordering: _,} = &**limit;let input =ctx.decode_required_child(input.as_deref(), "GlobalLimitExec", "input")?;let fetch = (*fetch >= 0).then(|| usize_from_wire(*fetch, "GlobalLimitExec", "fetch")).transpose()?;- let required_ordering = optional_ordering_try_from_proto(- required_ordering,- &ctx.expr_ctx(input.schema().as_ref()),- )?;let skip = usize_from_wire(*skip, "GlobalLimitExec", "skip")?;- let mut exec = GlobalLimitExec::new(input, skip, fetch);- exec.set_required_ordering(required_ordering);- Ok(Arc::new(exec))+ Ok(Arc::new(GlobalLimitExec::new(input, skip, fetch)))}}@@ -384,9 +376,6 @@ pub struct LocalLimitExec {fetch: usize,/// Execution metricsmetrics: ExecutionPlanMetricsSet,- /// Input ordering that must be preserved so limit pushdown does not change- /// which rows are returned.- required_ordering: Option<LexOrdering>,cache: Arc<PlanProperties>,}@@ -398,7 +387,6 @@ impl LocalLimitExec {input,fetch,metrics: ExecutionPlanMetricsSet::new(),- required_ordering: None,cache: Arc::new(cache),}}@@ -424,15 +412,21 @@ impl LocalLimitExec {)}- /// Get the required ordering from limit+ /// Always `None`: limits no longer record an ordering.+ #[deprecated(+ since = "56.0.0",+ note = "no longer used: a scan that advertises an output ordering always preserves it"+ )]pub fn required_ordering(&self) -> &Option<LexOrdering> {- &self.required_ordering+ &None}- /// Set the required ordering for limit- pub fn set_required_ordering(&mut self, required_ordering: Option<LexOrdering>) {- self.required_ordering = required_ordering;- }+ /// Does nothing: limits no longer record an ordering.+ #[deprecated(+ since = "56.0.0",+ note = "no longer used: a scan that advertises an output ordering always preserves it"+ )]+ pub fn set_required_ordering(&mut self, _required_ordering: Option<LexOrdering>) {}}impl DisplayAs for LocalLimitExec {@@ -493,12 +487,10 @@ impl ExecutionPlan for LocalLimitExec {metrics: ExecutionPlanMetricsSet::new(),..Self::clone(&*self)})),- ChildrenPropertiesMode::Recompute => {- let mut new_limit =- LocalLimitExec::new(children.swap_remove(0), self.fetch);- new_limit.set_required_ordering(self.required_ordering.clone());- Ok(Arc::new(new_limit))- }+ ChildrenPropertiesMode::Recompute => Ok(Arc::new(LocalLimitExec::new(+ children.swap_remove(0),+ self.fetch,+ ))),}}@@ -582,7 +574,6 @@ impl ExecutionPlan for LocalLimitExec {&self,ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {- use datafusion_physical_expr_common::sort_expr::optional_ordering_try_to_proto;use datafusion_proto_models::protobuf;// Destructure exhaustively (no `..`) so that adding a field to// `LocalLimitExec` is a compile error here until it is either@@ -590,22 +581,20 @@ impl ExecutionPlan for LocalLimitExec {let Self {input,fetch,- required_ordering,// Runtime metrics, not part of the plan shape.metrics: _,// Derived plan properties, recomputed on decode.cache: _,} = self;let input = ctx.encode_child(input)?;- let required_ordering =- optional_ordering_try_to_proto(required_ordering.as_ref(), &ctx.expr_ctx())?;Ok(Some(protobuf::PhysicalPlanNode {physical_plan_type: Some(protobuf::physical_plan_node::PhysicalPlanType::LocalLimit(Box::new(protobuf::LocalLimitExecNode {input: Some(Box::new(input)),fetch: usize_to_wire(*fetch, "LocalLimitExec", "fetch")?,- required_ordering,+ // Deprecated, no longer written.+ required_ordering: vec![],},)),),@@ -619,7 +608,6 @@ impl LocalLimitExec {node: &datafusion_proto_models::protobuf::PhysicalPlanNode,ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,) -> Result<Arc<dyn ExecutionPlan>> {- use datafusion_physical_expr_common::sort_expr::optional_ordering_try_from_proto;use datafusion_proto_models::protobuf;let limit = crate::expect_plan_variant!(node,@@ -631,18 +619,13 @@ impl LocalLimitExec {let protobuf::LocalLimitExecNode {input,fetch,- required_ordering,+ // Deprecated, ignored.+ required_ordering: _,} = &**limit;let input =ctx.decode_required_child(input.as_deref(), "LocalLimitExec", "input")?;- let required_ordering = optional_ordering_try_from_proto(- required_ordering,- &ctx.expr_ctx(input.schema().as_ref()),- )?;let fetch = usize_from_wire(*fetch, "LocalLimitExec", "fetch")?;- let mut exec = LocalLimitExec::new(input, fetch);- exec.set_required_ordering(required_ordering);- Ok(Arc::new(exec))+ Ok(Arc::new(LocalLimitExec::new(input, fetch)))}}@@ -781,11 +764,10 @@ mod tests {use crate::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy};use arrow::array::RecordBatchOptions;- use arrow::compute::SortOptions;use arrow::datatypes::Schema;use datafusion_common::stats::Precision;+ use datafusion_physical_expr::PhysicalExpr;use datafusion_physical_expr::expressions::col;- use datafusion_physical_expr::{PhysicalExpr, PhysicalSortExpr};#[tokio::test]async fn limit() -> Result<()> {@@ -975,39 +957,6 @@ mod tests {Ok(())}- #[test]- fn replace_children_preserves_required_ordering() -> Result<()> {- let source = test::scan_partitioned(1);- let schema = source.schema();- let ordering = LexOrdering::new(vec![PhysicalSortExpr {- expr: col("i", &schema)?,- options: SortOptions {- descending: true,- nulls_first: false,- },- }]);-- let mut global = GlobalLimitExec::new(Arc::clone(&source), 0, Some(10));- global.set_required_ordering(ordering.clone());- let rebuilt = Arc::new(global).replace_children(- vec![test::scan_partitioned(1)],- ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),- )?;- let rebuilt = rebuilt.downcast_ref::<GlobalLimitExec>().unwrap();- assert_eq!(rebuilt.required_ordering(), &ordering);-- let mut local = LocalLimitExec::new(source, 10);- local.set_required_ordering(ordering.clone());- let rebuilt = Arc::new(local).replace_children(- vec![test::scan_partitioned(1)],- ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),- )?;- let rebuilt = rebuilt.downcast_ref::<LocalLimitExec>().unwrap();- assert_eq!(rebuilt.required_ordering(), &ordering);-- Ok(())- }-#[test]fn test_row_number_statistics_for_global_limit() -> Result<()> {let row_count = row_number_statistics_for_global_limit(0, Some(10))?;diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rsindex 063ae6d138..52a8471913 100644--- a/datafusion/physical-plan/src/projection.rs+++ b/datafusion/physical-plan/src/projection.rs@@ -622,18 +622,6 @@ impl ExecutionPlan for ProjectionExec {}}- fn with_preserve_order(- &self,- preserve_order: bool,- ) -> Option<Arc<dyn ExecutionPlan>> {- self.input- .with_preserve_order(preserve_order)- .and_then(|new_input| {- replace_children_if_necessary(Arc::new(self.clone()), vec![new_input])- .ok()- })- }-#[cfg(feature = "proto")]fn try_to_proto(&self,diff --git a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rsindex 7c924ff32b..e865fef98f 100644--- a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs+++ b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs@@ -38,9 +38,7 @@ use datafusion_execution::memory_pool::MemoryConsumer;use datafusion_physical_expr::PhysicalExpr;use datafusion_physical_expr_common::sort_expr::{LexOrdering, OrderingRequirements};-use crate::execution_plan::{- CardinalityEffect, EvaluationType, SchedulingType, replace_children_if_necessary,-};+use crate::execution_plan::{CardinalityEffect, EvaluationType, SchedulingType};use log::{debug, trace};/// Sort preserving merge execution plan@@ -243,18 +241,6 @@ impl ExecutionPlan for SortPreservingMergeExec {}))}- fn with_preserve_order(- &self,- preserve_order: bool,- ) -> Option<Arc<dyn ExecutionPlan>> {- self.input- .with_preserve_order(preserve_order)- .and_then(|new_input| {- replace_children_if_necessary(Arc::new(self.clone()), vec![new_input])- .ok()- })- }-fn required_input_distribution(&self) -> Vec<Distribution> {self.input_distribution_requirements().into_per_child()}diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.protoindex 5a3231751a..c1c7e34bd8 100644--- a/datafusion/proto-models/proto/datafusion.proto+++ b/datafusion/proto-models/proto/datafusion.proto@@ -1637,14 +1637,14 @@ message GlobalLimitExecNode {uint32 skip = 2;// Maximum number of rows to fetch; negative means no limitint64 fetch = 3;- // Ordering the limit must preserve; empty means none+ // Deprecated since 56.0.0: no longer written, ignored when readrepeated PhysicalSortExprNode required_ordering = 4;}message LocalLimitExecNode {PhysicalPlanNode input = 1;uint32 fetch = 2;- // Ordering the limit must preserve; empty means none+ // Deprecated since 56.0.0: no longer written, ignored when readrepeated PhysicalSortExprNode required_ordering = 3;}diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rsindex 83b5bfaad7..119d465281 100644--- a/datafusion/proto-models/src/generated/prost.rs+++ b/datafusion/proto-models/src/generated/prost.rs@@ -2452,7 +2452,7 @@ pub struct GlobalLimitExecNode {/// Maximum number of rows to fetch; negative means no limit#[prost(int64, tag = "3")]pub fetch: i64,- /// Ordering the limit must preserve; empty means none+ /// Deprecated since 56.0.0: no longer written, ignored when read#[prost(message, repeated, tag = "4")]pub required_ordering: ::prost::alloc::vec::Vec<PhysicalSortExprNode>,}@@ -2462,7 +2462,7 @@ pub struct LocalLimitExecNode {pub input: ::core::option::Option<::prost::alloc::boxed::Box<PhysicalPlanNode>>,#[prost(uint32, tag = "2")]pub fetch: u32,- /// Ordering the limit must preserve; empty means none+ /// Deprecated since 56.0.0: no longer written, ignored when read#[prost(message, repeated, tag = "3")]pub required_ordering: ::prost::alloc::vec::Vec<PhysicalSortExprNode>,}diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rsindex ecf5b70796..596c856c0c 100644--- a/datafusion/proto/src/physical_plan/mod.rs+++ b/datafusion/proto/src/physical_plan/mod.rs@@ -419,7 +419,10 @@ mod file_scan_config_serde {assert!(!do_not_preserve_with_ordering.output_ordering.is_empty());let mut encoded = serde.encode(&do_not_preserve_with_ordering)?;assert_eq!(encoded.preserve_order, Some(false));- assert!(!serde.decode(&encoded)?.preserve_order);+ let decoded = serde.decode(&encoded)?;+ assert!(!decoded.preserve_order);+ // The output ordering alone keeps the files in order.+ assert!(decoded.preserves_file_order());// Older payloads had no flag and derived it from the ordering.encoded.preserve_order = None;diff --git a/datafusion/proto/tests/cases/plans/limits.rs b/datafusion/proto/tests/cases/plans/limits.rsindex abcdd35fd8..72285c02b8 100644--- a/datafusion/proto/tests/cases/plans/limits.rs+++ b/datafusion/proto/tests/cases/plans/limits.rs@@ -19,35 +19,20 @@//! execs, the coalescing execs, `BufferExec` and `CooperativeExec`.use super::{roundtrip_test, roundtrip_test_and_return};-use datafusion::arrow::compute::kernels::sort::SortOptions;use datafusion::arrow::datatypes::{DataType, Field, Schema};-use datafusion::datasource::listing::PartitionedFile;-use datafusion::datasource::object_store::ObjectStoreUrl;-use datafusion::datasource::physical_plan::{- FileGroup, FileScanConfig, FileScanConfigBuilder, ParquetSource,-};-use datafusion::datasource::source::DataSourceExec;-use datafusion::physical_expr::LexOrdering;-use datafusion::physical_optimizer::PhysicalOptimizerRule;-use datafusion::physical_optimizer::limit_pushdown::LimitPushdown;+use datafusion::physical_plan::ExecutionPlan;use datafusion::physical_plan::buffer::BufferExec;#[expect(deprecated)]use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;use datafusion::physical_plan::coop::CooperativeExec;use datafusion::physical_plan::empty::EmptyExec;-use datafusion::physical_plan::expressions::{PhysicalSortExpr, col};use datafusion::physical_plan::limit::{GlobalLimitExec, LocalLimitExec};-use datafusion::physical_plan::{- ChildrenPropertiesMode, ExecutionPlan, ReplaceChildrenOptions,-};use datafusion::prelude::SessionContext;use datafusion_common::Result;-use datafusion_common::config::ConfigOptions;use datafusion_proto::physical_plan::{AsExecutionPlan, DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,};-#[cfg(target_pointer_width = "32")]use datafusion_proto::protobuf;use datafusion_proto::protobuf::PhysicalPlanNode;use std::sync::Arc;@@ -79,6 +64,51 @@ fn roundtrip_global_skip_no_limit() -> Result<()> {)))}+/// Plans written before 56.0.0 can carry a `required_ordering` on limit+/// nodes. The decoder must accept them and ignore the field. The sort+/// expression has no `expr`, so a decoder that still reads the field fails.+#[test]+fn limit_decode_ignores_required_ordering() -> Result<()> {+ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let required_ordering = vec![protobuf::PhysicalSortExprNode {+ expr: None,+ asc: false,+ nulls_first: true,+ }];+ let input = || Arc::new(EmptyExec::new(Arc::new(Schema::empty())));++ let global: Arc<dyn ExecutionPlan> =+ Arc::new(GlobalLimitExec::new(input(), 3, Some(25)));+ let mut node = PhysicalPlanNode::try_from_physical_plan(global, &codec)?;+ let Some(protobuf::physical_plan_node::PhysicalPlanType::GlobalLimit(limit)) =+ &mut node.physical_plan_type+ else {+ panic!("expected GlobalLimitExecNode");+ };+ limit.required_ordering = required_ordering.clone();+ let decoded = node.try_into_physical_plan(&ctx.task_ctx(), &codec)?;+ let decoded = decoded+ .downcast_ref::<GlobalLimitExec>()+ .expect("expected GlobalLimitExec");+ assert_eq!((decoded.skip(), decoded.fetch()), (3, Some(25)));++ let local: Arc<dyn ExecutionPlan> = Arc::new(LocalLimitExec::new(input(), 25));+ let mut node = PhysicalPlanNode::try_from_physical_plan(local, &codec)?;+ let Some(protobuf::physical_plan_node::PhysicalPlanType::LocalLimit(limit)) =+ &mut node.physical_plan_type+ else {+ panic!("expected LocalLimitExecNode");+ };+ limit.required_ordering = required_ordering;+ let decoded = node.try_into_physical_plan(&ctx.task_ctx(), &codec)?;+ let decoded = decoded+ .downcast_ref::<LocalLimitExec>()+ .expect("expected LocalLimitExec");+ assert_eq!(decoded.fetch(), 25);+ Ok(())+}+#[cfg(target_pointer_width = "64")]#[test]fn local_limit_rejects_fetch_above_wire_range() {@@ -127,111 +157,6 @@ fn global_limit_rejects_fetch_above_usize() -> Result<()> {Ok(())}-/// Sort key at index 1, so a decoder that misbinds column name vs index-/// cannot pass.-fn limit_test_schema() -> Arc<Schema> {- Arc::new(Schema::new(vec![- Field::new("a", DataType::Int64, false),- Field::new("b", DataType::Int64, false),- ]))-}--/// Non-default sort options, so a decode that falls back to defaults cannot-/// pass.-fn limit_required_ordering(schema: &Schema) -> Result<Option<LexOrdering>> {- Ok(LexOrdering::new(vec![PhysicalSortExpr {- expr: col("b", schema)?,- options: SortOptions {- descending: true,- nulls_first: false,- },- }]))-}--#[test]-fn roundtrip_limit_with_required_ordering() -> Result<()> {- let schema = limit_test_schema();- let required_ordering = limit_required_ordering(&schema)?;- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DefaultPhysicalProtoConverter {};-- let mut global =- GlobalLimitExec::new(Arc::new(EmptyExec::new(Arc::clone(&schema))), 3, Some(25));- global.set_required_ordering(required_ordering.clone());- let decoded =- roundtrip_test_and_return(Arc::new(global), &ctx, &codec, &proto_converter)?;- let decoded = decoded- .downcast_ref::<GlobalLimitExec>()- .expect("expected GlobalLimitExec");- assert_eq!(decoded.required_ordering(), &required_ordering);-- let mut local = LocalLimitExec::new(Arc::new(EmptyExec::new(schema)), 25);- local.set_required_ordering(required_ordering.clone());- let decoded =- roundtrip_test_and_return(Arc::new(local), &ctx, &codec, &proto_converter)?;- let decoded = decoded- .downcast_ref::<LocalLimitExec>()- .expect("expected LocalLimitExec");- assert_eq!(decoded.required_ordering(), &required_ordering);- Ok(())-}--/// A limit's `required_ordering` is the only record that an `ORDER BY ... LIMIT`-/// whose sort node was optimized away is order-sensitive, so it must survive-/// serde all the way into the scan's `preserve_order` flag.-#[test]-fn roundtrip_limit_required_ordering_reaches_data_source() -> Result<()> {- let file_schema = limit_test_schema();- let make_scan = || {- let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema)));- let scan_config =- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(- "/path/to/file.parquet".to_string(),- 1024,- )])])- .build();- DataSourceExec::from_data_source(scan_config)- };- let scan_after_limit_pushdown = |limit: GlobalLimitExec| -> Result<FileScanConfig> {- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DefaultPhysicalProtoConverter {};- let decoded =- roundtrip_test_and_return(Arc::new(limit), &ctx, &codec, &proto_converter)?;-- // Child replacement must not erase the decoded ordering before pushdown.- let rebuilt = decoded.replace_children(- vec![make_scan()],- ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),- )?;-- let optimized =- LimitPushdown::new().optimize(rebuilt, &ConfigOptions::default())?;- let scan = optimized- .downcast_ref::<DataSourceExec>()- .expect("limit should be absorbed into the scan");- Ok(scan- .data_source()- .downcast_ref::<FileScanConfig>()- .expect("expected FileScanConfig")- .clone())- };-- let mut limit = GlobalLimitExec::new(make_scan(), 0, Some(10));- limit.set_required_ordering(limit_required_ordering(&file_schema)?);- let scan_config = scan_after_limit_pushdown(limit)?;- assert_eq!(scan_config.limit, Some(10));- assert!(scan_config.preserve_order);-- let scan_config =- scan_after_limit_pushdown(GlobalLimitExec::new(make_scan(), 0, Some(10)))?;- assert_eq!(scan_config.limit, Some(10));- assert!(!scan_config.preserve_order);- Ok(())-}-#[test]fn roundtrip_coalesce_batches_with_fetch() -> Result<()> {let field_a = Field::new("a", DataType::Boolean, false);diff --git a/datafusion/sqllogictest/test_files/limit_pruning.slt b/datafusion/sqllogictest/test_files/limit_pruning.sltindex 5fc58ed75b..0057758962 100644--- a/datafusion/sqllogictest/test_files/limit_pruning.slt+++ b/datafusion/sqllogictest/test_files/limit_pruning.slt@@ -128,6 +128,120 @@ drop table tracking_data;statement okreset datafusion.explain.analyze_level;+# A scan that advertises an ordering must keep it when the fetch of a+# SortPreservingMergeExec is pushed into it: no limit pruning, no work stealing.+statement ok+set datafusion.execution.target_partitions = 2;++statement ok+set datafusion.explain.physical_plan_only = true;++# Row group 0 of a.parquet partially matches `passes = 1` (id 1 fails), row+# group 1 fully matches. Limit pruning would skip row group 0 and return 4, 5, 6.+query I+COPY (SELECT value AS id, CASE WHEN value = 1 THEN 0 ELSE 1 END AS passes FROM generate_series(1, 6))+TO 'test_files/scratch/limit_pruning/ordered_prune/a.parquet'+STORED AS PARQUET OPTIONS ('format.max_row_group_size' '3');+----+6++query I+COPY (SELECT value AS id, 1 AS passes FROM generate_series(101, 103))+TO 'test_files/scratch/limit_pruning/ordered_prune/b.parquet'+STORED AS PARQUET OPTIONS ('format.max_row_group_size' '3');+----+3++statement ok+CREATE EXTERNAL TABLE ordered_prune STORED AS PARQUET+LOCATION 'test_files/scratch/limit_pruning/ordered_prune/'+WITH ORDER (id ASC);++query TT+explain select id from ordered_prune where passes = 1 order by id limit 3;+----+physical_plan+01)SortPreservingMergeExec: [id@0 ASC NULLS LAST], fetch=3+02)--DataSourceExec: <slt:ignore>limit=3, output_ordering=[id@0 ASC NULLS LAST], file_type=parquet<slt:ignore>++query I+select id from ordered_prune where passes = 1 order by id limit 3;+----+2+3+4++statement ok+drop table ordered_prune;++# File groups are [a, b] and [c], each sorted, so the scan advertises `a ASC`.+# c.parquet holds the smallest values. If the two partitions shared one file+# queue (a, b, c), no partition would start with c and the limit would return 0..4.+query I+COPY (SELECT value AS a FROM generate_series(0, 9))+TO 'test_files/scratch/limit_pruning/ordered_steal/a.parquet';+----+10++query I+COPY (SELECT value AS a FROM generate_series(10, 19))+TO 'test_files/scratch/limit_pruning/ordered_steal/b.parquet';+----+10++query I+COPY (SELECT value AS a FROM generate_series(-10, -1))+TO 'test_files/scratch/limit_pruning/ordered_steal/c.parquet';+----+10++statement ok+CREATE EXTERNAL TABLE ordered_steal STORED AS PARQUET+LOCATION 'test_files/scratch/limit_pruning/ordered_steal/'+WITH ORDER (a ASC);++query TT+explain select a from ordered_steal order by a limit 5;+----+physical_plan+01)SortPreservingMergeExec: [a@0 ASC NULLS LAST], fetch=5+02)--DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/ordered_steal/a.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/ordered_steal/b.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/ordered_steal/c.parquet]]}, projection=[a], limit=5, output_ordering=[a@0 ASC NULLS LAST], file_type=parquet++query I+select a from ordered_steal order by a limit 5;+----+-10+-9+-8+-7+-6++# With an OFFSET, the scan gets a fetch of skip + fetch = 5 and must still+# start with c.parquet.+query TT+explain select a from ordered_steal order by a limit 3 offset 2;+----+physical_plan+01)GlobalLimitExec: skip=2, fetch=3+02)--SortPreservingMergeExec: [a@0 ASC NULLS LAST], fetch=5+03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/ordered_steal/a.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/ordered_steal/b.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/ordered_steal/c.parquet]]}, projection=[a], limit=5, output_ordering=[a@0 ASC NULLS LAST], file_type=parquet++query I+select a from ordered_steal order by a limit 3 offset 2;+----+-8+-7+-6++statement ok+drop table ordered_steal;++statement ok+set datafusion.execution.target_partitions = 4;++statement ok+reset datafusion.explain.physical_plan_only;+# Config resetstatement okRESET datafusion.execution.parquet.pushdown_filters;diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.mdindex e0f1aff1a7..648071a251 100644--- a/docs/source/library-user-guide/upgrading/56.0.0.md+++ b/docs/source/library-user-guide/upgrading/56.0.0.md@@ -1183,3 +1183,42 @@ A registered plan needs no `PhysicalExtensionCodec` at all, and two crates' plan`PhysicalExtensionCodec` is not deprecated: extension `PhysicalExpr`s and UDF payloads still need it, and it remains the fallback for unmigrated plans.See [issue #24625](https://github.com/apache/datafusion/issues/24625) for details.++### Order-preservation hooks for limit pushdown are deprecated++A file scan that advertises an output ordering now always reads its files in+order, so `ORDER BY ... LIMIT` queries no longer depend on `LimitPushdown`+telling the scan to keep its order. Before this change, pushing the fetch of a+`SortPreservingMergeExec` into an ordered Parquet scan could return the wrong+rows, because the scan then shared files between partitions or skipped row+groups based on its limit.++The plumbing that carried this request is deprecated and no longer used:++- `ExecutionPlan::with_preserve_order` and `DataSource::with_preserve_order`+ are never called. Built-in operators no longer override them. Overrides in+ custom `ExecutionPlan` or `DataSource` implementations are now dead code,+ and the compiler does not warn about them, because `#[deprecated]` on a+ trait method only warns callers.+- A custom scan that advertises an output ordering must keep that order by+ itself, also when a limit is pushed into it with `with_fetch`. DataFusion no+ longer tells it whether the limit is order-sensitive.+- `GlobalLimitExec::required_ordering` and `LocalLimitExec::required_ordering`+ always return `None`, and their `set_required_ordering` methods do nothing.+ The `required_ordering` protobuf fields are no longer written and are+ ignored when read.++A plain `LIMIT` without `ORDER BY` over a table with a declared ordering now+also reads files in order, so it no longer uses file work stealing or Parquet+limit pruning. Other queries over such tables already read files in order. If+the file groups are uneven, for example a few large files next to many small+ones, a selective `WHERE ... LIMIT` can be slower than before.++**Migration guide:**++To keep a file scan's order, give it an output ordering+(`FileScanConfigBuilder::with_output_ordering`). To read files in group order+without declaring one, use `FileScanConfigBuilder::with_preserve_order(true)`.+Code that reads `FileScanConfig::preserve_order` to decide whether files may be+reordered should call `FileScanConfig::preserves_file_order()` instead, which+also accounts for the output ordering.