Skip to content

Apache DataFusion SQL Query Engine

fix: keep file order for scans that advertise an output ordering (#26434)

Jay Zhan63af08da1b74
diff --git a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs
index b5a2a720cb..084ae59a76 100644
--- a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs
+++ b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs
@@ -2360,14 +2360,12 @@ fn test_spm_fetch_preserves_ordering_through_child_rewrite() -> Result<()> {
.downcast_ref::<LocalLimitExec>()
.expect("SPM fetch should become a local limit");
assert_eq!(limit.fetch(), 100);
- assert_eq!(limit.required_ordering().as_ref(), Some(&ordering));
let config = ConfigOptions::new();
let optimized = ProjectionPushdown::new().optimize(optimized, &config)?;
let limit = optimized
.downcast_ref::<LocalLimitExec>()
.expect("projection rewrite should retain the local limit");
- assert_eq!(limit.required_ordering().as_ref(), Some(&ordering));
assert!(limit.input().is::<DataSourceExec>());
let optimized = LimitPushdown::new().optimize(optimized, &config)?;
@@ -2379,7 +2377,7 @@ fn test_spm_fetch_preserves_ordering_through_child_rewrite() -> Result<()> {
.downcast_ref::<FileScanConfig>()
.expect("parquet scan should use FileScanConfig");
assert_eq!(config.limit, Some(100));
- assert!(config.preserve_order);
+ assert!(config.preserves_file_order());
Ok(())
}
diff --git a/datafusion/core/tests/physical_optimizer/ensure_requirements.rs b/datafusion/core/tests/physical_optimizer/ensure_requirements.rs
index fe73518bf1..8e2c026dcd 100644
--- a/datafusion/core/tests/physical_optimizer/ensure_requirements.rs
+++ b/datafusion/core/tests/physical_optimizer/ensure_requirements.rs
@@ -877,10 +877,10 @@ fn test_fetch_fallback_is_stable() {
let output_children = pushed_down.children();
let merge_children = output_children[0].children();
- let limit = merge_children[0]
- .downcast_ref::<LocalLimitExec>()
- .expect("the fetch should be carried by a local limit");
- assert_eq!(limit.required_ordering(), &Some(ordering));
+ assert!(
+ merge_children[0].is::<LocalLimitExec>(),
+ "the fetch should be carried by a local limit"
+ );
let pushed_down = plan_string(&pushed_down);
let next_pass = plan_string(&next_pass);
diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rs
index 2a8b5d428f..0c7c1ee280 100644
--- a/datafusion/datasource-parquet/src/source.rs
+++ b/datafusion/datasource-parquet/src/source.rs
@@ -636,7 +636,7 @@ impl FileSource for ParquetSource {
.batch_size
.expect("Batch size must set before creating ParquetMorselizer"),
limit: base_config.limit,
- preserve_order: base_config.preserve_order,
+ preserve_order: base_config.preserves_file_order(),
predicate: self.predicate.clone(),
table_schema: self.table_schema.clone(),
metadata_size_hint: self.metadata_size_hint,
diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rs
index 18f952172a..3aa68826cb 100644
--- a/datafusion/datasource/src/file_scan_config/mod.rs
+++ b/datafusion/datasource/src/file_scan_config/mod.rs
@@ -184,10 +184,12 @@ pub struct FileScanConfig {
/// The maximum number of records to read from this plan. If `None`,
/// all records after filtering are returned.
pub limit: Option<usize>,
- /// Whether the scan's limit is order sensitive
- /// When `true`, files must be read in the exact order specified to produce
- /// correct results (e.g., for `ORDER BY ... LIMIT` queries). When `false`,
- /// DataFusion may reorder file processing for optimization without affecting correctness.
+ /// Read files in the order of [`Self::file_groups`].
+ ///
+ /// [`FileScanConfigBuilder::build`] sets this when the scan has an output
+ /// ordering. Readers use [`Self::preserves_file_order`], which also checks
+ /// the current [`Self::output_ordering`], so clearing this flag never
+ /// reorders a scan that advertises an ordering.
pub preserve_order: bool,
/// All equivalent lexicographical output orderings of this file scan, in terms of
/// [`FileSource::table_schema`]. See [`FileScanConfigBuilder::with_output_ordering`] for more
@@ -333,14 +335,10 @@ impl FileScanConfigBuilder {
self
}
- /// Set whether the limit should be order-sensitive.
- ///
- /// When `true`, files must be read in the exact order specified to produce
- /// correct results (e.g., for `ORDER BY ... LIMIT` queries). When `false`,
- /// DataFusion may reorder file processing for optimization without
- /// affecting correctness.
- pub fn with_preserve_order(mut self, order_sensitive: bool) -> Self {
- self.preserve_order = order_sensitive;
+ /// Set whether files must be read in the order of the file groups even
+ /// when the scan has no output ordering. See [`FileScanConfig::preserve_order`].
+ pub fn with_preserve_order(mut self, preserve_order: bool) -> Self {
+ self.preserve_order = preserve_order;
self
}
@@ -1188,18 +1186,6 @@ impl DataSource for FileScanConfig {
}
}
- fn with_preserve_order(&self, preserve_order: bool) -> Option<Arc<dyn DataSource>> {
- if self.preserve_order == preserve_order {
- return Some(Arc::new(self.clone()));
- }
-
- let new_config = FileScanConfig {
- preserve_order,
- ..self.clone()
- };
- Some(Arc::new(new_config))
- }
-
fn apply_expressions(
&self,
f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
@@ -1220,7 +1206,7 @@ impl DataSource for FileScanConfig {
&self,
config: &ConfigOptions,
) -> Option<Arc<dyn Any + Send + Sync>> {
- if self.preserve_order
+ if self.preserves_file_order()
|| self.output_partitioning.is_some()
|| !config.execution.enable_file_stream_work_stealing
{
@@ -1269,6 +1255,16 @@ fn append_known_empty_files(
}
impl FileScanConfig {
+ /// Returns true if files must be read in the order of the file groups.
+ ///
+ /// Parents such as `SortPreservingMergeExec` rely on the output ordering
+ /// this scan advertises, so a scan with an output ordering never shares
+ /// files between partitions (work stealing) and never skips rows based on
+ /// its limit (limit pruning).
+ pub fn preserves_file_order(&self) -> bool {
+ self.preserve_order || !self.output_ordering.is_empty()
+ }
+
/// Returns only the output orderings that are validated against actual
/// file group statistics.
///
@@ -3036,6 +3032,39 @@ mod tests {
}
}
+ /// A scan that advertises an output ordering never shares files between
+ /// partitions, even if `preserve_order` is cleared after it is built.
+ #[test]
+ fn output_ordering_keeps_files_in_order() {
+ let schema =
+ Arc::new(Schema::new(vec![Field::new("a", DataType::Float64, false)]));
+ let sort_expr = PhysicalSortExpr::new_default(Arc::new(Column::new("a", 0)));
+ let mut config = FileScanConfigBuilder::new(
+ ObjectStoreUrl::local_filesystem(),
+ Arc::new(MockSource::new(schema)),
+ )
+ .with_file_groups(vec![
+ FileGroup::new(vec![make_file_with_stats("file1", 0.0, 9.0)]),
+ FileGroup::new(vec![make_file_with_stats("file2", 10.0, 19.0)]),
+ ])
+ .with_output_ordering(vec![LexOrdering::new(vec![sort_expr]).unwrap()])
+ .build();
+ let options = ConfigOptions::default();
+
+ config.preserve_order = false;
+ assert!(config.preserves_file_order());
+ assert!(config.create_sibling_state(&options).is_none());
+
+ config.output_ordering = vec![];
+ assert!(!config.preserves_file_order());
+ assert!(config.create_sibling_state(&options).is_some());
+
+ // An explicit `preserve_order` keeps file order without an output ordering
+ config.preserve_order = true;
+ assert!(config.preserves_file_order());
+ assert!(config.create_sibling_state(&options).is_none());
+ }
+
#[test]
fn test_output_partitioning_not_partitioned_by_file_group() {
let file_schema = aggr_test_schema();
diff --git a/datafusion/datasource/src/source.rs b/datafusion/datasource/src/source.rs
index 741010c595..32a300ff31 100644
--- a/datafusion/datasource/src/source.rs
+++ b/datafusion/datasource/src/source.rs
@@ -223,6 +223,10 @@ pub trait DataSource: Any + Send + Sync + Debug {
}
/// Returns a variant of this `DataSource` that is aware of order-sensitivity.
+ #[deprecated(
+ since = "56.0.0",
+ note = "no longer called: a scan that advertises an output ordering always preserves it"
+ )]
fn with_preserve_order(&self, _preserve_order: bool) -> Option<Arc<dyn DataSource>> {
None
}
@@ -585,18 +589,6 @@ impl ExecutionPlan for DataSourceExec {
})
}
- fn with_preserve_order(
- &self,
- preserve_order: bool,
- ) -> Option<Arc<dyn ExecutionPlan>> {
- self.data_source
- .with_preserve_order(preserve_order)
- .map(|new_data_source| {
- Arc::new(self.clone().with_data_source(new_data_source))
- as Arc<dyn ExecutionPlan>
- })
- }
-
fn with_new_state(
&self,
state: Arc<dyn Any + Send + Sync>,
diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/mod.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/mod.rs
index 414a1d0938..8a4905365b 100644
--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/mod.rs
+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/mod.rs
@@ -467,8 +467,7 @@ pub fn ensure_sorting(
// single partition.
let child_node = requirements.children.swap_remove(0);
if let Some(fetch) = requirements.plan.fetch() {
- let mut limit = LocalLimitExec::new(Arc::clone(&child_node.plan), fetch);
- limit.set_required_ordering(requirements.plan.output_ordering().cloned());
+ let limit = LocalLimitExec::new(Arc::clone(&child_node.plan), fetch);
return Ok(Transformed::yes(PlanContext::new(
Arc::new(limit),
false,
@@ -516,17 +515,11 @@ fn analyze_immediate_sort_removal(
// Remove the sort:
node.children = node.children.swap_remove(0).children;
if let Some(fetch) = sort_exec.fetch() {
- let required_ordering = sort_exec.properties().output_ordering().cloned();
// If the sort has a fetch, we need to add a limit:
if properties.output_partitioning().partition_count() == 1 {
- let mut global_limit =
- GlobalLimitExec::new(Arc::clone(sort_input), 0, Some(fetch));
- global_limit.set_required_ordering(required_ordering);
- Arc::new(global_limit)
+ Arc::new(GlobalLimitExec::new(Arc::clone(sort_input), 0, Some(fetch)))
} else {
- let mut local_limit = LocalLimitExec::new(Arc::clone(sort_input), fetch);
- local_limit.set_required_ordering(required_ordering);
- Arc::new(local_limit)
+ Arc::new(LocalLimitExec::new(Arc::clone(sort_input), fetch))
}
} else {
Arc::clone(sort_input)
diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
index d7f556b90d..867dcb7525 100644
--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
@@ -189,13 +189,9 @@ fn preserve_fetch_above_ordered_plan(
let input = Arc::clone(&node.plan);
let limit: Arc<dyn ExecutionPlan> = if input_has_multiple_partitions {
- let mut limit = LocalLimitExec::new(input, fetch);
- limit.set_required_ordering(Some(ordering.clone()));
- Arc::new(limit)
+ Arc::new(LocalLimitExec::new(input, fetch))
} else {
- let mut limit = GlobalLimitExec::new(input, 0, Some(fetch));
- limit.set_required_ordering(Some(ordering.clone()));
- Arc::new(limit)
+ Arc::new(GlobalLimitExec::new(input, 0, Some(fetch)))
};
let limit_node = PlanContext::new(limit, ParentRequirements::default(), vec![node]);
diff --git a/datafusion/physical-optimizer/src/limit_pushdown.rs b/datafusion/physical-optimizer/src/limit_pushdown.rs
index 226c41cf8d..55e2a71038 100644
--- a/datafusion/physical-optimizer/src/limit_pushdown.rs
+++ b/datafusion/physical-optimizer/src/limit_pushdown.rs
@@ -100,7 +100,6 @@ pub struct GlobalRequirements {
fetch: Option<usize>,
skip: usize,
satisfied: bool,
- preserve_order: bool,
}
impl LimitPushdown {
@@ -128,7 +127,6 @@ impl PhysicalOptimizerRule for LimitPushdown {
fetch: None,
skip: 0,
satisfied: false,
- preserve_order: false,
};
with_statistics_context(context, |stats_ctx| {
pushdown_limits(plan, global_state, stats_ctx)
@@ -148,7 +146,6 @@ struct LimitInfo {
input: Arc<dyn ExecutionPlan>,
fetch: Option<usize>,
skip: usize,
- preserve_order: bool,
}
/// This function is the main helper function of the `LimitPushDown` rule.
@@ -198,7 +195,6 @@ pub fn pushdown_limit_helper_with_stats(
);
global_state.skip = skip;
global_state.fetch = fetch;
- global_state.preserve_order = limit_info.preserve_order;
global_state.satisfied = false;
if let Some(fetch) = fetch
@@ -315,28 +311,17 @@ pub fn pushdown_limit_helper_with_stats(
let maybe_fetchable = pushdown_plan.with_fetch(skip_and_fetch);
if global_state.satisfied {
if let Some(plan_with_fetch) = maybe_fetchable {
- let plan_with_preserve_order = plan_with_fetch
- .with_preserve_order(global_state.preserve_order)
- .unwrap_or(plan_with_fetch);
- Ok((Transformed::yes(plan_with_preserve_order), global_state))
+ Ok((Transformed::yes(plan_with_fetch), global_state))
} else {
Ok((Transformed::no(pushdown_plan), global_state))
}
} else {
global_state.satisfied = true;
pushdown_plan = if let Some(plan_with_fetch) = maybe_fetchable {
- let plan_with_preserve_order = plan_with_fetch
- .with_preserve_order(global_state.preserve_order)
- .unwrap_or(plan_with_fetch);
-
if global_skip > 0 {
- add_global_limit(
- plan_with_preserve_order,
- global_skip,
- Some(global_fetch),
- )
+ add_global_limit(plan_with_fetch, global_skip, Some(global_fetch))
} else {
- plan_with_preserve_order
+ plan_with_fetch
}
} else {
add_limit(pushdown_plan, global_skip, global_fetch)
@@ -460,7 +445,6 @@ fn extract_limit(plan: &Arc<dyn ExecutionPlan>) -> Option<LimitInfo> {
input: Arc::clone(global_limit.input()),
fetch: global_limit.fetch(),
skip: global_limit.skip(),
- preserve_order: global_limit.required_ordering().is_some(),
})
} else {
plan.downcast_ref::<LocalLimitExec>()
@@ -468,7 +452,6 @@ fn extract_limit(plan: &Arc<dyn ExecutionPlan>) -> Option<LimitInfo> {
input: Arc::clone(local_limit.input()),
fetch: Some(local_limit.fetch()),
skip: 0,
- preserve_order: local_limit.required_ordering().is_some(),
})
}
}
diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rs
index d3de70a3d9..7097cc780d 100644
--- a/datafusion/physical-plan/src/coalesce_partitions.rs
+++ b/datafusion/physical-plan/src/coalesce_partitions.rs
@@ -28,9 +28,7 @@ use super::{
DisplayAs, ExecutionPlanProperties, PlanProperties, SendableRecordBatchStream,
Statistics,
};
-use crate::execution_plan::{
- CardinalityEffect, EvaluationType, SchedulingType, replace_children_if_necessary,
-};
+use crate::execution_plan::{CardinalityEffect, EvaluationType, SchedulingType};
use crate::filter_pushdown::{FilterDescription, FilterPushdownPhase};
use crate::projection::{ProjectionExec, make_with_child};
use crate::sort_pushdown::SortOrderPushdownResult;
@@ -347,18 +345,6 @@ impl ExecutionPlan for CoalescePartitionsExec {
}))
}
- fn with_preserve_order(
- &self,
- preserve_order: bool,
- ) -> Option<Arc<dyn ExecutionPlan>> {
- self.input
- .with_preserve_order(preserve_order)
- .and_then(|new_input| {
- replace_children_if_necessary(Arc::new(self.clone()), vec![new_input])
- .ok()
- })
- }
-
fn gather_filters_for_pushdown(
&self,
_phase: FilterPushdownPhase,
diff --git a/datafusion/physical-plan/src/execution_plan.rs b/datafusion/physical-plan/src/execution_plan.rs
index 5d6afc2b9b..ee15653f7d 100644
--- a/datafusion/physical-plan/src/execution_plan.rs
+++ b/datafusion/physical-plan/src/execution_plan.rs
@@ -996,10 +996,12 @@ pub trait ExecutionPlan: Any + Debug + DisplayAs + Send + Sync {
/// Returns a variant of this `ExecutionPlan` that is aware of order-sensitivity.
///
- /// This is used to signal to data sources that the output ordering must be
- /// preserved, even if it might be more efficient to ignore it (e.g. by
- /// skipping some row groups in Parquet).
- ///
+ /// No longer called by DataFusion: a data source that advertises an output
+ /// ordering always preserves it, so no optimizer rule needs to request it.
+ #[deprecated(
+ since = "56.0.0",
+ note = "no longer called: a scan that advertises an output ordering always preserves it"
+ )]
fn with_preserve_order(
&self,
_preserve_order: bool,
diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs
index a9546864c9..2500822c5d 100644
--- a/datafusion/physical-plan/src/filter.rs
+++ b/datafusion/physical-plan/src/filter.rs
@@ -30,7 +30,7 @@ use super::{
};
use crate::coalesce::{LimitedBatchCoalescer, PushBatchStatus};
use crate::common::can_project;
-use crate::execution_plan::{CardinalityEffect, replace_children_if_necessary};
+use crate::execution_plan::CardinalityEffect;
use crate::filter_pushdown::{
ChildFilterDescription, ChildPushdownResult, FilterDescription, FilterPushdownPhase,
FilterPushdownPropagation, FilterRemapper, PushedDown,
@@ -962,18 +962,6 @@ impl ExecutionPlan for FilterExec {
}))
}
- fn with_preserve_order(
- &self,
- preserve_order: bool,
- ) -> Option<Arc<dyn ExecutionPlan>> {
- self.input
- .with_preserve_order(preserve_order)
- .and_then(|new_input| {
- replace_children_if_necessary(Arc::new(self.clone()), vec![new_input])
- .ok()
- })
- }
-
#[cfg(feature = "proto")]
fn try_to_proto(
&self,
diff --git a/datafusion/physical-plan/src/limit.rs b/datafusion/physical-plan/src/limit.rs
index ef504a40ea..e2bfefb13e 100644
--- a/datafusion/physical-plan/src/limit.rs
+++ b/datafusion/physical-plan/src/limit.rs
@@ -70,9 +70,6 @@ pub struct GlobalLimitExec {
fetch: Option<usize>,
/// Execution metrics
metrics: ExecutionPlanMetricsSet,
- /// Input ordering that must be preserved so limit pushdown does not change
- /// which rows are returned.
- required_ordering: Option<LexOrdering>,
cache: Arc<PlanProperties>,
}
@@ -85,7 +82,6 @@ impl GlobalLimitExec {
skip,
fetch,
metrics: ExecutionPlanMetricsSet::new(),
- required_ordering: None,
cache: Arc::new(cache),
}
}
@@ -116,15 +112,21 @@ impl GlobalLimitExec {
)
}
- /// Get the required ordering from limit
+ /// Always `None`: limits no longer record an ordering.
+ #[deprecated(
+ since = "56.0.0",
+ note = "no longer used: a scan that advertises an output ordering always preserves it"
+ )]
pub fn required_ordering(&self) -> &Option<LexOrdering> {
- &self.required_ordering
+ &None
}
- /// Set the required ordering for limit
- pub fn set_required_ordering(&mut self, required_ordering: Option<LexOrdering>) {
- self.required_ordering = required_ordering;
- }
+ /// Does nothing: limits no longer record an ordering.
+ #[deprecated(
+ since = "56.0.0",
+ note = "no longer used: a scan that advertises an output ordering always preserves it"
+ )]
+ pub fn set_required_ordering(&mut self, _required_ordering: Option<LexOrdering>) {}
}
impl DisplayAs for GlobalLimitExec {
@@ -202,12 +204,11 @@ impl ExecutionPlan for GlobalLimitExec {
metrics: ExecutionPlanMetricsSet::new(),
..Self::clone(&*self)
})),
- ChildrenPropertiesMode::Recompute => {
- let mut new_limit =
- GlobalLimitExec::new(children.swap_remove(0), self.skip, self.fetch);
- new_limit.set_required_ordering(self.required_ordering.clone());
- Ok(Arc::new(new_limit))
- }
+ ChildrenPropertiesMode::Recompute => Ok(Arc::new(GlobalLimitExec::new(
+ children.swap_remove(0),
+ self.skip,
+ self.fetch,
+ ))),
}
}
@@ -291,7 +292,6 @@ impl ExecutionPlan for GlobalLimitExec {
&self,
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
- use datafusion_physical_expr_common::sort_expr::optional_ordering_try_to_proto;
use datafusion_proto_models::protobuf;
// Destructure exhaustively (no `..`) so that adding a field to
// `GlobalLimitExec` is a compile error here until it is either
@@ -300,15 +300,12 @@ impl ExecutionPlan for GlobalLimitExec {
input,
skip,
fetch,
- required_ordering,
// Runtime metrics, not part of the plan shape.
metrics: _,
// Derived plan properties, recomputed on decode.
cache: _,
} = self;
let input = ctx.encode_child(input)?;
- let required_ordering =
- optional_ordering_try_to_proto(required_ordering.as_ref(), &ctx.expr_ctx())?;
Ok(Some(protobuf::PhysicalPlanNode {
physical_plan_type: Some(
protobuf::physical_plan_node::PhysicalPlanType::GlobalLimit(Box::new(
@@ -319,7 +316,8 @@ impl ExecutionPlan for GlobalLimitExec {
Some(n) => usize_to_wire(*n, "GlobalLimitExec", "fetch")?,
None => -1, // no limit
},
- required_ordering,
+ // Deprecated, no longer written.
+ required_ordering: vec![],
},
)),
),
@@ -333,7 +331,6 @@ impl GlobalLimitExec {
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
) -> Result<Arc<dyn ExecutionPlan>> {
- use datafusion_physical_expr_common::sort_expr::optional_ordering_try_from_proto;
use datafusion_proto_models::protobuf;
let limit = crate::expect_plan_variant!(
node,
@@ -347,21 +344,16 @@ impl GlobalLimitExec {
input,
skip,
fetch,
- required_ordering,
+ // Deprecated, ignored.
+ required_ordering: _,
} = &**limit;
let input =
ctx.decode_required_child(input.as_deref(), "GlobalLimitExec", "input")?;
let fetch = (*fetch >= 0)
.then(|| usize_from_wire(*fetch, "GlobalLimitExec", "fetch"))
.transpose()?;
- let required_ordering = optional_ordering_try_from_proto(
- required_ordering,
- &ctx.expr_ctx(input.schema().as_ref()),
- )?;
let skip = usize_from_wire(*skip, "GlobalLimitExec", "skip")?;
- let mut exec = GlobalLimitExec::new(input, skip, fetch);
- exec.set_required_ordering(required_ordering);
- Ok(Arc::new(exec))
+ Ok(Arc::new(GlobalLimitExec::new(input, skip, fetch)))
}
}
@@ -384,9 +376,6 @@ pub struct LocalLimitExec {
fetch: usize,
/// Execution metrics
metrics: ExecutionPlanMetricsSet,
- /// Input ordering that must be preserved so limit pushdown does not change
- /// which rows are returned.
- required_ordering: Option<LexOrdering>,
cache: Arc<PlanProperties>,
}
@@ -398,7 +387,6 @@ impl LocalLimitExec {
input,
fetch,
metrics: ExecutionPlanMetricsSet::new(),
- required_ordering: None,
cache: Arc::new(cache),
}
}
@@ -424,15 +412,21 @@ impl LocalLimitExec {
)
}
- /// Get the required ordering from limit
+ /// Always `None`: limits no longer record an ordering.
+ #[deprecated(
+ since = "56.0.0",
+ note = "no longer used: a scan that advertises an output ordering always preserves it"
+ )]
pub fn required_ordering(&self) -> &Option<LexOrdering> {
- &self.required_ordering
+ &None
}
- /// Set the required ordering for limit
- pub fn set_required_ordering(&mut self, required_ordering: Option<LexOrdering>) {
- self.required_ordering = required_ordering;
- }
+ /// Does nothing: limits no longer record an ordering.
+ #[deprecated(
+ since = "56.0.0",
+ note = "no longer used: a scan that advertises an output ordering always preserves it"
+ )]
+ pub fn set_required_ordering(&mut self, _required_ordering: Option<LexOrdering>) {}
}
impl DisplayAs for LocalLimitExec {
@@ -493,12 +487,10 @@ impl ExecutionPlan for LocalLimitExec {
metrics: ExecutionPlanMetricsSet::new(),
..Self::clone(&*self)
})),
- ChildrenPropertiesMode::Recompute => {
- let mut new_limit =
- LocalLimitExec::new(children.swap_remove(0), self.fetch);
- new_limit.set_required_ordering(self.required_ordering.clone());
- Ok(Arc::new(new_limit))
- }
+ ChildrenPropertiesMode::Recompute => Ok(Arc::new(LocalLimitExec::new(
+ children.swap_remove(0),
+ self.fetch,
+ ))),
}
}
@@ -582,7 +574,6 @@ impl ExecutionPlan for LocalLimitExec {
&self,
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
- use datafusion_physical_expr_common::sort_expr::optional_ordering_try_to_proto;
use datafusion_proto_models::protobuf;
// Destructure exhaustively (no `..`) so that adding a field to
// `LocalLimitExec` is a compile error here until it is either
@@ -590,22 +581,20 @@ impl ExecutionPlan for LocalLimitExec {
let Self {
input,
fetch,
- required_ordering,
// Runtime metrics, not part of the plan shape.
metrics: _,
// Derived plan properties, recomputed on decode.
cache: _,
} = self;
let input = ctx.encode_child(input)?;
- let required_ordering =
- optional_ordering_try_to_proto(required_ordering.as_ref(), &ctx.expr_ctx())?;
Ok(Some(protobuf::PhysicalPlanNode {
physical_plan_type: Some(
protobuf::physical_plan_node::PhysicalPlanType::LocalLimit(Box::new(
protobuf::LocalLimitExecNode {
input: Some(Box::new(input)),
fetch: usize_to_wire(*fetch, "LocalLimitExec", "fetch")?,
- required_ordering,
+ // Deprecated, no longer written.
+ required_ordering: vec![],
},
)),
),
@@ -619,7 +608,6 @@ impl LocalLimitExec {
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
) -> Result<Arc<dyn ExecutionPlan>> {
- use datafusion_physical_expr_common::sort_expr::optional_ordering_try_from_proto;
use datafusion_proto_models::protobuf;
let limit = crate::expect_plan_variant!(
node,
@@ -631,18 +619,13 @@ impl LocalLimitExec {
let protobuf::LocalLimitExecNode {
input,
fetch,
- required_ordering,
+ // Deprecated, ignored.
+ required_ordering: _,
} = &**limit;
let input =
ctx.decode_required_child(input.as_deref(), "LocalLimitExec", "input")?;
- let required_ordering = optional_ordering_try_from_proto(
- required_ordering,
- &ctx.expr_ctx(input.schema().as_ref()),
- )?;
let fetch = usize_from_wire(*fetch, "LocalLimitExec", "fetch")?;
- let mut exec = LocalLimitExec::new(input, fetch);
- exec.set_required_ordering(required_ordering);
- Ok(Arc::new(exec))
+ Ok(Arc::new(LocalLimitExec::new(input, fetch)))
}
}
@@ -781,11 +764,10 @@ mod tests {
use crate::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy};
use arrow::array::RecordBatchOptions;
- use arrow::compute::SortOptions;
use arrow::datatypes::Schema;
use datafusion_common::stats::Precision;
+ use datafusion_physical_expr::PhysicalExpr;
use datafusion_physical_expr::expressions::col;
- use datafusion_physical_expr::{PhysicalExpr, PhysicalSortExpr};
#[tokio::test]
async fn limit() -> Result<()> {
@@ -975,39 +957,6 @@ mod tests {
Ok(())
}
- #[test]
- fn replace_children_preserves_required_ordering() -> Result<()> {
- let source = test::scan_partitioned(1);
- let schema = source.schema();
- let ordering = LexOrdering::new(vec![PhysicalSortExpr {
- expr: col("i", &schema)?,
- options: SortOptions {
- descending: true,
- nulls_first: false,
- },
- }]);
-
- let mut global = GlobalLimitExec::new(Arc::clone(&source), 0, Some(10));
- global.set_required_ordering(ordering.clone());
- let rebuilt = Arc::new(global).replace_children(
- vec![test::scan_partitioned(1)],
- ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),
- )?;
- let rebuilt = rebuilt.downcast_ref::<GlobalLimitExec>().unwrap();
- assert_eq!(rebuilt.required_ordering(), &ordering);
-
- let mut local = LocalLimitExec::new(source, 10);
- local.set_required_ordering(ordering.clone());
- let rebuilt = Arc::new(local).replace_children(
- vec![test::scan_partitioned(1)],
- ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),
- )?;
- let rebuilt = rebuilt.downcast_ref::<LocalLimitExec>().unwrap();
- assert_eq!(rebuilt.required_ordering(), &ordering);
-
- Ok(())
- }
-
#[test]
fn test_row_number_statistics_for_global_limit() -> Result<()> {
let row_count = row_number_statistics_for_global_limit(0, Some(10))?;
diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rs
index 063ae6d138..52a8471913 100644
--- a/datafusion/physical-plan/src/projection.rs
+++ b/datafusion/physical-plan/src/projection.rs
@@ -622,18 +622,6 @@ impl ExecutionPlan for ProjectionExec {
}
}
- fn with_preserve_order(
- &self,
- preserve_order: bool,
- ) -> Option<Arc<dyn ExecutionPlan>> {
- self.input
- .with_preserve_order(preserve_order)
- .and_then(|new_input| {
- replace_children_if_necessary(Arc::new(self.clone()), vec![new_input])
- .ok()
- })
- }
-
#[cfg(feature = "proto")]
fn try_to_proto(
&self,
diff --git a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs
index 7c924ff32b..e865fef98f 100644
--- a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs
+++ b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs
@@ -38,9 +38,7 @@ use datafusion_execution::memory_pool::MemoryConsumer;
use datafusion_physical_expr::PhysicalExpr;
use datafusion_physical_expr_common::sort_expr::{LexOrdering, OrderingRequirements};
-use crate::execution_plan::{
- CardinalityEffect, EvaluationType, SchedulingType, replace_children_if_necessary,
-};
+use crate::execution_plan::{CardinalityEffect, EvaluationType, SchedulingType};
use log::{debug, trace};
/// Sort preserving merge execution plan
@@ -243,18 +241,6 @@ impl ExecutionPlan for SortPreservingMergeExec {
}))
}
- fn with_preserve_order(
- &self,
- preserve_order: bool,
- ) -> Option<Arc<dyn ExecutionPlan>> {
- self.input
- .with_preserve_order(preserve_order)
- .and_then(|new_input| {
- replace_children_if_necessary(Arc::new(self.clone()), vec![new_input])
- .ok()
- })
- }
-
fn required_input_distribution(&self) -> Vec<Distribution> {
self.input_distribution_requirements().into_per_child()
}
diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.proto
index 5a3231751a..c1c7e34bd8 100644
--- a/datafusion/proto-models/proto/datafusion.proto
+++ b/datafusion/proto-models/proto/datafusion.proto
@@ -1637,14 +1637,14 @@ message GlobalLimitExecNode {
uint32 skip = 2;
// Maximum number of rows to fetch; negative means no limit
int64 fetch = 3;
- // Ordering the limit must preserve; empty means none
+ // Deprecated since 56.0.0: no longer written, ignored when read
repeated PhysicalSortExprNode required_ordering = 4;
}
message LocalLimitExecNode {
PhysicalPlanNode input = 1;
uint32 fetch = 2;
- // Ordering the limit must preserve; empty means none
+ // Deprecated since 56.0.0: no longer written, ignored when read
repeated PhysicalSortExprNode required_ordering = 3;
}
diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rs
index 83b5bfaad7..119d465281 100644
--- a/datafusion/proto-models/src/generated/prost.rs
+++ b/datafusion/proto-models/src/generated/prost.rs
@@ -2452,7 +2452,7 @@ pub struct GlobalLimitExecNode {
/// Maximum number of rows to fetch; negative means no limit
#[prost(int64, tag = "3")]
pub fetch: i64,
- /// Ordering the limit must preserve; empty means none
+ /// Deprecated since 56.0.0: no longer written, ignored when read
#[prost(message, repeated, tag = "4")]
pub required_ordering: ::prost::alloc::vec::Vec<PhysicalSortExprNode>,
}
@@ -2462,7 +2462,7 @@ pub struct LocalLimitExecNode {
pub input: ::core::option::Option<::prost::alloc::boxed::Box<PhysicalPlanNode>>,
#[prost(uint32, tag = "2")]
pub fetch: u32,
- /// Ordering the limit must preserve; empty means none
+ /// Deprecated since 56.0.0: no longer written, ignored when read
#[prost(message, repeated, tag = "3")]
pub required_ordering: ::prost::alloc::vec::Vec<PhysicalSortExprNode>,
}
diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs
index ecf5b70796..596c856c0c 100644
--- a/datafusion/proto/src/physical_plan/mod.rs
+++ b/datafusion/proto/src/physical_plan/mod.rs
@@ -419,7 +419,10 @@ mod file_scan_config_serde {
assert!(!do_not_preserve_with_ordering.output_ordering.is_empty());
let mut encoded = serde.encode(&do_not_preserve_with_ordering)?;
assert_eq!(encoded.preserve_order, Some(false));
- assert!(!serde.decode(&encoded)?.preserve_order);
+ let decoded = serde.decode(&encoded)?;
+ assert!(!decoded.preserve_order);
+ // The output ordering alone keeps the files in order.
+ assert!(decoded.preserves_file_order());
// Older payloads had no flag and derived it from the ordering.
encoded.preserve_order = None;
diff --git a/datafusion/proto/tests/cases/plans/limits.rs b/datafusion/proto/tests/cases/plans/limits.rs
index abcdd35fd8..72285c02b8 100644
--- a/datafusion/proto/tests/cases/plans/limits.rs
+++ b/datafusion/proto/tests/cases/plans/limits.rs
@@ -19,35 +19,20 @@
//! execs, the coalescing execs, `BufferExec` and `CooperativeExec`.
use super::{roundtrip_test, roundtrip_test_and_return};
-use datafusion::arrow::compute::kernels::sort::SortOptions;
use datafusion::arrow::datatypes::{DataType, Field, Schema};
-use datafusion::datasource::listing::PartitionedFile;
-use datafusion::datasource::object_store::ObjectStoreUrl;
-use datafusion::datasource::physical_plan::{
- FileGroup, FileScanConfig, FileScanConfigBuilder, ParquetSource,
-};
-use datafusion::datasource::source::DataSourceExec;
-use datafusion::physical_expr::LexOrdering;
-use datafusion::physical_optimizer::PhysicalOptimizerRule;
-use datafusion::physical_optimizer::limit_pushdown::LimitPushdown;
+use datafusion::physical_plan::ExecutionPlan;
use datafusion::physical_plan::buffer::BufferExec;
#[expect(deprecated)]
use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;
use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
use datafusion::physical_plan::coop::CooperativeExec;
use datafusion::physical_plan::empty::EmptyExec;
-use datafusion::physical_plan::expressions::{PhysicalSortExpr, col};
use datafusion::physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
-use datafusion::physical_plan::{
- ChildrenPropertiesMode, ExecutionPlan, ReplaceChildrenOptions,
-};
use datafusion::prelude::SessionContext;
use datafusion_common::Result;
-use datafusion_common::config::ConfigOptions;
use datafusion_proto::physical_plan::{
AsExecutionPlan, DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,
};
-#[cfg(target_pointer_width = "32")]
use datafusion_proto::protobuf;
use datafusion_proto::protobuf::PhysicalPlanNode;
use std::sync::Arc;
@@ -79,6 +64,51 @@ fn roundtrip_global_skip_no_limit() -> Result<()> {
)))
}
+/// Plans written before 56.0.0 can carry a `required_ordering` on limit
+/// nodes. The decoder must accept them and ignore the field. The sort
+/// expression has no `expr`, so a decoder that still reads the field fails.
+#[test]
+fn limit_decode_ignores_required_ordering() -> Result<()> {
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let required_ordering = vec![protobuf::PhysicalSortExprNode {
+ expr: None,
+ asc: false,
+ nulls_first: true,
+ }];
+ let input = || Arc::new(EmptyExec::new(Arc::new(Schema::empty())));
+
+ let global: Arc<dyn ExecutionPlan> =
+ Arc::new(GlobalLimitExec::new(input(), 3, Some(25)));
+ let mut node = PhysicalPlanNode::try_from_physical_plan(global, &codec)?;
+ let Some(protobuf::physical_plan_node::PhysicalPlanType::GlobalLimit(limit)) =
+ &mut node.physical_plan_type
+ else {
+ panic!("expected GlobalLimitExecNode");
+ };
+ limit.required_ordering = required_ordering.clone();
+ let decoded = node.try_into_physical_plan(&ctx.task_ctx(), &codec)?;
+ let decoded = decoded
+ .downcast_ref::<GlobalLimitExec>()
+ .expect("expected GlobalLimitExec");
+ assert_eq!((decoded.skip(), decoded.fetch()), (3, Some(25)));
+
+ let local: Arc<dyn ExecutionPlan> = Arc::new(LocalLimitExec::new(input(), 25));
+ let mut node = PhysicalPlanNode::try_from_physical_plan(local, &codec)?;
+ let Some(protobuf::physical_plan_node::PhysicalPlanType::LocalLimit(limit)) =
+ &mut node.physical_plan_type
+ else {
+ panic!("expected LocalLimitExecNode");
+ };
+ limit.required_ordering = required_ordering;
+ let decoded = node.try_into_physical_plan(&ctx.task_ctx(), &codec)?;
+ let decoded = decoded
+ .downcast_ref::<LocalLimitExec>()
+ .expect("expected LocalLimitExec");
+ assert_eq!(decoded.fetch(), 25);
+ Ok(())
+}
+
#[cfg(target_pointer_width = "64")]
#[test]
fn local_limit_rejects_fetch_above_wire_range() {
@@ -127,111 +157,6 @@ fn global_limit_rejects_fetch_above_usize() -> Result<()> {
Ok(())
}
-/// Sort key at index 1, so a decoder that misbinds column name vs index
-/// cannot pass.
-fn limit_test_schema() -> Arc<Schema> {
- Arc::new(Schema::new(vec![
- Field::new("a", DataType::Int64, false),
- Field::new("b", DataType::Int64, false),
- ]))
-}
-
-/// Non-default sort options, so a decode that falls back to defaults cannot
-/// pass.
-fn limit_required_ordering(schema: &Schema) -> Result<Option<LexOrdering>> {
- Ok(LexOrdering::new(vec![PhysicalSortExpr {
- expr: col("b", schema)?,
- options: SortOptions {
- descending: true,
- nulls_first: false,
- },
- }]))
-}
-
-#[test]
-fn roundtrip_limit_with_required_ordering() -> Result<()> {
- let schema = limit_test_schema();
- let required_ordering = limit_required_ordering(&schema)?;
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DefaultPhysicalProtoConverter {};
-
- let mut global =
- GlobalLimitExec::new(Arc::new(EmptyExec::new(Arc::clone(&schema))), 3, Some(25));
- global.set_required_ordering(required_ordering.clone());
- let decoded =
- roundtrip_test_and_return(Arc::new(global), &ctx, &codec, &proto_converter)?;
- let decoded = decoded
- .downcast_ref::<GlobalLimitExec>()
- .expect("expected GlobalLimitExec");
- assert_eq!(decoded.required_ordering(), &required_ordering);
-
- let mut local = LocalLimitExec::new(Arc::new(EmptyExec::new(schema)), 25);
- local.set_required_ordering(required_ordering.clone());
- let decoded =
- roundtrip_test_and_return(Arc::new(local), &ctx, &codec, &proto_converter)?;
- let decoded = decoded
- .downcast_ref::<LocalLimitExec>()
- .expect("expected LocalLimitExec");
- assert_eq!(decoded.required_ordering(), &required_ordering);
- Ok(())
-}
-
-/// A limit's `required_ordering` is the only record that an `ORDER BY ... LIMIT`
-/// whose sort node was optimized away is order-sensitive, so it must survive
-/// serde all the way into the scan's `preserve_order` flag.
-#[test]
-fn roundtrip_limit_required_ordering_reaches_data_source() -> Result<()> {
- let file_schema = limit_test_schema();
- let make_scan = || {
- let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema)));
- let scan_config =
- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
- "/path/to/file.parquet".to_string(),
- 1024,
- )])])
- .build();
- DataSourceExec::from_data_source(scan_config)
- };
- let scan_after_limit_pushdown = |limit: GlobalLimitExec| -> Result<FileScanConfig> {
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DefaultPhysicalProtoConverter {};
- let decoded =
- roundtrip_test_and_return(Arc::new(limit), &ctx, &codec, &proto_converter)?;
-
- // Child replacement must not erase the decoded ordering before pushdown.
- let rebuilt = decoded.replace_children(
- vec![make_scan()],
- ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),
- )?;
-
- let optimized =
- LimitPushdown::new().optimize(rebuilt, &ConfigOptions::default())?;
- let scan = optimized
- .downcast_ref::<DataSourceExec>()
- .expect("limit should be absorbed into the scan");
- Ok(scan
- .data_source()
- .downcast_ref::<FileScanConfig>()
- .expect("expected FileScanConfig")
- .clone())
- };
-
- let mut limit = GlobalLimitExec::new(make_scan(), 0, Some(10));
- limit.set_required_ordering(limit_required_ordering(&file_schema)?);
- let scan_config = scan_after_limit_pushdown(limit)?;
- assert_eq!(scan_config.limit, Some(10));
- assert!(scan_config.preserve_order);
-
- let scan_config =
- scan_after_limit_pushdown(GlobalLimitExec::new(make_scan(), 0, Some(10)))?;
- assert_eq!(scan_config.limit, Some(10));
- assert!(!scan_config.preserve_order);
- Ok(())
-}
-
#[test]
fn roundtrip_coalesce_batches_with_fetch() -> Result<()> {
let field_a = Field::new("a", DataType::Boolean, false);
diff --git a/datafusion/sqllogictest/test_files/limit_pruning.slt b/datafusion/sqllogictest/test_files/limit_pruning.slt
index 5fc58ed75b..0057758962 100644
--- a/datafusion/sqllogictest/test_files/limit_pruning.slt
+++ b/datafusion/sqllogictest/test_files/limit_pruning.slt
@@ -128,6 +128,120 @@ drop table tracking_data;
statement ok
reset datafusion.explain.analyze_level;
+# A scan that advertises an ordering must keep it when the fetch of a
+# SortPreservingMergeExec is pushed into it: no limit pruning, no work stealing.
+statement ok
+set datafusion.execution.target_partitions = 2;
+
+statement ok
+set datafusion.explain.physical_plan_only = true;
+
+# Row group 0 of a.parquet partially matches `passes = 1` (id 1 fails), row
+# group 1 fully matches. Limit pruning would skip row group 0 and return 4, 5, 6.
+query I
+COPY (SELECT value AS id, CASE WHEN value = 1 THEN 0 ELSE 1 END AS passes FROM generate_series(1, 6))
+TO 'test_files/scratch/limit_pruning/ordered_prune/a.parquet'
+STORED AS PARQUET OPTIONS ('format.max_row_group_size' '3');
+----
+6
+
+query I
+COPY (SELECT value AS id, 1 AS passes FROM generate_series(101, 103))
+TO 'test_files/scratch/limit_pruning/ordered_prune/b.parquet'
+STORED AS PARQUET OPTIONS ('format.max_row_group_size' '3');
+----
+3
+
+statement ok
+CREATE EXTERNAL TABLE ordered_prune STORED AS PARQUET
+LOCATION 'test_files/scratch/limit_pruning/ordered_prune/'
+WITH ORDER (id ASC);
+
+query TT
+explain select id from ordered_prune where passes = 1 order by id limit 3;
+----
+physical_plan
+01)SortPreservingMergeExec: [id@0 ASC NULLS LAST], fetch=3
+02)--DataSourceExec: <slt:ignore>limit=3, output_ordering=[id@0 ASC NULLS LAST], file_type=parquet<slt:ignore>
+
+query I
+select id from ordered_prune where passes = 1 order by id limit 3;
+----
+2
+3
+4
+
+statement ok
+drop table ordered_prune;
+
+# File groups are [a, b] and [c], each sorted, so the scan advertises `a ASC`.
+# c.parquet holds the smallest values. If the two partitions shared one file
+# queue (a, b, c), no partition would start with c and the limit would return 0..4.
+query I
+COPY (SELECT value AS a FROM generate_series(0, 9))
+TO 'test_files/scratch/limit_pruning/ordered_steal/a.parquet';
+----
+10
+
+query I
+COPY (SELECT value AS a FROM generate_series(10, 19))
+TO 'test_files/scratch/limit_pruning/ordered_steal/b.parquet';
+----
+10
+
+query I
+COPY (SELECT value AS a FROM generate_series(-10, -1))
+TO 'test_files/scratch/limit_pruning/ordered_steal/c.parquet';
+----
+10
+
+statement ok
+CREATE EXTERNAL TABLE ordered_steal STORED AS PARQUET
+LOCATION 'test_files/scratch/limit_pruning/ordered_steal/'
+WITH ORDER (a ASC);
+
+query TT
+explain select a from ordered_steal order by a limit 5;
+----
+physical_plan
+01)SortPreservingMergeExec: [a@0 ASC NULLS LAST], fetch=5
+02)--DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/ordered_steal/a.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/ordered_steal/b.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/ordered_steal/c.parquet]]}, projection=[a], limit=5, output_ordering=[a@0 ASC NULLS LAST], file_type=parquet
+
+query I
+select a from ordered_steal order by a limit 5;
+----
+-10
+-9
+-8
+-7
+-6
+
+# With an OFFSET, the scan gets a fetch of skip + fetch = 5 and must still
+# start with c.parquet.
+query TT
+explain select a from ordered_steal order by a limit 3 offset 2;
+----
+physical_plan
+01)GlobalLimitExec: skip=2, fetch=3
+02)--SortPreservingMergeExec: [a@0 ASC NULLS LAST], fetch=5
+03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/ordered_steal/a.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/ordered_steal/b.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/ordered_steal/c.parquet]]}, projection=[a], limit=5, output_ordering=[a@0 ASC NULLS LAST], file_type=parquet
+
+query I
+select a from ordered_steal order by a limit 3 offset 2;
+----
+-8
+-7
+-6
+
+statement ok
+drop table ordered_steal;
+
+statement ok
+set datafusion.execution.target_partitions = 4;
+
+statement ok
+reset datafusion.explain.physical_plan_only;
+
# Config reset
statement ok
RESET datafusion.execution.parquet.pushdown_filters;
diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.md
index e0f1aff1a7..648071a251 100644
--- a/docs/source/library-user-guide/upgrading/56.0.0.md
+++ b/docs/source/library-user-guide/upgrading/56.0.0.md
@@ -1183,3 +1183,42 @@ A registered plan needs no `PhysicalExtensionCodec` at all, and two crates' plan
`PhysicalExtensionCodec` is not deprecated: extension `PhysicalExpr`s and UDF payloads still need it, and it remains the fallback for unmigrated plans.
See [issue #24625](https://github.com/apache/datafusion/issues/24625) for details.
+
+### Order-preservation hooks for limit pushdown are deprecated
+
+A file scan that advertises an output ordering now always reads its files in
+order, so `ORDER BY ... LIMIT` queries no longer depend on `LimitPushdown`
+telling the scan to keep its order. Before this change, pushing the fetch of a
+`SortPreservingMergeExec` into an ordered Parquet scan could return the wrong
+rows, because the scan then shared files between partitions or skipped row
+groups based on its limit.
+
+The plumbing that carried this request is deprecated and no longer used:
+
+- `ExecutionPlan::with_preserve_order` and `DataSource::with_preserve_order`
+ are never called. Built-in operators no longer override them. Overrides in
+ custom `ExecutionPlan` or `DataSource` implementations are now dead code,
+ and the compiler does not warn about them, because `#[deprecated]` on a
+ trait method only warns callers.
+- A custom scan that advertises an output ordering must keep that order by
+ itself, also when a limit is pushed into it with `with_fetch`. DataFusion no
+ longer tells it whether the limit is order-sensitive.
+- `GlobalLimitExec::required_ordering` and `LocalLimitExec::required_ordering`
+ always return `None`, and their `set_required_ordering` methods do nothing.
+ The `required_ordering` protobuf fields are no longer written and are
+ ignored when read.
+
+A plain `LIMIT` without `ORDER BY` over a table with a declared ordering now
+also reads files in order, so it no longer uses file work stealing or Parquet
+limit pruning. Other queries over such tables already read files in order. If
+the file groups are uneven, for example a few large files next to many small
+ones, a selective `WHERE ... LIMIT` can be slower than before.
+
+**Migration guide:**
+
+To keep a file scan's order, give it an output ordering
+(`FileScanConfigBuilder::with_output_ordering`). To read files in group order
+without declaring one, use `FileScanConfigBuilder::with_preserve_order(true)`.
+Code that reads `FileScanConfig::preserve_order` to decide whether files may be
+reordered should call `FileScanConfig::preserves_file_order()` instead, which
+also accounts for the output ordering.