Skip to content

Apache DataFusion SQL Query Engine

refactor: one vocabulary for moving LIMIT through the physical plan (#26660)

Jay Zhan8df818a4e618
diff --git a/datafusion/core/tests/physical_optimizer/ensure_requirements.rs b/datafusion/core/tests/physical_optimizer/ensure_requirements.rs
index 85c969ca00..86adb307db 100644
--- a/datafusion/core/tests/physical_optimizer/ensure_requirements.rs
+++ b/datafusion/core/tests/physical_optimizer/ensure_requirements.rs
@@ -493,6 +493,35 @@ fn test_skip_and_fetch_multi_partition() {
");
}
+/// A merge with its own fetch already selects rows, so the fetch of a
+/// stricter TopK above it must stay above it.
+#[test]
+fn test_fetched_merge_keeps_stricter_topk_above() {
+ let source = Arc::new(MockMultiPartitionExec::new(4));
+ let a = sort_expr_on("a", 0, false, false);
+ let a_b = LexOrdering::new(
+ a.iter()
+ .cloned()
+ .chain(sort_expr_on("b", 1, false, false).iter().cloned()),
+ )
+ .unwrap();
+ let inner = Arc::new(
+ SortExec::new(a.clone(), source)
+ .with_fetch(Some(5))
+ .with_preserve_partitioning(true),
+ );
+ let merge = Arc::new(SortPreservingMergeExec::new(a, inner).with_fetch(Some(5)));
+ let topk: Arc<dyn ExecutionPlan> =
+ Arc::new(SortExec::new(a_b, merge).with_fetch(Some(2)));
+
+ assert_ensure_requirements_plan!(topk, @r"
+ SortExec: TopK(fetch=2), expr=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST], preserve_partitioning=[false], sort_prefix=[a@0 ASC NULLS LAST]
+ SortPreservingMergeExec: [a@0 ASC NULLS LAST], fetch=5
+ LocalLimitExec: fetch=5
+ MockMultiPartitionExec
+ ");
+}
+
// ========================================================================
// Multiple sort columns test
// ========================================================================
diff --git a/datafusion/core/tests/physical_optimizer/limit_pushdown.rs b/datafusion/core/tests/physical_optimizer/limit_pushdown.rs
index 50ba524f97..1d05d326e0 100644
--- a/datafusion/core/tests/physical_optimizer/limit_pushdown.rs
+++ b/datafusion/core/tests/physical_optimizer/limit_pushdown.rs
@@ -2,6 +2,7 @@
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
+use std::fmt;
use std::sync::Arc;
use crate::physical_optimizer::test_utils::{
@@ -11,15 +12,17 @@ use crate::physical_optimizer::test_utils::{
use arrow::compute::SortOptions;
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+use datafusion::execution::TaskContext;
use datafusion::execution::session_state::SessionStateBuilder;
use datafusion::prelude::{SessionConfig, SessionContext};
use datafusion_common::config::ConfigOptions;
use datafusion_common::error::Result;
use datafusion_common::stats::Precision;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_expr::{JoinType, Operator};
-use datafusion_physical_expr::Partitioning;
use datafusion_physical_expr::expressions::{BinaryExpr, col, lit};
-use datafusion_physical_expr_common::physical_expr::PhysicalExprRef;
+use datafusion_physical_expr::{EquivalenceProperties, Partitioning};
+use datafusion_physical_expr_common::physical_expr::{PhysicalExpr, PhysicalExprRef};
use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};
use datafusion_physical_optimizer::PhysicalOptimizerRule;
use datafusion_physical_optimizer::limit_pushdown::LimitPushdown;
@@ -33,7 +36,10 @@ use datafusion_physical_plan::operator_statistics::{
use datafusion_physical_plan::projection::ProjectionExec;
use datafusion_physical_plan::repartition::RepartitionExec;
use datafusion_physical_plan::statistics::StatisticsArgs;
-use datafusion_physical_plan::{ExecutionPlan, get_plan_string};
+use datafusion_physical_plan::{
+ DisplayAs, DisplayFormatType, ExecutionPlan, LimitBehavior, PlanProperties,
+ SendableRecordBatchStream, get_plan_string,
+};
fn create_schema() -> SchemaRef {
Arc::new(Schema::new(vec![
@@ -439,6 +445,168 @@ fn keeps_pushed_local_limit_exec_when_there_are_multiple_input_partitions() -> R
Ok(())
}
+/// Outputs the rows of all its children's partitions in one partition, and
+/// declares `behavior` as its `limit_behavior`. It can't take a fetch.
+#[derive(Debug)]
+struct CombineExec {
+ inputs: Vec<Arc<dyn ExecutionPlan>>,
+ behavior: LimitBehavior,
+ properties: Arc<PlanProperties>,
+}
+
+impl CombineExec {
+ fn new(inputs: Vec<Arc<dyn ExecutionPlan>>, behavior: LimitBehavior) -> Self {
+ let properties = Arc::new(PlanProperties::new(
+ EquivalenceProperties::new(inputs[0].schema()),
+ Partitioning::UnknownPartitioning(1),
+ inputs[0].properties().emission_type,
+ inputs[0].properties().boundedness,
+ ));
+ Self {
+ inputs,
+ behavior,
+ properties,
+ }
+ }
+}
+
+impl DisplayAs for CombineExec {
+ fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {
+ write!(f, "CombineExec: {:?}", self.behavior)
+ }
+}
+
+impl ExecutionPlan for CombineExec {
+ fn name(&self) -> &str {
+ "CombineExec"
+ }
+
+ fn properties(&self) -> &Arc<PlanProperties> {
+ &self.properties
+ }
+
+ fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
+ self.inputs.iter().collect()
+ }
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
+ fn with_new_children(
+ self: Arc<Self>,
+ children: Vec<Arc<dyn ExecutionPlan>>,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ Ok(Arc::new(Self::new(children, self.behavior)))
+ }
+
+ fn limit_behavior(&self) -> LimitBehavior {
+ self.behavior
+ }
+
+ fn execute(
+ &self,
+ _partition: usize,
+ _context: Arc<TaskContext>,
+ ) -> Result<SendableRecordBatchStream> {
+ unimplemented!("only planned in these tests")
+ }
+}
+
+#[test]
+fn merge_without_fetch_gets_offset_rows_below() -> Result<()> {
+ let schema = create_schema();
+ let streaming_table = stream_exec(&schema);
+ let repartition = repartition_exec(streaming_table)?;
+ let filter = filter_exec(schema, repartition)?;
+ let merge: Arc<dyn ExecutionPlan> =
+ Arc::new(CombineExec::new(vec![filter], LimitBehavior::Merge));
+ let global_limit = global_limit_exec(merge, 5, Some(3));
+
+ let after_optimize =
+ LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;
+
+ // The merge can't take the limit, so it stays above it. Each partition
+ // below must still give the 5 skipped rows plus the 3 kept ones.
+ let optimized = format_plan(&after_optimize);
+ insta::assert_snapshot!(
+ optimized,
+ @r"
+ GlobalLimitExec: skip=5, fetch=3
+ CombineExec: Merge
+ FilterExec: c3@2 > 0, fetch=8
+ RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1
+ StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true
+ "
+ );
+
+ Ok(())
+}
+
+#[test]
+fn copy_node_with_several_children_and_one_output_partition_keeps_limit_above()
+-> Result<()> {
+ let schema = create_schema();
+ let left = filter_exec(Arc::clone(&schema), stream_exec(&schema))?;
+ let right = filter_exec(Arc::clone(&schema), stream_exec(&schema))?;
+ let combine: Arc<dyn ExecutionPlan> = Arc::new(CombineExec::new(
+ vec![left, right],
+ LimitBehavior::CopyToChildren,
+ ));
+ let global_limit = global_limit_exec(combine, 0, Some(5));
+
+ let after_optimize =
+ LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;
+
+ // Moved below, the limit would keep 5 rows of each child: 10 in total.
+ let optimized = format_plan(&after_optimize);
+ insta::assert_snapshot!(
+ optimized,
+ @r"
+ GlobalLimitExec: skip=0, fetch=5
+ CombineExec: CopyToChildren
+ FilterExec: c3@2 > 0, fetch=5
+ StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true
+ FilterExec: c3@2 > 0, fetch=5
+ StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true
+ "
+ );
+
+ Ok(())
+}
+
+#[test]
+fn move_node_that_merges_partitions_keeps_limit_above() -> Result<()> {
+ let schema = create_schema();
+ let streaming_table = stream_exec(&schema);
+ let repartition = repartition_exec(streaming_table)?;
+ let filter = filter_exec(schema, repartition)?;
+ let combine: Arc<dyn ExecutionPlan> =
+ Arc::new(CombineExec::new(vec![filter], LimitBehavior::MoveToInput));
+ let global_limit = global_limit_exec(combine, 0, Some(5));
+
+ let after_optimize =
+ LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;
+
+ // Moved below, the limit would keep 5 rows of each of the 8 partitions.
+ let optimized = format_plan(&after_optimize);
+ insta::assert_snapshot!(
+ optimized,
+ @r"
+ GlobalLimitExec: skip=0, fetch=5
+ CombineExec: MoveToInput
+ FilterExec: c3@2 > 0, fetch=5
+ RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1
+ StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true
+ "
+ );
+
+ Ok(())
+}
+
#[test]
fn merges_local_limit_with_local_limit() -> Result<()> {
let schema = create_schema();
diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/mod.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/mod.rs
index a340339454..bcbc311c32 100644
--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/mod.rs
+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/mod.rs
@@ -36,6 +36,7 @@ pub mod sort_pushdown;
use std::sync::Arc;
use crate::output_requirements::OutputRequirementExec;
+use crate::row_bound::{RowBound, limit_node};
use crate::utils::{
add_sort_above, add_sort_above_with_check, is_coalesce_partitions, is_limit,
is_repartition, is_sort, is_sort_preserving_merge, is_window,
@@ -47,7 +48,7 @@ use datafusion_common::tree_node::Transformed;
use datafusion_physical_expr::{Distribution, Partitioning};
use datafusion_physical_expr_common::sort_expr::{LexOrdering, LexRequirement};
use datafusion_physical_plan::coalesce_partitions::CoalescePartitionsExec;
-use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
+use datafusion_physical_plan::limit::LocalLimitExec;
use datafusion_physical_plan::repartition::RepartitionExec;
use datafusion_physical_plan::sorts::partial_sort::PartialSortExec;
use datafusion_physical_plan::sorts::sort::SortExec;
@@ -500,17 +501,13 @@ fn analyze_immediate_sort_removal(
.with_fetch(sort_exec.fetch()),
) as _
} else {
- // Remove the sort:
+ // Remove the sort, keeping its fetch as a limit:
node.children = node.children.swap_remove(0).children;
- if let Some(fetch) = sort_exec.fetch() {
- // If the sort has a fetch, we need to add a limit:
- if properties.output_partitioning().partition_count() == 1 {
- Arc::new(GlobalLimitExec::new(Arc::clone(sort_input), 0, Some(fetch)))
- } else {
- Arc::new(LocalLimitExec::new(Arc::clone(sort_input), fetch))
+ match sort_exec.fetch() {
+ Some(fetch) => {
+ limit_node(Arc::clone(sort_input), RowBound::fetch(Some(fetch)))
}
- } else {
- Arc::clone(sort_input)
+ None => Arc::clone(sort_input),
}
};
for child in node.children.iter_mut() {
diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
index 65a71510c9..e05e8242b1 100644
--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
@@ -24,7 +24,6 @@ use datafusion_physical_expr_common::sort_expr::{
PhysicalSortRequirement,
};
use datafusion_physical_plan::aggregates::AggregateExec;
-use datafusion_physical_plan::execution_plan::CardinalityEffect;
use datafusion_physical_plan::filter::FilterExec;
use datafusion_physical_plan::joins::utils::{
ColumnIndex, calculate_join_output_ordering,
@@ -38,9 +37,11 @@ use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeE
use datafusion_physical_plan::tree_node::PlanContext;
use datafusion_physical_plan::{
ChildSatisfactionOptions, ChildrenPropertiesMode, ExecutionPlan,
- ExecutionPlanProperties, ReplaceChildrenOptions,
+ ExecutionPlanProperties, LimitBehavior, ReplaceChildrenOptions,
};
+use crate::row_bound::{RowBound, limit_node};
+
/// "Data class" used by sort pushdown (now driven from `EnsureRequirements`)
/// to push down [`SortExec`] in the plan. In some cases the total
/// computational cost is reduced by pushing down `SortExec`s through certain
@@ -83,21 +84,19 @@ pub type SortPushDown = PlanContext<ParentRequirements>;
/// fetch is the tighter bound. Using the bare `fetch` would be wrong too, as
/// it would turn `LIMIT 10 OFFSET 5` into `TopK(10)` below the limit, i.e. 5
/// result rows.
-///
-/// `skip` and `fetch` are independent `usize`s, so their sum can overflow. Like
-/// [`combine_limit`], we saturate: `usize::MAX` input rows is never a smaller
-/// bound than the real one, so the pushed-down fetch stays correct.
-///
-/// [`combine_limit`]: datafusion_common::utils::combine_limit
fn input_fetch(
plan: &Arc<dyn ExecutionPlan>,
parent_fetch: Option<usize>,
) -> Option<usize> {
- let fetch = min_fetch(plan.fetch(), parent_fetch)?;
let skip = plan
.downcast_ref::<GlobalLimitExec>()
.map_or(0, |limit| limit.skip());
- Some(fetch.saturating_add(skip))
+ let output = RowBound::fetch(plan.fetch()).min(RowBound::fetch(parent_fetch));
+ RowBound {
+ skip,
+ fetch: output.fetch,
+ }
+ .input_rows()
}
/// Assigns the ordering requirement of the root node to the its children.
@@ -129,18 +128,16 @@ pub fn pushdown_sorts(sort_push_down: SortPushDown) -> Result<SortPushDown> {
}
fn min_fetch(f1: Option<usize>, f2: Option<usize>) -> Option<usize> {
- match (f1, f2) {
- (Some(f1), Some(f2)) => Some(f1.min(f2)),
- (Some(_), _) => f1,
- (_, Some(_)) => f2,
- _ => None,
- }
+ RowBound::fetch(f1).min(RowBound::fetch(f2)).fetch
}
-/// Returns whether a fetch on `plan` can also be applied to each child.
+/// Returns whether a fetch on `plan` can move to its input.
fn can_push_fetch_through(plan: &Arc<dyn ExecutionPlan>) -> bool {
- plan.supports_limit_pushdown()
- && matches!(plan.cardinality_effect(), CardinalityEffect::Equal)
+ match plan.limit_behavior() {
+ LimitBehavior::MoveToInput => true,
+ LimitBehavior::Merge => plan.fetch().is_none(),
+ _ => false,
+ }
}
/// Returns a plan when the fast path for an already-satisfied ordering can also
@@ -177,11 +174,7 @@ fn preserve_fetch_above_ordered_plan(
node.plan.output_partitioning().partition_count() > 1;
let input = Arc::clone(&node.plan);
- let limit: Arc<dyn ExecutionPlan> = if input_has_multiple_partitions {
- Arc::new(LocalLimitExec::new(input, fetch))
- } else {
- Arc::new(GlobalLimitExec::new(input, 0, Some(fetch)))
- };
+ let limit = limit_node(input, RowBound::fetch(Some(fetch)));
let limit_node = PlanContext::new(limit, ParentRequirements::default(), vec![node]);
if input_has_multiple_partitions
@@ -449,29 +442,10 @@ fn pushdown_requirement_to_children(
parent_required: OrderingRequirements,
parent_fetch: Option<usize>,
) -> Result<Option<SortPushdownProposal>> {
- // A parent fetch can be pushed through a plan only if the plan explicitly
- // supports limit pushdown and does not change cardinality. For example, if
- // LIMIT 2 is applied below a FilterExec that removes half the rows, only one
- // row may remain. Applied after the filter, it correctly returns two rows
- // when at least two are available.
- //
- // Checking `supports_limit_pushdown()` alone is not enough. For example:
- //
- // UnionExec advertises `supports_limit_pushdown() == true` because it can
- // forward a LIMIT k to each of its children—i.e. apply “LIMIT k” separately
- // on each branch before merging them together.
- //
- // However, UnionExec’s `cardinality_effect() == GreaterEqual` (it sums up
- // all child row counts), so pushing a global TopK/LIMIT through it would
- // break the semantics of “take the first k rows of the combined result.”
- //
- // For example, with two branches A and B and k = 3:
- // — Global LIMIT: take the first 3 rows from (A ∪ B) after merging.
- // — Pushed down: take 3 from A, 3 from B, then merge → up to 6 rows!
- //
- // That’s why we also require equal cardinality: even though UnionExec can
- // push a LIMIT to its children, its GreaterEqual effect means it cannot
- // preserve the global TopK semantics.
+ // Pushing the sort below `plan` moves the parent fetch with it. For
+ // example, if LIMIT 2 is applied below a FilterExec that removes half the
+ // rows, only one row may remain; and a LIMIT 3 pushed into both inputs of
+ // a UnionExec keeps up to 6 rows. See `LimitBehavior`.
if parent_fetch.is_some() && !can_push_fetch_through(plan) {
return Ok(None);
}
diff --git a/datafusion/physical-optimizer/src/lib.rs b/datafusion/physical-optimizer/src/lib.rs
index 0f5c38a036..52fadbc736 100644
--- a/datafusion/physical-optimizer/src/lib.rs
+++ b/datafusion/physical-optimizer/src/lib.rs
@@ -32,6 +32,7 @@ pub mod projection_pushdown;
pub use datafusion_pruning as pruning;
pub mod hash_join_buffering;
pub mod pushdown_sort;
+mod row_bound;
pub mod sanity_checker;
pub mod topk_aggregation;
pub mod topk_repartition;
diff --git a/datafusion/physical-optimizer/src/limit_pushdown.rs b/datafusion/physical-optimizer/src/limit_pushdown.rs
index b1249db8b6..740553280d 100644
--- a/datafusion/physical-optimizer/src/limit_pushdown.rs
+++ b/datafusion/physical-optimizer/src/limit_pushdown.rs
@@ -54,41 +54,23 @@ use crate::PhysicalOptimizerRule;
use crate::optimizer::{
ConfigOnlyContext, PhysicalOptimizerContext, with_statistics_context,
};
+use crate::row_bound::{RowBound, limit_node, place};
use datafusion_common::config::ConfigOptions;
use datafusion_common::error::Result;
use datafusion_common::stats::Precision;
use datafusion_common::tree_node::{Transformed, TreeNodeRecursion};
-use datafusion_common::utils::combine_limit;
-use datafusion_physical_plan::coalesce_partitions::CoalescePartitionsExec;
use datafusion_physical_plan::empty::EmptyExec;
use datafusion_physical_plan::execution_plan::replace_children_if_necessary;
-use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;
use datafusion_physical_plan::projection::ProjectionExec;
-use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;
use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext};
-use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties};
+use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties, LimitBehavior};
/// This rule inspects [`ExecutionPlan`]'s and pushes down the fetch limit from
/// the parent to the child if applicable.
#[derive(Default, Debug)]
pub struct LimitPushdown {}
-/// This is a "data class" we use within the [`LimitPushdown`] rule to push
-/// down limits in the plan. GlobalRequirements are hold as a rule-wide state
-/// and holds the fetch and skip information. The struct also has a field named
-/// satisfied which means if the "current" plan is valid in terms of limits or not.
-///
-/// For example: If the plan is satisfied with current fetch info, we decide to not add a LocalLimit
-///
-/// [`LimitPushdown`]: crate::limit_pushdown::LimitPushdown
-#[derive(Default, Clone, Debug)]
-pub struct GlobalRequirements {
- fetch: Option<usize>,
- skip: usize,
- satisfied: bool,
-}
-
impl LimitPushdown {
#[expect(missing_docs)]
pub fn new() -> Self {
@@ -110,13 +92,8 @@ impl PhysicalOptimizerRule for LimitPushdown {
plan: Arc<dyn ExecutionPlan>,
context: &dyn PhysicalOptimizerContext,
) -> Result<Arc<dyn ExecutionPlan>> {
- let global_state = GlobalRequirements {
- fetch: None,
- skip: 0,
- satisfied: false,
- };
with_statistics_context(context, |stats_ctx| {
- pushdown_limits(plan, global_state, stats_ctx)
+ push_limits(plan, Above::Nothing, stats_ctx)
})
}
@@ -129,206 +106,277 @@ impl PhysicalOptimizerRule for LimitPushdown {
}
}
-struct LimitInfo {
- input: Arc<dyn ExecutionPlan>,
- fetch: Option<usize>,
- skip: usize,
+/// The row bound a node receives from its ancestors.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum Above {
+ /// No bound.
+ Nothing,
+ /// `OFFSET skip LIMIT fetch` that no ancestor enforces yet: it must be
+ /// placed at or above the first node it can't pass.
+ Open(RowBound),
+ /// A bound an ancestor already enforces, with its `skip` applied. A node
+ /// below may stop after `fetch` rows; nothing else is required of it. A
+ /// limit node further down is still folded in and placed again.
+ Enforced(Option<usize>),
}
-/// This function is the main helper function of the `LimitPushDown` rule.
-/// The helper takes an `ExecutionPlan` and a global (algorithm) state which is
-/// an instance of `GlobalRequirements` and modifies these parameters while
-/// checking if the limits can be pushed down or not.
-///
-/// If a limit is encountered, a [`TreeNodeRecursion::Stop`] is returned. Otherwise,
-/// return a [`TreeNodeRecursion::Continue`].
-///
-/// Computes statistics with a new [`StatisticsContext`] that has no statistics
-/// providers. A context built from a statistics registry, as [`LimitPushdown`]
-/// uses, also consults the registered providers, so switching to
-/// [`pushdown_limit_helper_with_stats`] with such a context can change the
-/// result when providers are registered.
-#[deprecated(
- since = "56.0.0",
- note = "use `pushdown_limit_helper_with_stats` and share one `StatisticsContext` across calls"
-)]
-pub fn pushdown_limit_helper(
- pushdown_plan: Arc<dyn ExecutionPlan>,
- global_state: GlobalRequirements,
-) -> Result<(Transformed<Arc<dyn ExecutionPlan>>, GlobalRequirements)> {
- pushdown_limit_helper_with_stats(
- pushdown_plan,
- global_state,
- &StatisticsContext::new(),
- )
+impl Above {
+ fn open(bound: RowBound) -> Self {
+ if bound == RowBound::default() {
+ Above::Nothing
+ } else {
+ Above::Open(bound)
+ }
+ }
+
+ fn bound(self) -> RowBound {
+ match self {
+ Above::Nothing => RowBound::default(),
+ Above::Open(bound) => bound,
+ Above::Enforced(fetch) => RowBound::fetch(fetch),
+ }
+ }
+
+ /// `self` above a node that enforces its own `fetch` (a TopK, a scan with
+ /// a row limit, ...). The node enforces the combined bound unless rows
+ /// still have to be skipped above it.
+ fn above_fetching_node(self, fetch: usize) -> Self {
+ let bound = self.bound().over(RowBound::fetch(Some(fetch)));
+ if bound.skip > 0 {
+ Above::Open(bound)
+ } else {
+ Above::Enforced(bound.fetch)
+ }
+ }
}
-/// Like [`pushdown_limit_helper`], but computes statistics with `stats_ctx`,
-/// so its cache can be shared with other calls and other rules.
-pub fn pushdown_limit_helper_with_stats(
- mut pushdown_plan: Arc<dyn ExecutionPlan>,
- mut global_state: GlobalRequirements,
- stats_ctx: &StatisticsContext,
-) -> Result<(Transformed<Arc<dyn ExecutionPlan>>, GlobalRequirements)> {
- // Extract limit, if exist, and return child inputs.
- if let Some(limit_info) = extract_limit(&pushdown_plan) {
- // If we have fetch/skip info in the global state already, we need to
- // decide which one to continue with:
- let (skip, fetch) = combine_limit(
- global_state.skip,
- global_state.fetch,
- limit_info.skip,
- limit_info.fetch,
- );
- global_state.skip = skip;
- global_state.fetch = fetch;
- global_state.satisfied = false;
-
- if let Some(fetch) = fetch
- && limit_satisfied_by_input(&limit_info.input, skip, fetch, stats_ctx)?
+/// What a node does with a bound from its parent.
+enum Step {
+ /// A limit node: its bound is folded in and the node removed.
+ Limit(RowBound),
+ /// Passes the bound to every child, and drops it here. Below a node with
+ /// several output partitions the bound applies to each partition, so this
+ /// keeps the same rows only when each output partition is one input
+ /// partition (`UnionExec`, `ProjectionExec`).
+ Pass,
+ /// Combines partitions, so it sees every row and can enforce the whole
+ /// bound itself.
+ Merge,
+ /// The bound can't go below this node.
+ Stop,
+}
+
+fn step(plan: &Arc<dyn ExecutionPlan>) -> Step {
+ match plan.limit_behavior() {
+ LimitBehavior::Limit { skip, fetch } => Step::Limit(RowBound { skip, fetch }),
+ // Below a node whose children have more partitions than it outputs,
+ // the bound would apply to each of them and keep too many rows. The
+ // node takes the bound like a merge: it stays above, and each child
+ // gets a copy.
+ LimitBehavior::CopyToChildren | LimitBehavior::MoveToInput
+ if combines_partitions(plan) =>
{
- // The input already produces at most `fetch` rows, so no new limit
- // node is needed. Mark satisfied so downstream won't re-add one,
- // but preserve skip/fetch so any nested limit nodes (e.g. an inner
- // GlobalLimitExec) can still be merged with the outer constraint.
- global_state.satisfied = true;
-
- return Ok((
- Transformed {
- data: limit_info.input,
- transformed: true,
- tnr: TreeNodeRecursion::Stop,
- },
- global_state,
- ));
+ Step::Merge
}
+ LimitBehavior::CopyToChildren | LimitBehavior::MoveToInput => Step::Pass,
+ LimitBehavior::Merge => Step::Merge,
+ // LimitPushPastWindows handles windows.
+ LimitBehavior::Stops | LimitBehavior::Window(_) => Step::Stop,
+ }
+}
- // Now the global state has the most recent information, we can remove
- // the limit node. We will decide later if we should add it again or
- // not.
- return Ok((
- Transformed {
- data: limit_info.input,
- transformed: true,
- tnr: TreeNodeRecursion::Stop,
- },
- global_state,
- ));
+/// Whether the children of `plan` have more partitions in total than it
+/// outputs.
+fn combines_partitions(plan: &Arc<dyn ExecutionPlan>) -> bool {
+ let input_partitions: usize = plan
+ .children()
+ .iter()
+ .map(|child| child.output_partitioning().partition_count())
+ .sum();
+ input_partitions > plan.output_partitioning().partition_count()
+}
+
+/// Pushes the bounds of all limit nodes in `plan` down as far as they pass,
+/// and writes each one once, at the first node that can enforce it:
+///
+/// - a merge, which takes it as its own fetch;
+/// - otherwise the first node the bound can't pass, which takes it as its
+/// fetch or gets a limit node on top.
+///
+/// Below the place a bound is enforced, nodes that can stop early are given
+/// it as a fetch too. Operators can't skip rows, so a bound with an `OFFSET`
+/// always keeps a [`GlobalLimitExec`] where it is enforced.
+///
+/// [`GlobalLimitExec`]: datafusion_physical_plan::limit::GlobalLimitExec
+fn push_limits(
+ mut plan: Arc<dyn ExecutionPlan>,
+ mut above: Above,
+ stats_ctx: &StatisticsContext,
+) -> Result<Arc<dyn ExecutionPlan>> {
+ while let Some((input, folded)) = fold_limit(&plan, above, stats_ctx)? {
+ (plan, above) = (input, folded);
}
+ let (plan, below) = apply_bound(plan, above);
- // If we have a non-limit operator with fetch capability, update global
- // state as necessary:
- if pushdown_plan.fetch().is_some() {
- if global_state.skip == 0 {
- global_state.satisfied = true;
- }
- (global_state.skip, global_state.fetch) = combine_limit(
- global_state.skip,
- global_state.fetch,
- 0,
- pushdown_plan.fetch(),
- );
+ let children = plan.children();
+ let mut changed = false;
+ let new_children = children
+ .into_iter()
+ .map(|child| {
+ let new_child = push_limits(Arc::clone(child), below, stats_ctx)?;
+ changed |= !Arc::ptr_eq(child, &new_child);
+ Ok(new_child)
+ })
+ .collect::<Result<_>>()?;
+ if changed {
+ replace_children_if_necessary(plan, new_children)
+ } else {
+ Ok(plan)
}
+}
- let Some(global_fetch) = global_state.fetch else {
- // There's no valid fetch information, exit early:
- return if global_state.skip > 0 && !global_state.satisfied {
- // There might be a case with only offset, if so add a global limit:
- global_state.satisfied = true;
- Ok((
- Transformed::yes(add_global_limit(
- pushdown_plan,
- global_state.skip,
- None,
- )),
- global_state,
- ))
- } else {
- // There's no info on offset or fetch, nothing to do:
- Ok((Transformed::no(pushdown_plan), global_state))
- };
+/// If `plan` is a limit node, returns its input and `above` combined with the
+/// limit's bound. Otherwise returns `None`.
+fn fold_limit(
+ plan: &Arc<dyn ExecutionPlan>,
+ above: Above,
+ stats_ctx: &StatisticsContext,
+) -> Result<Option<(Arc<dyn ExecutionPlan>, Above)>> {
+ let Step::Limit(limit) = step(plan) else {
+ return Ok(None);
+ };
+ let input = Arc::clone(plan.children()[0]);
+ let bound = above.bound().over(limit);
+ // Exact statistics may prove the input already has few enough rows. The
+ // bound is kept so a nested limit can still be combined with it.
+ let above = if limit_satisfied_by_input(&input, bound, stats_ctx)? {
+ Above::Enforced(bound.fetch)
+ } else {
+ Above::open(bound)
};
+ Ok(Some((input, above)))
+}
- let skip_and_fetch = Some(global_fetch + global_state.skip);
-
- if pushdown_plan.supports_limit_pushdown() {
- if !combines_input_partitions(&pushdown_plan) {
- // We have information in the global state and the plan pushes down,
- // continue:
- Ok((Transformed::no(pushdown_plan), global_state))
- } else if let Some(plan_with_fetch) = pushdown_plan.with_fetch(skip_and_fetch) {
- // This plan is combining input partitions, so we need to add the
- // fetch info to plan if possible. If not, we must add a limit node
- // with the information from the global state.
- let mut new_plan = plan_with_fetch;
- // Execution plans can't (yet) handle skip, so if we have one,
- // we still need to add a global limit
- if global_state.skip > 0 {
- new_plan =
- add_global_limit(new_plan, global_state.skip, global_state.fetch);
- }
- global_state.fetch = skip_and_fetch;
- global_state.skip = 0;
- global_state.satisfied = true;
- Ok((Transformed::yes(new_plan), global_state))
- } else if global_state.satisfied {
- // If the plan is already satisfied, do not add a limit:
- Ok((Transformed::no(pushdown_plan), global_state))
- } else {
- global_state.satisfied = true;
- Ok((
- Transformed::yes(add_limit(
- pushdown_plan,
- global_state.skip,
- global_fetch,
- )),
- global_state,
- ))
+/// Applies `above` to `plan`, which is not a limit node. Returns the new plan
+/// and the bound its children receive.
+fn apply_bound(
+ plan: Arc<dyn ExecutionPlan>,
+ above: Above,
+) -> (Arc<dyn ExecutionPlan>, Above) {
+ let above = match plan.fetch() {
+ Some(fetch) => above.above_fetching_node(fetch),
+ None => above,
+ };
+ match (step(&plan), above) {
+ (_, Above::Nothing | Above::Enforced(None)) => (plan, Above::Nothing),
+ // Only an OFFSET: it is placed here, operators can't skip.
+ (_, Above::Open(bound)) if bound.fetch.is_none() => {
+ (limit_node(plan, bound), Above::Nothing)
}
- } else {
- // The plan does not support push down and it is not a limit. We will need
- // to add a limit or a fetch. If the plan is already satisfied, we will try
- // to add the fetch info and return the plan.
-
- // There's no push down, change fetch & skip to default values:
- let global_skip = global_state.skip;
- global_state.fetch = None;
- global_state.skip = 0;
-
- let maybe_fetchable = pushdown_plan.with_fetch(skip_and_fetch);
- if global_state.satisfied {
- if let Some(plan_with_fetch) = maybe_fetchable {
- Ok((Transformed::yes(plan_with_fetch), global_state))
- } else {
- Ok((Transformed::no(pushdown_plan), global_state))
- }
+ (Step::Limit(_), _) => unreachable!("limit nodes are folded before apply_bound"),
+ (Step::Pass, above) => (plan, above),
+ (Step::Merge, Above::Open(bound)) => {
+ (place(plan, bound), Above::Enforced(bound.input_rows()))
+ }
+ (Step::Merge, Above::Enforced(fetch)) => (
+ plan.with_fetch(fetch).unwrap_or(plan),
+ Above::Enforced(fetch),
+ ),
+ (Step::Stop, Above::Open(bound)) => (place(plan, bound), Above::Nothing),
+ (Step::Stop, Above::Enforced(fetch)) => {
+ (plan.with_fetch(fetch).unwrap_or(plan), Above::Nothing)
+ }
+ }
+}
+
+/// The state [`pushdown_limit_helper`] carries from one node to the next.
+#[deprecated(
+ since = "56.0.0",
+ note = "LimitPushdown no longer works one node at a time; run the `LimitPushdown` rule"
+)]
+#[derive(Default, Clone, Debug)]
+pub struct GlobalRequirements {
+ fetch: Option<usize>,
+ skip: usize,
+ satisfied: bool,
+}
+
+#[expect(deprecated)]
+impl From<GlobalRequirements> for Above {
+ fn from(state: GlobalRequirements) -> Self {
+ let bound = RowBound {
+ skip: state.skip,
+ fetch: state.fetch,
+ };
+ if state.satisfied && bound.skip == 0 {
+ Above::Enforced(bound.fetch)
} else {
- global_state.satisfied = true;
- pushdown_plan = if let Some(plan_with_fetch) = maybe_fetchable {
- if global_skip > 0 {
- add_global_limit(plan_with_fetch, global_skip, Some(global_fetch))
- } else {
- plan_with_fetch
- }
- } else {
- add_limit(pushdown_plan, global_skip, global_fetch)
- };
- Ok((Transformed::yes(pushdown_plan), global_state))
+ Above::open(bound)
}
}
}
+#[expect(deprecated)]
+impl From<Above> for GlobalRequirements {
+ fn from(above: Above) -> Self {
+ match above {
+ Above::Nothing => GlobalRequirements::default(),
+ Above::Open(bound) => GlobalRequirements {
+ fetch: bound.fetch,
+ skip: bound.skip,
+ satisfied: false,
+ },
+ Above::Enforced(fetch) => GlobalRequirements {
+ fetch,
+ skip: 0,
+ satisfied: true,
+ },
+ }
+ }
+}
+
+/// Applies the [`LimitPushdown`] rule to one node. A limit node is removed and
+/// [`TreeNodeRecursion::Stop`] is returned; call again on its input.
+///
+/// Computes statistics with a new [`StatisticsContext`] that has no statistics
+/// providers.
+#[deprecated(
+ since = "56.0.0",
+ note = "LimitPushdown no longer works one node at a time; run the `LimitPushdown` rule"
+)]
+#[expect(deprecated)]
+pub fn pushdown_limit_helper(
+ pushdown_plan: Arc<dyn ExecutionPlan>,
+ global_state: GlobalRequirements,
+) -> Result<(Transformed<Arc<dyn ExecutionPlan>>, GlobalRequirements)> {
+ let above = Above::from(global_state);
+ if let Some((input, above)) =
+ fold_limit(&pushdown_plan, above, &StatisticsContext::new())?
+ {
+ return Ok((
+ Transformed::new(input, true, TreeNodeRecursion::Stop),
+ above.into(),
+ ));
+ }
+ let original = Arc::clone(&pushdown_plan);
+ let (plan, below) = apply_bound(pushdown_plan, above);
+ let transformed = !Arc::ptr_eq(&original, &plan);
+ Ok((
+ Transformed::new_transformed(plan, transformed),
+ below.into(),
+ ))
+}
+
/// Returns true if exact input statistics prove that applying the limit would
/// not remove any rows.
fn limit_satisfied_by_input(
plan: &Arc<dyn ExecutionPlan>,
- skip: usize,
- fetch: usize,
+ bound: RowBound,
stats_ctx: &StatisticsContext,
) -> Result<bool> {
- if skip > 0 {
+ let (0, Some(fetch)) = (bound.skip, bound.fetch) else {
return Ok(false);
- }
+ };
if plan.output_partitioning().partition_count() != 1 {
return Ok(false);
@@ -375,100 +423,4 @@ fn limit_eliminable_exact_num_rows(
Ok(None)
}
-/// Pushes down the limit through the plan.
-pub(crate) fn pushdown_limits(
- pushdown_plan: Arc<dyn ExecutionPlan>,
- global_state: GlobalRequirements,
- stats_ctx: &StatisticsContext,
-) -> Result<Arc<dyn ExecutionPlan>> {
- // Call pushdown_limit_helper.
- // This will either extract the limit node (returning the child), or apply the limit pushdown.
- let (mut new_node, mut global_state) =
- pushdown_limit_helper_with_stats(pushdown_plan, global_state, stats_ctx)?;
-
- // While limits exist, continue combining the global_state.
- while new_node.tnr == TreeNodeRecursion::Stop {
- (new_node, global_state) =
- pushdown_limit_helper_with_stats(new_node.data, global_state, stats_ctx)?;
- }
-
- // Once a limit has been materialized above the current node, child
- // subtrees should not inherit its `skip`. Keep `fetch`, but clear
- // `skip` before recursing so child-local limits are not merged with
- // an `OFFSET` that has already been applied.
- if global_state.satisfied {
- global_state.skip = 0;
- }
-
- // Apply pushdown limits in children
- let children = new_node.data.children();
- let mut changed = false;
- let new_children = children
- .into_iter()
- .map(|child: &Arc<dyn ExecutionPlan>| {
- let new_child = pushdown_limits(
- Arc::<dyn ExecutionPlan>::clone(child),
- global_state.clone(),
- stats_ctx,
- )?;
- // Tracking if any of the children changed
- changed |= !Arc::ptr_eq(child, &new_child);
- Ok(new_child)
- })
- .collect::<Result<_>>()?;
-
- if changed {
- replace_children_if_necessary(new_node.data, new_children)
- } else {
- Ok(new_node.data)
- }
-}
-
-/// Extracts limit information from the [`ExecutionPlan`] if it is a
-/// [`GlobalLimitExec`] or a [`LocalLimitExec`].
-fn extract_limit(plan: &Arc<dyn ExecutionPlan>) -> Option<LimitInfo> {
- if let Some(global_limit) = plan.downcast_ref::<GlobalLimitExec>() {
- Some(LimitInfo {
- input: Arc::clone(global_limit.input()),
- fetch: global_limit.fetch(),
- skip: global_limit.skip(),
- })
- } else {
- plan.downcast_ref::<LocalLimitExec>()
- .map(|local_limit| LimitInfo {
- input: Arc::clone(local_limit.input()),
- fetch: Some(local_limit.fetch()),
- skip: 0,
- })
- }
-}
-
-/// Checks if the given plan combines input partitions.
-fn combines_input_partitions(plan: &Arc<dyn ExecutionPlan>) -> bool {
- plan.is::<CoalescePartitionsExec>() || plan.is::<SortPreservingMergeExec>()
-}
-
-/// Adds a limit to the plan, chooses between global and local limits based on
-/// skip value and the number of partitions.
-fn add_limit(
- pushdown_plan: Arc<dyn ExecutionPlan>,
- skip: usize,
- fetch: usize,
-) -> Arc<dyn ExecutionPlan> {
- if skip > 0 || pushdown_plan.output_partitioning().partition_count() == 1 {
- add_global_limit(pushdown_plan, skip, Some(fetch))
- } else {
- Arc::new(LocalLimitExec::new(pushdown_plan, fetch + skip)) as _
- }
-}
-
-/// Adds a global limit to the plan.
-fn add_global_limit(
- pushdown_plan: Arc<dyn ExecutionPlan>,
- skip: usize,
- fetch: Option<usize>,
-) -> Arc<dyn ExecutionPlan> {
- Arc::new(GlobalLimitExec::new(pushdown_plan, skip, fetch)) as _
-}
-
// See tests in datafusion/core/tests/physical_optimizer
diff --git a/datafusion/physical-optimizer/src/limit_pushdown_past_window.rs b/datafusion/physical-optimizer/src/limit_pushdown_past_window.rs
index 3aa31872c8..763f5d79c3 100644
--- a/datafusion/physical-optimizer/src/limit_pushdown_past_window.rs
+++ b/datafusion/physical-optimizer/src/limit_pushdown_past_window.rs
@@ -3,28 +3,17 @@
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
use crate::PhysicalOptimizerRule;
-use datafusion_common::ScalarValue;
+use crate::row_bound::RowBound;
use datafusion_common::config::ConfigOptions;
use datafusion_common::tree_node::{Transformed, TreeNode};
-use datafusion_expr::{LimitEffect, WindowFrameBound, WindowFrameUnits};
-use datafusion_physical_expr::window::{
- PlainAggregateWindowExpr, SlidingAggregateWindowExpr, StandardWindowExpr,
- StandardWindowFunctionExpr, WindowExpr,
-};
-use datafusion_physical_plan::execution_plan::CardinalityEffect;
-use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
-use datafusion_physical_plan::repartition::RepartitionExec;
use datafusion_physical_plan::sorts::sort::SortExec;
use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;
-use datafusion_physical_plan::windows::{BoundedWindowAggExec, WindowUDFExpr};
-use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties};
-use std::cmp;
+use datafusion_physical_plan::{ExecutionPlan, LimitBehavior, WindowLookahead};
use std::sync::Arc;
/// This rule inspects [`ExecutionPlan`]'s attempting to find fetch limits that were not pushed
-/// down by `LimitPushdown` because [BoundedWindowAggExec]s were "in the way". If the window is
-/// bounded by [WindowFrameUnits::Rows] then we calculate the adjustment needed to grow the limit
-/// and continue pushdown.
+/// down by `LimitPushdown` because windows ([`LimitBehavior::Window`]) were "in the way". If a
+/// window's lookahead is known, the limit grows by it and pushdown continues.
#[derive(Default, Clone, Debug)]
pub struct LimitPushPastWindows;
@@ -91,9 +80,9 @@ impl PhysicalOptimizerRule for LimitPushPastWindows {
}
// grow the limit if we hit a window function
- if let Some(window) = node.downcast_ref::<BoundedWindowAggExec>() {
+ if let LimitBehavior::Window(lookahead) = node.limit_behavior() {
phase = Phase::Apply;
- if !grow_limit(window, &mut ctx) {
+ if !grow_limit(lookahead, &mut ctx) {
return reset(node, &mut ctx);
}
return Ok(Transformed::no(node));
@@ -107,21 +96,13 @@ impl PhysicalOptimizerRule for LimitPushPastWindows {
}
// nodes along the way
- if !node.supports_limit_pushdown() {
- return reset(node, &mut ctx);
- }
- if let Some(part) = node.downcast_ref::<RepartitionExec>() {
- let output = part.partitioning().partition_count();
- let input = part.input().output_partitioning().partition_count();
- if output < input {
- return reset(node, &mut ctx);
- }
- }
- match node.cardinality_effect() {
- CardinalityEffect::Unknown => return reset(node, &mut ctx),
- CardinalityEffect::LowerEqual => return reset(node, &mut ctx),
- CardinalityEffect::Equal => {}
- CardinalityEffect::GreaterEqual => {}
+ match node.limit_behavior() {
+ LimitBehavior::CopyToChildren | LimitBehavior::MoveToInput => {}
+ LimitBehavior::Merge if node.fetch().is_none() => {}
+ LimitBehavior::Stops
+ | LimitBehavior::Limit { .. }
+ | LimitBehavior::Merge
+ | LimitBehavior::Window(_) => return reset(node, &mut ctx),
}
Ok(Transformed::no(node))
@@ -138,36 +119,18 @@ impl PhysicalOptimizerRule for LimitPushPastWindows {
}
}
-fn grow_limit(window: &BoundedWindowAggExec, ctx: &mut TraverseState) -> bool {
- let mut max_rel = 0;
- for expr in window.window_expr().iter() {
- // grow based on function requirements
- match get_limit_effect(expr) {
- LimitEffect::None => {}
- LimitEffect::Unknown => return false,
- LimitEffect::Relative(rel) => max_rel = max_rel.max(rel),
- LimitEffect::Absolute(val) => {
- let cur = ctx.limit.unwrap_or(0);
- ctx.limit = Some(cur.max(val))
- }
- }
-
- // grow based on frames
- let frame = expr.get_window_frame();
- if frame.units != WindowFrameUnits::Rows {
- return false; // expression-based limits not statically evaluatable
- }
- let Some(end_bound) = bound_to_usize(&frame.end_bound) else {
- return false; // can't optimize unbounded window expressions
- };
- ctx.max_lookahead(end_bound);
+fn grow_limit(lookahead: Option<WindowLookahead>, ctx: &mut TraverseState) -> bool {
+ let Some(lookahead) = lookahead else {
+ return false;
+ };
+ if let Some(min_rows) = lookahead.min_rows {
+ ctx.limit = Some(ctx.limit.unwrap_or(0).max(min_rows));
}
-
- // finish grow
+ ctx.max_lookahead(lookahead.frame_end);
// Frame bounds come straight from the query and can be as large as
// `u64::MAX`, so saturate instead of overflowing. A saturated lookahead
// makes the `checked_add` in `apply_limit` fail, which skips the pushdown.
- ctx.max_lookahead(ctx.lookahead.saturating_add(max_rel));
+ ctx.max_lookahead(ctx.lookahead.saturating_add(lookahead.relative));
true
}
@@ -192,24 +155,16 @@ fn apply_limit(
};
let fetch = match node.fetch() {
None => fetch,
- Some(existing) => cmp::min(existing, fetch),
+ Some(existing) => existing.min(fetch),
};
Some(Transformed::complete(node.with_fetch(Some(fetch)).unwrap()))
}
fn get_limit(node: &Arc<dyn ExecutionPlan>, ctx: &mut TraverseState) -> bool {
- if let Some(limit) = node.downcast_ref::<GlobalLimitExec>() {
- ctx.reset_limit(
- limit
- .fetch()
- .map(|fetch| fetch.saturating_add(limit.skip())),
- );
- return true;
- }
// In distributed execution, GlobalLimitExec becomes LocalLimitExec
// per partition. Handle it the same way (LocalLimitExec has no skip).
- if let Some(limit) = node.downcast_ref::<LocalLimitExec>() {
- ctx.reset_limit(Some(limit.fetch()));
+ if let Some(limit) = RowBound::of_limit_node(node) {
+ ctx.reset_limit(limit.input_rows());
return true;
}
if let Some(limit) = node.downcast_ref::<SortPreservingMergeExec>() {
@@ -219,58 +174,20 @@ fn get_limit(node: &Arc<dyn ExecutionPlan>, ctx: &mut TraverseState) -> bool {
false
}
-/// Examines the `WindowExpr` and decides:
-/// 1. The expression does not change the window size
-/// 2. The expression grows it by X amount
-/// 3. We don't know
-///
-/// # Arguments
-///
-/// * `expr` the expression to examine
-///
-/// # Returns
-///
-/// The effect on the limit
-fn get_limit_effect(expr: &Arc<dyn WindowExpr>) -> LimitEffect {
- // White list aggregates
- if expr.as_any().is::<PlainAggregateWindowExpr>()
- || expr.as_any().is::<SlidingAggregateWindowExpr>()
- {
- return LimitEffect::None;
- }
-
- // Grab the window function
- let Some(swe) = expr.as_any().downcast_ref::<StandardWindowExpr>() else {
- return LimitEffect::Unknown; // should be only remaining type
- };
- let swfe = swe.get_standard_func_expr();
- let Some(udf) = swfe.as_any().downcast_ref::<WindowUDFExpr>() else {
- return LimitEffect::Unknown; // should be only remaining type
- };
- udf.limit_effect()
-}
-
-fn bound_to_usize(bound: &WindowFrameBound) -> Option<usize> {
- match bound {
- WindowFrameBound::Preceding(_) => Some(0),
- WindowFrameBound::CurrentRow => Some(0),
- WindowFrameBound::Following(ScalarValue::UInt64(Some(scalar))) => {
- Some(*scalar as usize)
- }
- _ => None,
- }
-}
-
#[cfg(test)]
mod tests {
use super::*;
use arrow::datatypes::{DataType, Field, Schema};
+ use datafusion_common::ScalarValue;
use datafusion_expr::WindowFrame;
+ use datafusion_expr::{WindowFrameBound, WindowFrameUnits};
use datafusion_functions_window::row_number::row_number_udwf;
use datafusion_physical_expr::expressions::col;
+ use datafusion_physical_expr::window::StandardWindowExpr;
use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};
use datafusion_physical_plan::InputOrderMode;
use datafusion_physical_plan::displayable;
+ use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;
use datafusion_physical_plan::windows::{
BoundedWindowAggExec, create_udwf_window_expr,
diff --git a/datafusion/physical-optimizer/src/row_bound.rs b/datafusion/physical-optimizer/src/row_bound.rs
new file mode 100644
index 0000000000..669b80b3de
--- /dev/null
+++ b/datafusion/physical-optimizer/src/row_bound.rs
@@ -0,0 +1,161 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
+
+//! [`RowBound`]: the `OFFSET skip LIMIT fetch` that an optimizer rule wants on
+//! an operator's output.
+//!
+//! A bound is written into a plan in one of two forms: a limit node
+//! ([`GlobalLimitExec`], [`LocalLimitExec`]) or an operator's own fetch
+//! ([`ExecutionPlan::with_fetch`]). Operators can't skip rows, so a bound with
+//! `skip > 0` always keeps a [`GlobalLimitExec`].
+
+use std::sync::Arc;
+
+use datafusion_common::utils::combine_limit;
+use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
+use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties, LimitBehavior};
+
+/// `OFFSET skip LIMIT fetch` on the output of one operator.
+///
+/// `fetch == None` means no row limit (only `skip`). The default bound,
+/// `skip = 0, fetch = None`, keeps every row.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub(crate) struct RowBound {
+ pub skip: usize,
+ pub fetch: Option<usize>,
+}
+
+impl RowBound {
+ /// `LIMIT fetch` with no offset.
+ pub fn fetch(fetch: Option<usize>) -> Self {
+ Self { skip: 0, fetch }
+ }
+
+ /// The bound a limit node imposes, or `None` if `plan` is not a limit node.
+ pub fn of_limit_node(plan: &Arc<dyn ExecutionPlan>) -> Option<Self> {
+ match plan.limit_behavior() {
+ LimitBehavior::Limit { skip, fetch } => Some(Self { skip, fetch }),
+ _ => None,
+ }
+ }
+
+ /// `self` applied over the output of `inner`: first `inner`, then `self`.
+ pub fn over(self, inner: RowBound) -> RowBound {
+ let (skip, fetch) = combine_limit(self.skip, self.fetch, inner.skip, inner.fetch);
+ RowBound { skip, fetch }
+ }
+
+ /// The tighter of two bounds on the same output. Both must have `skip == 0`.
+ pub fn min(self, other: RowBound) -> RowBound {
+ debug_assert!(self.skip == 0 && other.skip == 0);
+ let fetch = match (self.fetch, other.fetch) {
+ (Some(a), Some(b)) => Some(a.min(b)),
+ (a, b) => a.or(b),
+ };
+ RowBound::fetch(fetch)
+ }
+
+ /// Rows the input must produce so this bound can be applied on top of it:
+ /// `skip + fetch`. `None` when there is no row limit.
+ ///
+ /// Saturates: `usize::MAX` input rows is never a smaller bound than the
+ /// real one, so a fetch computed from it stays correct.
+ pub fn input_rows(self) -> Option<usize> {
+ self.fetch.map(|fetch| fetch.saturating_add(self.skip))
+ }
+}
+
+/// Writes `bound` on top of `plan`'s output.
+///
+/// `plan` absorbs `bound.input_rows()` as its own fetch when it can; a
+/// [`GlobalLimitExec`] on top still applies the skip, if any. When `plan`
+/// can't absorb it, a [`limit_node`] is added instead.
+pub(crate) fn place(
+ plan: Arc<dyn ExecutionPlan>,
+ bound: RowBound,
+) -> Arc<dyn ExecutionPlan> {
+ match plan.with_fetch(bound.input_rows()) {
+ Some(absorbed) if bound.skip == 0 => absorbed,
+ Some(absorbed) => {
+ Arc::new(GlobalLimitExec::new(absorbed, bound.skip, bound.fetch))
+ }
+ None => limit_node(plan, bound),
+ }
+}
+
+/// A limit node applying `bound` to `plan`: a [`GlobalLimitExec`] when it
+/// skips rows or `plan` has one output partition, otherwise a per-partition
+/// [`LocalLimitExec`].
+pub(crate) fn limit_node(
+ plan: Arc<dyn ExecutionPlan>,
+ bound: RowBound,
+) -> Arc<dyn ExecutionPlan> {
+ match bound.input_rows() {
+ Some(fetch)
+ if bound.skip == 0 && plan.output_partitioning().partition_count() > 1 =>
+ {
+ Arc::new(LocalLimitExec::new(plan, fetch))
+ }
+ _ => Arc::new(GlobalLimitExec::new(plan, bound.skip, bound.fetch)),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use arrow::datatypes::{DataType, Field, Schema};
+ use datafusion_physical_plan::empty::EmptyExec;
+
+ fn bound(skip: usize, fetch: Option<usize>) -> RowBound {
+ RowBound { skip, fetch }
+ }
+
+ #[test]
+ fn over_applies_inner_first() {
+ // LIMIT 5 over (OFFSET 2 LIMIT 10): rows 2..7 of the input.
+ assert_eq!(
+ bound(0, Some(5)).over(bound(2, Some(10))),
+ bound(2, Some(5))
+ );
+ // OFFSET 3 over LIMIT 10: 7 rows remain after the skip.
+ assert_eq!(bound(3, None).over(bound(0, Some(10))), bound(3, Some(7)));
+ }
+
+ #[test]
+ fn min_keeps_the_tighter_fetch() {
+ assert_eq!(
+ RowBound::fetch(Some(3)).min(RowBound::fetch(None)),
+ bound(0, Some(3))
+ );
+ assert_eq!(
+ RowBound::fetch(Some(3)).min(RowBound::fetch(Some(2))),
+ bound(0, Some(2))
+ );
+ }
+
+ #[test]
+ fn input_rows_saturates() {
+ assert_eq!(bound(5, Some(10)).input_rows(), Some(15));
+ assert_eq!(bound(5, None).input_rows(), None);
+ assert_eq!(bound(usize::MAX, Some(1)).input_rows(), Some(usize::MAX));
+ }
+
+ #[test]
+ fn limit_node_is_local_only_without_skip_over_many_partitions() {
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]));
+ let one: Arc<dyn ExecutionPlan> = Arc::new(EmptyExec::new(Arc::clone(&schema)));
+ let many: Arc<dyn ExecutionPlan> =
+ Arc::new(EmptyExec::new(schema).with_partitions(4));
+
+ let node = limit_node(Arc::clone(&many), bound(0, Some(3)));
+ assert_eq!(node.downcast_ref::<LocalLimitExec>().unwrap().fetch(), 3);
+
+ let node = limit_node(Arc::clone(&many), bound(2, Some(3)));
+ let global = node.downcast_ref::<GlobalLimitExec>().unwrap();
+ assert_eq!((global.skip(), global.fetch()), (2, Some(3)));
+
+ let node = limit_node(one, bound(0, Some(3)));
+ assert!(node.is::<GlobalLimitExec>());
+ }
+}
diff --git a/datafusion/physical-plan/src/buffer.rs b/datafusion/physical-plan/src/buffer.rs
index 728bd1226e..4532ebb4ad 100644
--- a/datafusion/physical-plan/src/buffer.rs
+++ b/datafusion/physical-plan/src/buffer.rs
@@ -5,6 +5,7 @@
//! [`BufferExec`] decouples production and consumption on messages by buffering the input in the
//! background up to a certain capacity.
+use crate::execution_plan::LimitBehavior;
use crate::execution_plan::{
CardinalityEffect, EvaluationType, SchedulingType, replace_children_if_necessary,
};
@@ -264,9 +265,22 @@ impl ExecutionPlan for BufferExec {
}
fn supports_limit_pushdown(&self) -> bool {
+ #[expect(deprecated)]
self.input.supports_limit_pushdown()
}
+ fn limit_behavior(&self) -> LimitBehavior {
+ // Buffering keeps rows 1:1, so a limit moves below it whenever it can
+ // also go below the input.
+ match self.input.limit_behavior() {
+ LimitBehavior::Stops | LimitBehavior::Window(_) => LimitBehavior::Stops,
+ LimitBehavior::Limit { .. }
+ | LimitBehavior::CopyToChildren
+ | LimitBehavior::MoveToInput
+ | LimitBehavior::Merge => LimitBehavior::MoveToInput,
+ }
+ }
+
fn cardinality_effect(&self) -> CardinalityEffect {
CardinalityEffect::Equal
}
diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rs
index 698327b870..3cd4e8c94f 100644
--- a/datafusion/physical-plan/src/coalesce_partitions.rs
+++ b/datafusion/physical-plan/src/coalesce_partitions.rs
@@ -15,6 +15,7 @@ use super::{
DisplayAs, ExecutionPlanProperties, PlanProperties, SendableRecordBatchStream,
Statistics,
};
+use crate::execution_plan::LimitBehavior;
use crate::execution_plan::{CardinalityEffect, EvaluationType, SchedulingType};
use crate::filter_pushdown::{FilterDescription, FilterPushdownPhase};
use crate::projection::{ProjectionExec, make_with_child};
@@ -288,6 +289,10 @@ impl ExecutionPlan for CoalescePartitionsExec {
true
}
+ fn limit_behavior(&self) -> LimitBehavior {
+ LimitBehavior::Merge
+ }
+
fn cardinality_effect(&self) -> CardinalityEffect {
if self.fetch.is_none() {
CardinalityEffect::Equal
diff --git a/datafusion/physical-plan/src/coop.rs b/datafusion/physical-plan/src/coop.rs
index baca859876..6443cd9fc9 100644
--- a/datafusion/physical-plan/src/coop.rs
+++ b/datafusion/physical-plan/src/coop.rs
@@ -67,6 +67,7 @@ use std::sync::Arc;
use std::task::{Context, Poll};
use crate::execution_plan::CardinalityEffect::{self, Equal};
+use crate::execution_plan::LimitBehavior;
use crate::filter_pushdown::{
ChildPushdownResult, FilterDescription, FilterPushdownPhase,
FilterPushdownPropagation,
@@ -325,6 +326,10 @@ impl ExecutionPlan for CooperativeExec {
true
}
+ fn limit_behavior(&self) -> LimitBehavior {
+ LimitBehavior::MoveToInput
+ }
+
fn cardinality_effect(&self) -> CardinalityEffect {
Equal
}
diff --git a/datafusion/physical-plan/src/execution_plan.rs b/datafusion/physical-plan/src/execution_plan.rs
index e7c1c9d761..b236cc2f7c 100644
--- a/datafusion/physical-plan/src/execution_plan.rs
+++ b/datafusion/physical-plan/src/execution_plan.rs
@@ -764,13 +764,37 @@ pub trait ExecutionPlan: Any + Debug + DisplayAs + Send + Sync {
/// Returns `true` if a limit can be safely pushed down through this
/// `ExecutionPlan` node.
///
- /// If this method returns `true`, and the query plan contains a limit at
- /// the output of this node, DataFusion will push the limit to the input
- /// of this node.
+ /// DataFusion's own operators still return the same values as before, so
+ /// a wrapper that forwards this to its input keeps working.
+ #[deprecated(
+ since = "56.0.0",
+ note = "implement `limit_behavior` instead; the default `limit_behavior` reads this"
+ )]
fn supports_limit_pushdown(&self) -> bool {
false
}
+ /// What a row limit (`LIMIT` / `OFFSET`) on this node's output means for
+ /// its input. The optimizer rules that move limits read only this; see
+ /// [`LimitBehavior`].
+ ///
+ /// The default keeps the behavior of nodes that still implement the
+ /// deprecated [`Self::supports_limit_pushdown`]: [`LimitBehavior::Stops`]
+ /// if it returns `false`, otherwise [`LimitBehavior::MoveToInput`] when
+ /// [`Self::cardinality_effect`] is [`CardinalityEffect::Equal`] and
+ /// [`LimitBehavior::CopyToChildren`] when it is not.
+ fn limit_behavior(&self) -> LimitBehavior {
+ #[expect(deprecated)]
+ let supports_limit_pushdown = self.supports_limit_pushdown();
+ if !supports_limit_pushdown {
+ return LimitBehavior::Stops;
+ }
+ match self.cardinality_effect() {
+ CardinalityEffect::Equal => LimitBehavior::MoveToInput,
+ _ => LimitBehavior::CopyToChildren,
+ }
+ }
+
/// Returns a fetching variant of this `ExecutionPlan` node, if it supports
/// fetch limits. Returns `None` otherwise.
///
@@ -2064,6 +2088,53 @@ pub fn get_plan_string(plan: &Arc<dyn ExecutionPlan>) -> Vec<String> {
actual.iter().map(|elem| (*elem).to_string()).collect()
}
+/// What a row limit on an operator's output means for the operator's input.
+/// Returned by [`ExecutionPlan::limit_behavior`].
+///
+/// A limit can be *copied* below a node as an early-stop hint while it also
+/// stays above the node, or *moved* below a node, leaving nothing above it.
+/// Copying is safe whenever the first `n` output rows need at most the first
+/// `n` rows of each child. Moving also needs the node to output exactly the
+/// rows of its input.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum LimitBehavior {
+ /// A limit above this node must stay above it. The default.
+ Stops,
+ /// This node applies `OFFSET skip LIMIT fetch` to its single input
+ /// (`GlobalLimitExec`, `LocalLimitExec`).
+ Limit { skip: usize, fetch: Option<usize> },
+ /// A limit may be copied to each child, but may not move: this node can
+ /// output more rows than any one child (`UnionExec`).
+ CopyToChildren,
+ /// This node outputs exactly the rows of its input: a limit may move to
+ /// its input (`ProjectionExec`).
+ MoveToInput,
+ /// This node merges its input partitions, so it sees every row and can
+ /// enforce a limit with its own fetch (`CoalescePartitionsExec`,
+ /// `SortPreservingMergeExec`). Without a fetch of its own
+ /// ([`ExecutionPlan::fetch`]) it outputs exactly the rows of its input,
+ /// like [`Self::MoveToInput`]. With one, it already selects rows, so a
+ /// limit above it may not move below it.
+ Merge,
+ /// A window function node, with how far it reads ahead of each output
+ /// row; `None` if that is not known before execution. Only
+ /// `LimitPushPastWindows` copies a limit below a window.
+ Window(Option<WindowLookahead>),
+}
+
+/// How many input rows a window needs for its first `n` output rows. See
+/// [`LimitBehavior::Window`].
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub struct WindowLookahead {
+ /// At least this many input rows, whatever `n` is.
+ pub min_rows: Option<usize>,
+ /// Output row `i` reads input rows up to `i + frame_end` (`ROWS` frames).
+ pub frame_end: usize,
+ /// The window functions read this many rows past the frame
+ /// (e.g. `LEAD(x, 2)`).
+ pub relative: usize,
+}
+
/// Indicates the effect an execution plan operator will have on the cardinality
/// of its input stream
pub enum CardinalityEffect {
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs
index ddacf5a1b3..8273a99438 100644
--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs
@@ -2052,13 +2052,6 @@ impl ExecutionPlan for HashJoinExec {
Ok(result)
}
- fn supports_limit_pushdown(&self) -> bool {
- // Hash join execution plan does not support pushing limit down through to children
- // because the children don't know about the join condition and can't
- // determine how many rows to produce
- false
- }
-
fn fetch(&self) -> Option<usize> {
self.fetch
}
diff --git a/datafusion/physical-plan/src/lib.rs b/datafusion/physical-plan/src/lib.rs
index eefecb6a36..7e74ad8f73 100644
--- a/datafusion/physical-plan/src/lib.rs
+++ b/datafusion/physical-plan/src/lib.rs
@@ -34,10 +34,11 @@ pub use crate::distribution_requirements::{
#[expect(deprecated)]
pub use crate::execution_plan::{
AsPhysicalExprRef, ChildrenPropertiesMode, ExecutionPlan, ExecutionPlanProperties,
- PlanProperties, ReplaceChildrenOptions, apply_expression_roots, collect,
- collect_partitioned, displayable, execute_input_stream, execute_stream,
- execute_stream_partitioned, get_plan_string, plan_contains_expression_id,
- replace_children_if_necessary, with_new_children_if_necessary,
+ LimitBehavior, PlanProperties, ReplaceChildrenOptions, WindowLookahead,
+ apply_expression_roots, collect, collect_partitioned, displayable,
+ execute_input_stream, execute_stream, execute_stream_partitioned, get_plan_string,
+ plan_contains_expression_id, replace_children_if_necessary,
+ with_new_children_if_necessary,
};
pub use crate::metrics::Metric;
pub use crate::ordering::InputOrderMode;
diff --git a/datafusion/physical-plan/src/limit.rs b/datafusion/physical-plan/src/limit.rs
index 7f06c314d3..cc346c02a6 100644
--- a/datafusion/physical-plan/src/limit.rs
+++ b/datafusion/physical-plan/src/limit.rs
@@ -13,6 +13,7 @@ use super::{
DisplayAs, ExecutionPlanProperties, PlanProperties, RecordBatchStream,
SendableRecordBatchStream, Statistics,
};
+use crate::execution_plan::LimitBehavior;
use crate::execution_plan::{Boundedness, CardinalityEffect};
use crate::statistics::{ChildStats, StatisticsArgs, with_per_partition_fetch};
use crate::{
@@ -274,6 +275,13 @@ impl ExecutionPlan for GlobalLimitExec {
true
}
+ fn limit_behavior(&self) -> LimitBehavior {
+ LimitBehavior::Limit {
+ skip: self.skip,
+ fetch: self.fetch,
+ }
+ }
+
#[cfg(feature = "proto")]
fn try_to_proto(
&self,
@@ -552,6 +560,13 @@ impl ExecutionPlan for LocalLimitExec {
true
}
+ fn limit_behavior(&self) -> LimitBehavior {
+ LimitBehavior::Limit {
+ skip: 0,
+ fetch: Some(self.fetch),
+ }
+ }
+
fn cardinality_effect(&self) -> CardinalityEffect {
CardinalityEffect::LowerEqual
}
diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rs
index 81d840ee90..726af00d7a 100644
--- a/datafusion/physical-plan/src/projection.rs
+++ b/datafusion/physical-plan/src/projection.rs
@@ -13,6 +13,7 @@ use super::{
DisplayAs, ExecutionPlanProperties, PlanProperties, RecordBatchStream,
SendableRecordBatchStream, SortOrderPushdownResult, Statistics,
};
+use crate::execution_plan::LimitBehavior;
use crate::execution_plan::{CardinalityEffect, replace_children_if_necessary};
use crate::filter_pushdown::{
ChildFilterDescription, ChildPushdownResult, FilterDescription, FilterPushdownPhase,
@@ -491,6 +492,10 @@ impl ExecutionPlan for ProjectionExec {
true
}
+ fn limit_behavior(&self) -> LimitBehavior {
+ LimitBehavior::MoveToInput
+ }
+
fn cardinality_effect(&self) -> CardinalityEffect {
CardinalityEffect::Equal
}
diff --git a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs
index 76fb4bb441..4cfe4e5098 100644
--- a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs
+++ b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs
@@ -7,6 +7,7 @@
use std::sync::Arc;
use crate::common::spawn_buffered;
+use crate::execution_plan::LimitBehavior;
use crate::limit::LimitStream;
use crate::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet};
use crate::projection::{ProjectionExec, make_with_child, update_ordering};
@@ -417,6 +418,10 @@ impl ExecutionPlan for SortPreservingMergeExec {
true
}
+ fn limit_behavior(&self) -> LimitBehavior {
+ LimitBehavior::Merge
+ }
+
/// Tries to swap the projection with its input [`SortPreservingMergeExec`].
/// If this is possible, it returns the new [`SortPreservingMergeExec`] whose
/// child is a projection. Otherwise, it returns None.
diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rs
index e0d62584f9..3b4cebc67c 100644
--- a/datafusion/physical-plan/src/union.rs
+++ b/datafusion/physical-plan/src/union.rs
@@ -18,6 +18,7 @@ use super::{
PlanProperties, RecordBatchStream, SendableRecordBatchStream, Statistics,
metrics::{ExecutionPlanMetricsSet, MetricsSet},
};
+use crate::execution_plan::LimitBehavior;
use crate::execution_plan::{
CardinalityEffect, InvariantLevel, boundedness_from_children,
check_default_invariants, emission_type_from_children,
@@ -450,6 +451,10 @@ impl ExecutionPlan for UnionExec {
true
}
+ fn limit_behavior(&self) -> LimitBehavior {
+ LimitBehavior::CopyToChildren
+ }
+
/// Tries to push `projection` down through `union`. If possible, performs the
/// pushdown and returns a new [`UnionExec`] as the top plan which has projections
/// as its children. Otherwise, returns `None`.
diff --git a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs
index d5fbc354c6..5c09fa2261 100644
--- a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs
+++ b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs
@@ -56,8 +56,14 @@ use datafusion_physical_expr_common::sort_expr::{
OrderingRequirements, PhysicalSortExpr,
};
-use crate::execution_plan::CardinalityEffect;
+use crate::execution_plan::{CardinalityEffect, LimitBehavior, WindowLookahead};
+use crate::windows::WindowUDFExpr;
use datafusion_common::hash_utils::RandomState;
+use datafusion_expr::{LimitEffect, WindowFrameBound, WindowFrameUnits};
+use datafusion_physical_expr::window::{
+ PlainAggregateWindowExpr, SlidingAggregateWindowExpr, StandardWindowExpr,
+ StandardWindowFunctionExpr,
+};
use futures::stream::Stream;
use futures::{StreamExt, ready};
use hashbrown::hash_table::HashTable;
@@ -535,6 +541,10 @@ impl ExecutionPlan for BoundedWindowAggExec {
CardinalityEffect::Equal
}
+ fn limit_behavior(&self) -> LimitBehavior {
+ LimitBehavior::Window(window_lookahead(&self.window_expr))
+ }
+
#[cfg(feature = "proto")]
fn try_to_proto(
&self,
@@ -1597,6 +1607,56 @@ pub(crate) fn get_last_row_batch(batch: &RecordBatch) -> Result<RecordBatch> {
Ok(batch.slice(batch.num_rows() - 1, 1))
}
+/// The [`WindowLookahead`] of a window computing `exprs`, or `None` when an
+/// expression's frame or function does not bound it before execution.
+fn window_lookahead(exprs: &[Arc<dyn WindowExpr>]) -> Option<WindowLookahead> {
+ let mut lookahead = WindowLookahead::default();
+ for expr in exprs {
+ match limit_effect(expr) {
+ LimitEffect::None => {}
+ LimitEffect::Unknown => return None,
+ LimitEffect::Relative(rows) => {
+ lookahead.relative = lookahead.relative.max(rows)
+ }
+ LimitEffect::Absolute(rows) => {
+ lookahead.min_rows = Some(lookahead.min_rows.unwrap_or(0).max(rows))
+ }
+ }
+ // Only a ROWS frame with a literal end bounds the lookahead.
+ let frame = expr.get_window_frame();
+ if frame.units != WindowFrameUnits::Rows {
+ return None;
+ }
+ let end = match &frame.end_bound {
+ WindowFrameBound::Preceding(_) | WindowFrameBound::CurrentRow => 0,
+ WindowFrameBound::Following(ScalarValue::UInt64(Some(rows))) => {
+ *rows as usize
+ }
+ _ => return None,
+ };
+ lookahead.frame_end = lookahead.frame_end.max(end);
+ }
+ Some(lookahead)
+}
+
+/// The [`LimitEffect`] of one window expression. Plain and sliding aggregates
+/// never read ahead; window functions declare it.
+fn limit_effect(expr: &Arc<dyn WindowExpr>) -> LimitEffect {
+ if expr.as_any().is::<PlainAggregateWindowExpr>()
+ || expr.as_any().is::<SlidingAggregateWindowExpr>()
+ {
+ return LimitEffect::None;
+ }
+ expr.as_any()
+ .downcast_ref::<StandardWindowExpr>()
+ .and_then(|e| {
+ e.get_standard_func_expr()
+ .as_any()
+ .downcast_ref::<WindowUDFExpr>()
+ })
+ .map_or(LimitEffect::Unknown, |udf| udf.limit_effect())
+}
+
#[cfg(test)]
mod tests {
use std::pin::Pin;
diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.md
index e5ee4c2b14..9e09a53377 100644
--- a/docs/source/library-user-guide/upgrading/56.0.0.md
+++ b/docs/source/library-user-guide/upgrading/56.0.0.md
@@ -876,14 +876,74 @@ default) are not affected. As for operators, `Exact` statistics from a provider
must be guaranteed values: `AggregateStatistics` answers `COUNT(*)`, `MIN` and
`MAX` from them, and `LimitPushdown` removes a limit over an `Exact(0)` input.
-For the same reason,
-`datafusion_physical_optimizer::limit_pushdown::pushdown_limit_helper` is
-deprecated in favour of `pushdown_limit_helper_with_stats`, which takes the
-`StatisticsContext` from the caller. The deprecated form creates a new context
-on every call, without statistics providers. When providers are registered,
-switching to `pushdown_limit_helper_with_stats` with a context built from the
-statistics registry can change the result, because the providers are then
-consulted.
+`datafusion_physical_optimizer::limit_pushdown::pushdown_limit_helper` and
+`GlobalRequirements` are deprecated. `LimitPushdown` no longer works one node at
+a time; run the `LimitPushdown` rule instead. `pushdown_limit_helper` creates a
+new `StatisticsContext` on every call, without statistics providers, so its
+result can differ from the rule's when providers are registered.
+
+### `ExecutionPlan::supports_limit_pushdown` is deprecated in favour of `limit_behavior`
+
+Operators now declare what a `LIMIT` on their output means for their input with
+`ExecutionPlan::limit_behavior`, which returns a `LimitBehavior`:
+
+- `Stops` (the default): a limit above the node stays above it.
+- `Limit { skip, fetch }`: the node is a limit.
+- `CopyToChildren`: a limit may be copied to each child as an early-stop hint,
+ but must also stay above the node, which can output more rows than one child
+ (`UnionExec`).
+- `MoveToInput`: the node outputs exactly the rows of its input, so a limit may
+ move below it (`ProjectionExec`).
+- `Merge`: the node merges its input partitions, so it can enforce a limit with
+ its own fetch (`CoalescePartitionsExec`, `SortPreservingMergeExec`). Without a
+ fetch of its own (`ExecutionPlan::fetch`) it behaves like `MoveToInput`.
+- `Window(lookahead)`: a window function node.
+
+`LimitPushdown`, `LimitPushPastWindows` and the sort pushdown in
+`EnsureRequirements` read only `limit_behavior`. Its default is derived from
+`supports_limit_pushdown` and `cardinality_effect`, so an operator that
+implements `supports_limit_pushdown` keeps working: `false` maps to `Stops`;
+`true` maps to `MoveToInput` when `cardinality_effect` is `Equal` and to
+`CopyToChildren` otherwise. Implement `limit_behavior` instead:
+
+```rust,ignore
+fn limit_behavior(&self) -> LimitBehavior {
+ LimitBehavior::MoveToInput
+}
+```
+
+DataFusion's own operators keep returning the same `supports_limit_pushdown`
+values until it is removed, so a wrapper that forwards it to its input keeps
+working, but new code should call `limit_behavior`.
+
+Two differences for operators that still implement `supports_limit_pushdown`:
+
+- `LimitPushPastWindows` now pushes a window's limit below a single-input
+ operator that returns `true` and whose `cardinality_effect` is `Unknown` or
+ `LowerEqual`. Before, it stopped there. `Unknown` is the default, so this
+ includes a wrapper that forwards `supports_limit_pushdown` to its input and
+ does not override `cardinality_effect`:
+
+ ```rust,ignore
+ impl ExecutionPlan for MyWrapper {
+ fn supports_limit_pushdown(&self) -> bool {
+ self.input.supports_limit_pushdown()
+ }
+ // no `cardinality_effect`: the default is `Unknown`
+ }
+ ```
+
+ `LimitPushdown` already moved plain limits below such an operator.
+ `LimitPushPastWindows` only copies the limit below and keeps it above, which
+ asks less of the operator, so this is safe wherever the existing
+ `LimitPushdown` behavior is.
+
+- `LimitPushdown` no longer moves a limit below an operator whose children
+ have more partitions in total than it outputs, such as one that combines
+ several inputs into one partition. Moved below, the limit would apply to each
+ input partition and keep too many rows. The limit now stays above the
+ operator, as it does above `CoalescePartitionsExec`, and is copied below as
+ an early-stop fetch.
### `MergeIntoOp` requires the SQL-visible target qualifier