Skip to content

Apache DataFusion SQL Query Engine

fix: keep per-partition constant sort keys in a partitioned TopK (#26455)

Chao Ning7032b23172a1
diff --git a/datafusion/physical-optimizer/src/utils.rs b/datafusion/physical-optimizer/src/utils.rs
index 04229e1cc2..e4eb746c4e 100644
--- a/datafusion/physical-optimizer/src/utils.rs
+++ b/datafusion/physical-optimizer/src/utils.rs
@@ -18,7 +18,9 @@
use std::sync::Arc;
use datafusion_common::Result;
-use datafusion_physical_expr::{Distribution, LexOrdering, LexRequirement};
+use datafusion_physical_expr::{
+ AcrossPartitions, Distribution, LexOrdering, LexRequirement,
+};
use datafusion_physical_plan::coalesce_partitions::CoalescePartitionsExec;
use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
use datafusion_physical_plan::repartition::RepartitionExec;
@@ -29,6 +31,35 @@ use datafusion_physical_plan::union::UnionExec;
use datafusion_physical_plan::windows::{BoundedWindowAggExec, WindowAggExec};
use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties};
+/// Builds the ordering of a sort placed above `plan` from `sort_requirements`,
+/// leaving out the requirements whose expression is constant in `plan`'s
+/// output.
+///
+/// A constant with the same value in every partition
+/// ([`AcrossPartitions::Uniform`]) is always left out. A constant whose value
+/// differs between partitions ([`AcrossPartitions::Heterogeneous`]) is left out
+/// only when `keep_heterogeneous` is false. Callers set it when an operator
+/// compares rows from different partitions on this ordering, since there the
+/// constant still decides the order. `EquivalenceGroup::is_uniform_constant`
+/// makes the same distinction when it normalizes orderings (#23528).
+fn ordering_without_constants(
+ plan: &dyn ExecutionPlan,
+ sort_requirements: LexRequirement,
+ keep_heterogeneous: bool,
+) -> Option<LexOrdering> {
+ let eq_properties = plan.equivalence_properties();
+ let sort_exprs = sort_requirements
+ .into_iter()
+ .filter(|req| match eq_properties.is_expr_constant(&req.expr) {
+ None => true,
+ Some(AcrossPartitions::Heterogeneous) => keep_heterogeneous,
+ Some(AcrossPartitions::Uniform(_)) => false,
+ })
+ .map(Into::into)
+ .collect::<Vec<_>>();
+ LexOrdering::new(sort_exprs)
+}
+
/// This utility function adds a `SortExec` above an operator according to the
/// given ordering requirements while preserving the original partitioning.
///
@@ -40,19 +71,19 @@ pub fn add_sort_above<T: Clone + Default>(
sort_requirements: LexRequirement,
fetch: Option<usize>,
) -> PlanContext<T> {
- let mut sort_reqs: Vec<_> = sort_requirements.into();
- sort_reqs.retain(|sort_expr| {
- node.plan
- .equivalence_properties()
- .is_expr_constant(&sort_expr.expr)
- .is_none()
- });
- let sort_exprs = sort_reqs.into_iter().map(Into::into).collect::<Vec<_>>();
- let Some(ordering) = LexOrdering::new(sort_exprs) else {
+ let preserve_partitioning = node.plan.output_partitioning().partition_count() > 1;
+ // The partitions of a `SortExec` with a fetch and `preserve_partitioning`
+ // share one TopK threshold, which compares rows across partitions.
+ let keep_heterogeneous = preserve_partitioning && fetch.is_some();
+ let Some(ordering) = ordering_without_constants(
+ node.plan.as_ref(),
+ sort_requirements,
+ keep_heterogeneous,
+ ) else {
return node;
};
let mut new_sort = SortExec::new(ordering, Arc::clone(&node.plan)).with_fetch(fetch);
- if node.plan.output_partitioning().partition_count() > 1 {
+ if preserve_partitioning {
new_sort = new_sort.with_preserve_partitioning(true);
}
PlanContext::new(Arc::new(new_sort), T::default(), vec![node])
@@ -68,19 +99,22 @@ pub fn add_sort_above_with_distribution<T: Clone + Default>(
fetch: Option<usize>,
required_distribution: &Distribution,
) -> PlanContext<T> {
- let mut sort_reqs: Vec<_> = sort_requirements.into();
- sort_reqs.retain(|sort_expr| {
- node.plan
- .equivalence_properties()
- .is_expr_constant(&sort_expr.expr)
- .is_none()
- });
- let sort_exprs = sort_reqs.into_iter().map(Into::into).collect::<Vec<_>>();
- let Some(ordering) = LexOrdering::new(sort_exprs) else {
- return node;
- };
let input_has_multiple_partitions =
node.plan.output_partitioning().partition_count() > 1;
+ let add_merge = matches!(required_distribution, Distribution::SinglePartition)
+ && input_has_multiple_partitions;
+ // A merge compares rows across partitions, and so do the partitions of a
+ // `SortExec` with a fetch and `preserve_partitioning`, which share one TopK
+ // threshold. Both use this ordering.
+ let keep_heterogeneous =
+ input_has_multiple_partitions && (fetch.is_some() || add_merge);
+ let Some(ordering) = ordering_without_constants(
+ node.plan.as_ref(),
+ sort_requirements,
+ keep_heterogeneous,
+ ) else {
+ return node;
+ };
let mut new_sort =
SortExec::new(ordering.clone(), Arc::clone(&node.plan)).with_fetch(fetch);
@@ -92,9 +126,7 @@ pub fn add_sort_above_with_distribution<T: Clone + Default>(
// If the parent requires SinglePartition and the input has multiple partitions,
// wrap the partition-preserving sort in SortPreservingMergeExec.
- if matches!(required_distribution, Distribution::SinglePartition)
- && input_has_multiple_partitions
- {
+ if add_merge {
PlanContext::new(
Arc::new(
SortPreservingMergeExec::new(ordering, Arc::clone(&sort_node.plan))
@@ -163,3 +195,145 @@ pub fn is_repartition(plan: &Arc<dyn ExecutionPlan>) -> bool {
pub fn is_limit(plan: &Arc<dyn ExecutionPlan>) -> bool {
plan.is::<GlobalLimitExec>() || plan.is::<LocalLimitExec>()
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ use arrow::datatypes::{DataType, Field, Schema};
+ use datafusion_common::ScalarValue;
+ use datafusion_physical_expr::PhysicalSortExpr;
+ use datafusion_physical_expr::expressions::{col, lit};
+ use datafusion_physical_plan::empty::EmptyExec;
+ use datafusion_physical_plan::projection::ProjectionExec;
+
+ /// `UnionExec` of one `SELECT <c> AS c0, id AS c1` arm per constant, so
+ /// `c0` is constant in every partition, and differs between them unless
+ /// the constants are equal.
+ fn union_of_constants(constants: &[i64]) -> Arc<dyn ExecutionPlan> {
+ let schema =
+ Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));
+ let arms = constants
+ .iter()
+ .map(|c| {
+ let input = Arc::new(EmptyExec::new(Arc::clone(&schema)));
+ let exprs = vec![
+ (lit(ScalarValue::Int64(Some(*c))), "c0".to_string()),
+ (col("id", &schema).unwrap(), "c1".to_string()),
+ ];
+ Arc::new(ProjectionExec::try_new(exprs, input).unwrap())
+ as Arc<dyn ExecutionPlan>
+ })
+ .collect();
+ UnionExec::try_new(arms).unwrap()
+ }
+
+ /// The requirement `c0 ASC, c1 ASC` on `plan`.
+ fn c0_c1(plan: &Arc<dyn ExecutionPlan>) -> LexRequirement {
+ let schema = plan.schema();
+ LexRequirement::new(["c0", "c1"].map(|name| {
+ PhysicalSortExpr::new_default(col(name, &schema).unwrap()).into()
+ }))
+ .unwrap()
+ }
+
+ fn columns(ordering: &LexOrdering) -> Vec<String> {
+ ordering.iter().map(|e| e.expr.to_string()).collect()
+ }
+
+ /// The ordering of the `SortExec` and of the `SortPreservingMergeExec`, if
+ /// any, at the top of `plan`.
+ fn orderings(plan: &Arc<dyn ExecutionPlan>) -> (Vec<String>, Option<Vec<String>>) {
+ if let Some(merge) = plan.downcast_ref::<SortPreservingMergeExec>() {
+ let (sort, _) = orderings(merge.input());
+ return (sort, Some(columns(merge.expr())));
+ }
+ let sort = plan.downcast_ref::<SortExec>().unwrap();
+ (columns(sort.expr()), None)
+ }
+
+ fn sort_above(
+ plan: Arc<dyn ExecutionPlan>,
+ fetch: Option<usize>,
+ ) -> Arc<dyn ExecutionPlan> {
+ let reqs = c0_c1(&plan);
+ add_sort_above(PlanContext::<()>::new_default(plan), reqs, fetch).plan
+ }
+
+ fn sort_above_with_distribution(
+ plan: Arc<dyn ExecutionPlan>,
+ fetch: Option<usize>,
+ distribution: Distribution,
+ ) -> Arc<dyn ExecutionPlan> {
+ let reqs = c0_c1(&plan);
+ let node = PlanContext::<()>::new_default(plan);
+ add_sort_above_with_distribution(node, reqs, fetch, &distribution).plan
+ }
+
+ #[test]
+ fn partitioned_top_k_keeps_a_constant_that_differs_between_partitions() {
+ let c0_c1 = vec!["c0@0".to_string(), "c1@1".to_string()];
+ assert_eq!(
+ orderings(&sort_above(union_of_constants(&[5, 1]), Some(2))),
+ (c0_c1.clone(), None)
+ );
+ assert_eq!(
+ orderings(&sort_above_with_distribution(
+ union_of_constants(&[5, 1]),
+ Some(2),
+ Distribution::UnspecifiedDistribution,
+ )),
+ (c0_c1, None)
+ );
+ }
+
+ #[test]
+ fn merge_keeps_a_constant_that_differs_between_partitions() {
+ let c0_c1 = vec!["c0@0".to_string(), "c1@1".to_string()];
+ for fetch in [None, Some(2)] {
+ assert_eq!(
+ orderings(&sort_above_with_distribution(
+ union_of_constants(&[5, 1]),
+ fetch,
+ Distribution::SinglePartition,
+ )),
+ (c0_c1.clone(), Some(c0_c1.clone())),
+ "fetch {fetch:?}"
+ );
+ }
+ }
+
+ #[test]
+ fn sort_without_fetch_or_merge_drops_a_constant_that_differs_between_partitions() {
+ let c1 = vec!["c1@1".to_string()];
+ assert_eq!(
+ orderings(&sort_above(union_of_constants(&[5, 1]), None)),
+ (c1.clone(), None)
+ );
+ assert_eq!(
+ orderings(&sort_above_with_distribution(
+ union_of_constants(&[5, 1]),
+ None,
+ Distribution::UnspecifiedDistribution,
+ )),
+ (c1, None)
+ );
+ }
+
+ #[test]
+ fn sort_drops_a_constant_with_the_same_value_in_every_partition() {
+ let c1 = vec!["c1@1".to_string()];
+ assert_eq!(
+ orderings(&sort_above(union_of_constants(&[1, 1]), Some(2))),
+ (c1.clone(), None)
+ );
+ assert_eq!(
+ orderings(&sort_above_with_distribution(
+ union_of_constants(&[1, 1]),
+ Some(2),
+ Distribution::SinglePartition,
+ )),
+ (c1.clone(), Some(c1))
+ );
+ }
+}
diff --git a/datafusion/physical-plan/src/topk/mod.rs b/datafusion/physical-plan/src/topk/mod.rs
index 56a61ccd61..98625816b7 100644
--- a/datafusion/physical-plan/src/topk/mod.rs
+++ b/datafusion/physical-plan/src/topk/mod.rs
@@ -3222,10 +3222,10 @@ mod tests {
use crate::metrics::MetricValue;
use crate::spill::get_record_batch_memory_size;
use arrow::array::{
- AsArray, BooleanArray, Float64Array, Int32Array, StringArray,
+ AsArray, BooleanArray, Float64Array, Int32Array, Int64Array, StringArray,
StringDictionaryBuilder,
};
- use arrow::datatypes::{DataType, Field, Int32Type, Schema, UInt64Type};
+ use arrow::datatypes::{DataType, Field, Int32Type, Int64Type, Schema, UInt64Type};
use arrow_schema::SortOptions;
use datafusion_common::utils::memory::RecordBatchMemoryCounter;
use datafusion_common::{assert_batches_eq, exec_datafusion_err};
@@ -3961,6 +3961,181 @@ mod tests {
Ok(())
}
+ /// Schema for the `UNION ALL` fixture: `c0` is constant within each
+ /// partition but differs between partitions, and `c1` orders the rows.
+ fn make_c0_c1_schema() -> SchemaRef {
+ Arc::new(Schema::new(vec![
+ Field::new("c0", DataType::Int64, false),
+ Field::new("c1", DataType::Int64, false),
+ ]))
+ }
+
+ fn make_c0_c1_batch(schema: SchemaRef, rows: &[(i64, i64)]) -> Result<RecordBatch> {
+ let c0: ArrayRef =
+ Arc::new(Int64Array::from_iter_values(rows.iter().map(|row| row.0)));
+ let c1: ArrayRef =
+ Arc::new(Int64Array::from_iter_values(rows.iter().map(|row| row.1)));
+ Ok(RecordBatch::try_new(schema, vec![c0, c1])?)
+ }
+
+ fn c0_c1_sort_exprs(
+ schema: &SchemaRef,
+ ) -> Result<(PhysicalSortExpr, PhysicalSortExpr)> {
+ Ok((
+ PhysicalSortExpr {
+ expr: col("c0", schema.as_ref())?,
+ options: SortOptions::default(),
+ },
+ PhysicalSortExpr {
+ expr: col("c1", schema.as_ref())?,
+ options: SortOptions::default(),
+ },
+ ))
+ }
+
+ fn make_c0_c1_topk(
+ partition_id: usize,
+ schema: SchemaRef,
+ common_sort_prefix: Vec<PhysicalSortExpr>,
+ expr: LexOrdering,
+ filter: Arc<RwLock<TopKDynamicFilters>>,
+ ) -> Result<TopK> {
+ TopK::try_new(
+ partition_id,
+ schema,
+ common_sort_prefix,
+ expr,
+ 2,
+ 8192,
+ Arc::new(RuntimeEnv::default()),
+ &ExecutionPlanMetricsSet::new(),
+ filter,
+ )
+ }
+
+ /// Inserts the `c0 = 5` and `c0 = 1` partitions into two TopKs that share
+ /// `filter`, in the requested order, then returns the top 2 rows after
+ /// merging the partition outputs on the full `[c0, c1]` key (what the
+ /// `SortPreservingMergeExec` above the TopK does).
+ async fn merged_c0_c1_top2(
+ schema: &SchemaRef,
+ topk_ordering: LexOrdering,
+ common_sort_prefix: Vec<PhysicalSortExpr>,
+ five_first: bool,
+ ) -> Result<Vec<(i64, i64)>> {
+ let filter = make_shared_topk_filter(2);
+ let mut five = make_c0_c1_topk(
+ 0,
+ Arc::clone(schema),
+ common_sort_prefix.clone(),
+ topk_ordering.clone(),
+ Arc::clone(&filter),
+ )?;
+ let mut one = make_c0_c1_topk(
+ 1,
+ Arc::clone(schema),
+ common_sort_prefix,
+ topk_ordering,
+ filter,
+ )?;
+
+ let five_batch = make_c0_c1_batch(Arc::clone(schema), &[(5, 0), (5, 1)])?;
+ let one_batch = make_c0_c1_batch(Arc::clone(schema), &[(1, 0), (1, 1)])?;
+ if five_first {
+ five.insert_batch(five_batch)?;
+ one.insert_batch(one_batch)?;
+ } else {
+ one.insert_batch(one_batch)?;
+ five.insert_batch(five_batch)?;
+ }
+
+ collect_c0_c1_top2([five, one]).await
+ }
+
+ async fn collect_c0_c1_top2(topks: [TopK; 2]) -> Result<Vec<(i64, i64)>> {
+ let mut rows = Vec::new();
+ for topk in topks {
+ let batches: Vec<RecordBatch> = topk.emit()?.try_collect().await?;
+ for batch in batches {
+ let c0 = batch.column(0).as_primitive::<Int64Type>();
+ let c1 = batch.column(1).as_primitive::<Int64Type>();
+ rows.extend((0..batch.num_rows()).map(|i| (c0.value(i), c1.value(i))));
+ }
+ }
+ // `SortPreservingMergeExec` merges on the full `[c0, c1]` key. With
+ // four rows, sorting and truncating is equivalent.
+ rows.sort_unstable();
+ rows.truncate(2);
+ Ok(rows)
+ }
+
+ /// The shared TopK threshold is comparable across partitions only when the
+ /// sort key covers the constant that differs between them. With `[c0, c1]`
+ /// as the TopK key and `[c0]` as the input sort prefix, both feeding orders
+ /// keep both rows of the `c0 = 1` partition that the top 2 needs.
+ #[tokio::test]
+ async fn test_shared_filter_keeps_heterogeneous_constant_in_key() -> Result<()> {
+ let schema = make_c0_c1_schema();
+ let (sort_c0, sort_c1) = c0_c1_sort_exprs(&schema)?;
+
+ for five_first in [true, false] {
+ let rows = merged_c0_c1_top2(
+ &schema,
+ LexOrdering::from([sort_c0.clone(), sort_c1.clone()]),
+ vec![sort_c0.clone()],
+ five_first,
+ )
+ .await?;
+ assert_eq!(rows, vec![(1, 0), (1, 1)], "five_first={five_first}");
+ }
+
+ Ok(())
+ }
+
+ /// A better partition must keep reading until it has enough rows, while a
+ /// worse partition can stop once the shared prefix threshold passes it.
+ #[tokio::test]
+ async fn test_shared_filter_heterogeneous_prefix_across_batches() -> Result<()> {
+ let schema = make_c0_c1_schema();
+ let (sort_c0, sort_c1) = c0_c1_sort_exprs(&schema)?;
+ let ordering = LexOrdering::from([sort_c0.clone(), sort_c1]);
+ let filter = make_shared_topk_filter(2);
+ let mut five = make_c0_c1_topk(
+ 0,
+ Arc::clone(&schema),
+ vec![sort_c0.clone()],
+ ordering.clone(),
+ Arc::clone(&filter),
+ )?;
+ let mut one = make_c0_c1_topk(
+ 1,
+ Arc::clone(&schema),
+ vec![sort_c0],
+ ordering,
+ Arc::clone(&filter),
+ )?;
+
+ five.insert_batch(make_c0_c1_batch(Arc::clone(&schema), &[(5, 0), (5, 1)])?)?;
+ assert!(filter.read().shared_threshold.is_some());
+ assert!(!five.finished);
+
+ one.insert_batch(make_c0_c1_batch(Arc::clone(&schema), &[(1, 0)])?)?;
+ assert!(!one.finished);
+ assert_eq!(one.heap.inner.len(), 1);
+
+ // The first batch must not stop this partition before its second row.
+ one.insert_batch(make_c0_c1_batch(Arc::clone(&schema), &[(1, 1)])?)?;
+ assert!(!one.finished);
+ assert_eq!(one.heap.inner.len(), 2);
+
+ // The better partition now has k rows, so the worse one may stop.
+ five.insert_batch(make_c0_c1_batch(schema, &[(5, 2)])?)?;
+ assert!(five.finished);
+ assert_eq!(five.metrics.row_replacements.value(), 2);
+ assert_eq!(collect_c0_c1_top2([five, one]).await?, vec![(1, 0), (1, 1)]);
+ Ok(())
+ }
+
/// Tests that memory-based compaction triggers when a large batch
/// has very few rows referenced by the top-k heap.
#[tokio::test]
diff --git a/datafusion/sqllogictest/test_files/topk.slt b/datafusion/sqllogictest/test_files/topk.slt
index db55ddca1f..eee05c990b 100644
--- a/datafusion/sqllogictest/test_files/topk.slt
+++ b/datafusion/sqllogictest/test_files/topk.slt
@@ -397,6 +397,256 @@ set datafusion.explain.physical_plan_only = false
statement ok
set datafusion.execution.batch_size = 8192
+# A partitioned TopK must not drop a sort key that is constant within each
+# partition but differs between partitions. All partitions of one `SortExec`
+# share a single TopK threshold, so a threshold on `c1` alone, published by the
+# `c0 = 5` arm, would discard rows that the `c0 = 1` arm needs.
+statement ok
+CREATE TABLE heterogeneous_const(id BIGINT, g BIGINT) AS VALUES (0, 0), (1, 0), (2, 1), (3, 1), (4, 1);
+
+statement ok
+set datafusion.explain.physical_plan_only = true
+
+statement ok
+set datafusion.execution.target_partitions = 1;
+
+query TT
+EXPLAIN SELECT 5 AS c0, id AS c1 FROM heterogeneous_const
+UNION ALL
+SELECT 1 AS c0, id AS c1 FROM heterogeneous_const WHERE g = 0
+ORDER BY c0, c1 LIMIT 2;
+----
+physical_plan
+01)SortPreservingMergeExec: [c0@0 ASC NULLS LAST, c1@1 ASC NULLS LAST], fetch=2
+02)--SortExec: TopK(fetch=2), expr=[c0@0 ASC NULLS LAST, c1@1 ASC NULLS LAST], preserve_partitioning=[true], sort_prefix=[c0@0 ASC NULLS LAST]
+03)----UnionExec
+04)------ProjectionExec: expr=[5 as c0, id@0 as c1]
+05)--------DataSourceExec: partitions=1, partition_sizes=[1]
+06)------ProjectionExec: expr=[1 as c0, id@0 as c1]
+07)--------FilterExec: g@1 = 0, projection=[id@0]
+08)----------DataSourceExec: partitions=1, partition_sizes=[1]
+
+query II
+SELECT 5 AS c0, id AS c1 FROM heterogeneous_const
+UNION ALL
+SELECT 1 AS c0, id AS c1 FROM heterogeneous_const WHERE g = 0
+ORDER BY c0, c1 LIMIT 2;
+----
+1 0
+1 1
+
+# Control: with LIMIT 3 the `c0 = 1` arm never needs a row the threshold cuts
+query II
+SELECT 5 AS c0, id AS c1 FROM heterogeneous_const
+UNION ALL
+SELECT 1 AS c0, id AS c1 FROM heterogeneous_const WHERE g = 0
+ORDER BY c0, c1 LIMIT 3;
+----
+1 0
+1 1
+5 0
+
+# Same bug without the filter: both arms hold the same `c1` values
+query II
+SELECT 5 AS c0, id AS c1 FROM heterogeneous_const
+UNION ALL
+SELECT 1 AS c0, id AS c1 FROM heterogeneous_const
+ORDER BY c0, c1 LIMIT 2;
+----
+1 0
+1 1
+
+# Descending on the constant key: the `c0 = 5` arm comes first. Without the
+# fix, whether the query returns wrong rows depends on which arm publishes its
+# threshold first, so the EXPLAIN is what pins the key
+query TT
+EXPLAIN SELECT 5 AS c0, id AS c1 FROM heterogeneous_const
+UNION ALL
+SELECT 1 AS c0, id AS c1 FROM heterogeneous_const WHERE g = 0
+ORDER BY c0 DESC, c1 LIMIT 2;
+----
+physical_plan
+01)SortPreservingMergeExec: [c0@0 DESC, c1@1 ASC NULLS LAST], fetch=2
+02)--SortExec: TopK(fetch=2), expr=[c0@0 DESC, c1@1 ASC NULLS LAST], preserve_partitioning=[true], sort_prefix=[c0@0 DESC]
+03)----UnionExec
+04)------ProjectionExec: expr=[5 as c0, id@0 as c1]
+05)--------DataSourceExec: partitions=1, partition_sizes=[1]
+06)------ProjectionExec: expr=[1 as c0, id@0 as c1]
+07)--------FilterExec: g@1 = 0, projection=[id@0]
+08)----------DataSourceExec: partitions=1, partition_sizes=[1]
+
+query II
+SELECT 5 AS c0, id AS c1 FROM heterogeneous_const
+UNION ALL
+SELECT 1 AS c0, id AS c1 FROM heterogeneous_const WHERE g = 0
+ORDER BY c0 DESC, c1 LIMIT 2;
+----
+5 0
+5 1
+
+# The constant is kept when it is not the leading key, where it breaks ties on
+# `c1`. The second arm's `c1` values are shifted so that both arms have `c1 = 1`
+query TT
+EXPLAIN SELECT 5 AS c0, id AS c1 FROM heterogeneous_const
+UNION ALL
+SELECT 1 AS c0, id + 1 AS c1 FROM heterogeneous_const WHERE g = 0
+ORDER BY c1, c0 LIMIT 2;
+----
+physical_plan
+01)SortPreservingMergeExec: [c1@1 ASC NULLS LAST, c0@0 ASC NULLS LAST], fetch=2
+02)--SortExec: TopK(fetch=2), expr=[c1@1 ASC NULLS LAST, c0@0 ASC NULLS LAST], preserve_partitioning=[true]
+03)----UnionExec
+04)------ProjectionExec: expr=[5 as c0, id@0 as c1]
+05)--------DataSourceExec: partitions=1, partition_sizes=[1]
+06)------ProjectionExec: expr=[1 as c0, id@0 + 1 as c1]
+07)--------FilterExec: g@1 = 0, projection=[id@0]
+08)----------DataSourceExec: partitions=1, partition_sizes=[1]
+
+query II
+SELECT 5 AS c0, id AS c1 FROM heterogeneous_const
+UNION ALL
+SELECT 1 AS c0, id + 1 AS c1 FROM heterogeneous_const WHERE g = 0
+ORDER BY c1, c0 LIMIT 2;
+----
+5 0
+1 1
+
+# Control: a constant with the same value in every partition stays out of the TopK key
+query TT
+EXPLAIN SELECT 1 AS c0, id AS c1 FROM heterogeneous_const
+UNION ALL
+SELECT 1 AS c0, id AS c1 FROM heterogeneous_const WHERE g = 0
+ORDER BY c0, c1 LIMIT 2;
+----
+physical_plan
+01)SortPreservingMergeExec: [c0@0 ASC NULLS LAST, c1@1 ASC NULLS LAST], fetch=2
+02)--SortExec: TopK(fetch=2), expr=[c1@1 ASC NULLS LAST], preserve_partitioning=[true]
+03)----UnionExec
+04)------ProjectionExec: expr=[1 as c0, id@0 as c1]
+05)--------DataSourceExec: partitions=1, partition_sizes=[1]
+06)------ProjectionExec: expr=[1 as c0, id@0 as c1]
+07)--------FilterExec: g@1 = 0, projection=[id@0]
+08)----------DataSourceExec: partitions=1, partition_sizes=[1]
+
+query II
+SELECT 1 AS c0, id AS c1 FROM heterogeneous_const
+UNION ALL
+SELECT 1 AS c0, id AS c1 FROM heterogeneous_const WHERE g = 0
+ORDER BY c0, c1 LIMIT 2;
+----
+1 0
+1 0
+
+# Control: without a fetch the sort is pushed into each arm, where `c0` has a
+# single value, and the merge above the union sorts on the full key
+query TT
+EXPLAIN SELECT 5 AS c0, id AS c1 FROM heterogeneous_const
+UNION ALL
+SELECT 1 AS c0, id AS c1 FROM heterogeneous_const WHERE g = 0
+ORDER BY c0, c1;
+----
+physical_plan
+01)SortPreservingMergeExec: [c0@0 ASC NULLS LAST, c1@1 ASC NULLS LAST]
+02)--UnionExec
+03)----SortExec: expr=[c1@1 ASC NULLS LAST], preserve_partitioning=[false]
+04)------ProjectionExec: expr=[5 as c0, id@0 as c1]
+05)--------DataSourceExec: partitions=1, partition_sizes=[1]
+06)----SortExec: expr=[c1@1 ASC NULLS LAST], preserve_partitioning=[false]
+07)------ProjectionExec: expr=[1 as c0, id@0 as c1]
+08)--------FilterExec: g@1 = 0, projection=[id@0]
+09)----------DataSourceExec: partitions=1, partition_sizes=[1]
+
+query II
+SELECT 5 AS c0, id AS c1 FROM heterogeneous_const
+UNION ALL
+SELECT 1 AS c0, id AS c1 FROM heterogeneous_const WHERE g = 0
+ORDER BY c0, c1;
+----
+1 0
+1 1
+5 0
+5 1
+5 2
+5 3
+5 4
+
+# With OFFSET the TopK fetches LIMIT + OFFSET rows. Three `c0 = 1` rows are
+# needed, and a threshold on `c1` alone from the `c0 = 5` arm would cut the third
+statement ok
+CREATE TABLE heterogeneous_const_offset(id BIGINT, g BIGINT) AS VALUES (0, 0), (1, 0), (2, 0), (3, 1), (4, 1);
+
+query TT
+EXPLAIN SELECT 5 AS c0, id AS c1 FROM heterogeneous_const_offset
+UNION ALL
+SELECT 1 AS c0, id AS c1 FROM heterogeneous_const_offset WHERE g = 0
+ORDER BY c0, c1 LIMIT 2 OFFSET 1;
+----
+physical_plan
+01)GlobalLimitExec: skip=1, fetch=2
+02)--SortPreservingMergeExec: [c0@0 ASC NULLS LAST, c1@1 ASC NULLS LAST], fetch=3
+03)----SortExec: TopK(fetch=3), expr=[c0@0 ASC NULLS LAST, c1@1 ASC NULLS LAST], preserve_partitioning=[true], sort_prefix=[c0@0 ASC NULLS LAST]
+04)------UnionExec
+05)--------ProjectionExec: expr=[5 as c0, id@0 as c1]
+06)----------DataSourceExec: partitions=1, partition_sizes=[1]
+07)--------ProjectionExec: expr=[1 as c0, id@0 as c1]
+08)----------FilterExec: g@1 = 0, projection=[id@0]
+09)------------DataSourceExec: partitions=1, partition_sizes=[1]
+
+query II
+SELECT 5 AS c0, id AS c1 FROM heterogeneous_const_offset
+UNION ALL
+SELECT 1 AS c0, id AS c1 FROM heterogeneous_const_offset WHERE g = 0
+ORDER BY c0, c1 LIMIT 2 OFFSET 1;
+----
+1 1
+1 2
+
+statement ok
+DROP TABLE heterogeneous_const_offset;
+
+# Three arms give the TopK three partitions. The `c0 = 1` arm is the last one,
+# and both other arms would publish a threshold that cuts its rows. The SLT
+# runner uses 4 partitions, so this also restores its setting.
+statement ok
+set datafusion.execution.target_partitions = 4;
+
+query TT
+EXPLAIN SELECT 5 AS c0, id AS c1 FROM heterogeneous_const
+UNION ALL
+SELECT 3 AS c0, id AS c1 FROM heterogeneous_const
+UNION ALL
+SELECT 1 AS c0, id AS c1 FROM heterogeneous_const WHERE g = 0
+ORDER BY c0, c1 LIMIT 2;
+----
+physical_plan
+01)SortPreservingMergeExec: [c0@0 ASC NULLS LAST, c1@1 ASC NULLS LAST], fetch=2
+02)--SortExec: TopK(fetch=2), expr=[c0@0 ASC NULLS LAST, c1@1 ASC NULLS LAST], preserve_partitioning=[true], sort_prefix=[c0@0 ASC NULLS LAST]
+03)----UnionExec
+04)------ProjectionExec: expr=[5 as c0, id@0 as c1]
+05)--------DataSourceExec: partitions=1, partition_sizes=[1]
+06)------ProjectionExec: expr=[3 as c0, id@0 as c1]
+07)--------DataSourceExec: partitions=1, partition_sizes=[1]
+08)------ProjectionExec: expr=[1 as c0, id@0 as c1]
+09)--------FilterExec: g@1 = 0, projection=[id@0]
+10)----------DataSourceExec: partitions=1, partition_sizes=[1]
+
+query II
+SELECT 5 AS c0, id AS c1 FROM heterogeneous_const
+UNION ALL
+SELECT 3 AS c0, id AS c1 FROM heterogeneous_const
+UNION ALL
+SELECT 1 AS c0, id AS c1 FROM heterogeneous_const WHERE g = 0
+ORDER BY c0, c1 LIMIT 2;
+----
+1 0
+1 1
+
+statement ok
+set datafusion.explain.physical_plan_only = false
+
+statement ok
+DROP TABLE heterogeneous_const;
+
#####################################
## TopK with nested sort keys containing NULL elements
## The dynamic filter must not drop rows that sort before the current