Apache DataFusion SQL Query Engine
Aggregations Support `Partitioning::Range` (#23239)
Gene Bordegaray044a85c57471
diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rsindex 9424322396..cab8dc67f9 100644--- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs+++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs@@ -59,7 +59,9 @@ use datafusion_physical_plan::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy,};-use datafusion_physical_expr::Distribution;+use datafusion_physical_expr::{+ Distribution, Partitioning, RangePartitioning, SplitPoint,+};use datafusion_physical_plan::coalesce_partitions::CoalescePartitionsExec;use datafusion_physical_plan::execution_plan::ExecutionPlan;use datafusion_physical_plan::expressions::col;@@ -325,6 +327,47 @@ fn parquet_exec_multiple_sorted(DataSourceExec::from_data_source(config)}+fn parquet_exec_with_output_partitioning(+ output_partitioning: Partitioning,+) -> Arc<DataSourceExec> {+ let file_groups = (0..output_partitioning.partition_count())+ .map(|partition| {+ FileGroup::new(vec![PartitionedFile::new(format!("p{partition}"), 100)])+ })+ .collect::<Vec<_>>();++ let config = FileScanConfigBuilder::new(+ ObjectStoreUrl::parse("test:///").unwrap(),+ Arc::new(ParquetSource::new(schema())),+ )+ .with_file_groups(file_groups)+ .with_output_partitioning(Some(output_partitioning))+ .build();++ DataSourceExec::from_data_source(config)+}++fn range_partitioning(+ column: &str,+ split_values: impl IntoIterator<Item = i64>,+ options: SortOptions,+) -> Result<Partitioning> {+ let ordering: LexOrdering = [PhysicalSortExpr {+ expr: col(column, &schema())?,+ options,+ }]+ .into();+ let split_points = split_values+ .into_iter()+ .map(|value| SplitPoint::new(vec![ScalarValue::Int64(Some(value))]))+ .collect::<Vec<_>>();++ Ok(Partitioning::Range(RangePartitioning::try_new(+ ordering,+ split_points,+ )?))+}+fn csv_exec() -> Arc<DataSourceExec> {csv_exec_with_sort(vec![])}@@ -700,6 +743,86 @@ impl TestConfig {}}+#[test]+fn range_aggregate_reuses_range_partitioning() -> Result<()> {+ let input = parquet_exec_with_output_partitioning(range_partitioning(+ "a",+ [10, 20, 30],+ SortOptions::default(),+ )?);+ let aggregate =+ aggregate_exec_with_alias(input, vec![("a".to_string(), "a".to_string())]);++ let plan = TestConfig::default()+ .with_query_execution_partitions(4)+ .to_plan(aggregate, &DISTRIB_DISTRIB_SORT);++ assert_plan!(+ plan,+ @r"+ AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]+ DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=Range([a@0 ASC], [(10), (20), (30)], 4), file_type=parquet+ "+ );++ Ok(())+}++#[test]+fn range_grouping_set_aggregate_rehashes_with_grouping_id() -> Result<()> {+ let input = parquet_exec_with_output_partitioning(range_partitioning(+ "a",+ [10, 20],+ SortOptions::default(),+ )?);+ let input_schema = input.schema();+ let group_by = PhysicalGroupBy::new(+ vec![+ (col("a", &input_schema)?, "a".to_string()),+ (col("b", &input_schema)?, "b".to_string()),+ ],+ vec![+ (lit(ScalarValue::Int64(None)), "a".to_string()),+ (lit(ScalarValue::Int64(None)), "b".to_string()),+ ],+ vec![vec![false, true], vec![false, false]],+ true,+ );+ let partial = Arc::new(AggregateExec::try_new(+ AggregateMode::Partial,+ group_by.clone(),+ vec![],+ vec![],+ input,+ Arc::clone(&input_schema),+ )?);+ let aggregate = Arc::new(AggregateExec::try_new(+ AggregateMode::FinalPartitioned,+ group_by.as_final(),+ vec![],+ vec![],+ Arc::clone(&partial) as _,+ partial.schema(),+ )?);++ let plan = TestConfig::default()+ .with_query_execution_partitions(3)+ .to_plan(aggregate, &DISTRIB_DISTRIB_SORT);++ assert_plan!(+ plan,+ @r"+ AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b, __grouping_id@2 as __grouping_id], aggr=[]+ RepartitionExec: partitioning=Hash([a@0, b@1, __grouping_id@2], 3), input_partitions=3+ AggregateExec: mode=Partial, gby=[(a@0 as a, NULL as b), (a@0 as a, b@1 as b)], aggr=[]+ DataSourceExec: file_groups={3 groups: [[p0], [p1], [p2]]}, projection=[a, b, c, d, e], output_partitioning=Range([a@0 ASC], [(10), (20)], 3), file_type=parquet+ "+ );++ Ok(())+}+#[test]fn multi_hash_joins() -> Result<()> {let left = parquet_exec();diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rsindex 305d811fa2..7f3a63ed91 100644--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs@@ -34,8 +34,9 @@ use std::sync::Arc;use crate::output_requirements::OutputRequirementExec;use crate::utils::{- add_sort_above_with_check, is_coalesce_partitions, is_repartition,- is_sort_preserving_merge,+ add_sort_above_with_check, aggregate_can_reuse_range_partitioning,+ is_coalesce_partitions, is_repartition, is_sort_preserving_merge,+ range_partitioning_satisfies_key_partitioning,};use arrow::compute::SortOptions;@@ -699,72 +700,47 @@ fn add_roundrobin_on_top(}}-/// Adds a hash repartition operator:-/// - to increase parallelism, and/or-/// - to satisfy requirements of the subsequent operators.-///-/// Repartition(Hash) is added on top of operator `input`.-///-/// # Arguments-///-/// * `input`: Current node.-/// * `hash_exprs`: Stores Physical Exprs that are used during hashing.-/// * `n_target`: desired target partition number, if partition number of the-/// current executor is less than this value. Partition number will be increased.-/// * `allow_subset_satisfy_partitioning`: Whether to allow subset partitioning logic in satisfaction checks.-/// Set to `false` for partitioned hash joins to ensure exact hash matching.-///-/// # Returns-///-/// A [`Result`] object that contains new execution plan where the desired-/// distribution is satisfied by adding a Hash repartition.-fn add_hash_on_top(- input: DistributionContext,- hash_exprs: Vec<Arc<dyn PhysicalExpr>>,- n_target: usize,+// TODO: remove this private helper once Range generally satisfies+// KeyPartitioned requirements through Partitioning::satisfaction.+// See <https://github.com/apache/datafusion/issues/23266>.+//+// Partial aggregates do not require key partitioning, but they preserve their+// input partitioning for the final aggregate. Until Range satisfies+// KeyPartitioned generally, this check keeps preserve_file_partitions from+// inserting RoundRobin between a reusable Range input and the partial aggregate.+fn partial_aggregate_preserves_reusable_partitioning(+ plan: &Arc<dyn ExecutionPlan>,+ child: &Arc<dyn ExecutionPlan>,allow_subset_satisfy_partitioning: bool,-) -> Result<DistributionContext> {- // Early return if hash repartition is unnecessary- // `RepartitionExec: partitioning=Hash([...], 1), input_partitions=1` is unnecessary.- if n_target == 1 && input.plan.output_partitioning().partition_count() == 1 {- return Ok(input);- }-- let dist = Distribution::KeyPartitioned(hash_exprs);- let satisfaction = input.plan.output_partitioning().satisfaction(- &dist,- input.plan.equivalence_properties(),- allow_subset_satisfy_partitioning,- );-- // Add hash repartitioning when:- // - When subset satisfaction is enabled (current >= threshold): only repartition if not satisfied- // - When below threshold (current < threshold): repartition if expressions don't match OR to increase parallelism- let needs_repartition = if allow_subset_satisfy_partitioning {- !satisfaction.is_satisfied()- } else {- !satisfaction.is_satisfied()- || n_target > input.plan.output_partitioning().partition_count()+) -> bool {+ let Some(aggregate) = plan.downcast_ref::<AggregateExec>() else {+ return false;};-- if needs_repartition {- // When there is an existing ordering, we preserve ordering during- // repartition. This will be rolled back in the future if any of the- // following conditions is true:- // - Preserving ordering is not helpful in terms of satisfying ordering- // requirements.- // - Usage of order preserving variants is not desirable (per the flag- // `config.optimizer.prefer_existing_sort`).- let partitioning = dist.create_partitioning(n_target);- let repartition =- RepartitionExec::try_new(Arc::clone(&input.plan), partitioning)?- .with_preserve_order();- let plan = Arc::new(repartition) as _;-- return Ok(DistributionContext::new(plan, true, vec![input]));+ if aggregate.mode() != &AggregateMode::Partial+ || aggregate.group_expr().is_empty()+ || aggregate.group_expr().has_grouping_set()+ {+ return false;}- Ok(input)+ let group_exprs = aggregate.group_expr().input_exprs();+ let output_partitioning = child.output_partitioning();+ let eq_properties = child.equivalence_properties();+ let key_distribution = Distribution::KeyPartitioned(group_exprs.clone());++ output_partitioning+ .satisfaction(+ &key_distribution,+ eq_properties,+ allow_subset_satisfy_partitioning,+ )+ .is_satisfied()+ || range_partitioning_satisfies_key_partitioning(+ output_partitioning,+ &group_exprs,+ eq_properties,+ allow_subset_satisfy_partitioning,+ )}/// Adds a [`SortPreservingMergeExec`] or a [`CoalescePartitionsExec`] operator@@ -1197,21 +1173,14 @@ pub fn ensure_distribution(hash_necessary,},)| {- let increases_partition_count =- child.plan.output_partitioning().partition_count() < target_partitions;-- let add_roundrobin = enable_round_robin- // Operator benefits from partitioning (e.g. filter):- && roundrobin_beneficial- && roundrobin_beneficial_stats- // Unless partitioning increases the partition count, it is not beneficial:- && increases_partition_count;-// Allow subset satisfaction when:// 1. Current partition count >= threshold// 2. Not a partitioned join since must use exact hash matching for joins// 3. Not a grouping set aggregate (requires exact hash including __grouping_id)let current_partitions = child.plan.output_partitioning().partition_count();+ let preserve_file_partition_threshold_met =+ config.optimizer.preserve_file_partitions > 0+ && current_partitions >= config.optimizer.preserve_file_partitions;// Check if the hash partitioning requirement includes __grouping_id column.// Grouping set aggregates (ROLLUP, CUBE, GROUPING SETS) require exact hash@@ -1232,11 +1201,29 @@ pub fn ensure_distribution(// partitioning to the optimizer. Respect it when the only// reason to repartition would be to increase partition count// beyond the preserved file-group count.- || (config.optimizer.preserve_file_partitions > 0+ || (preserve_file_partition_threshold_met&& current_partitions < target_partitions))&& !is_partitioned_join&& !requires_grouping_id;+ let increases_partition_count = current_partitions < target_partitions;++ let preserve_partial_aggregate_partitioning =+ preserve_file_partition_threshold_met+ && partial_aggregate_preserves_reusable_partitioning(+ &plan,+ &child.plan,+ allow_subset_satisfy_partitioning,+ );++ let add_roundrobin = enable_round_robin+ // Operator benefits from partitioning (e.g. filter):+ && roundrobin_beneficial+ && roundrobin_beneficial_stats+ // Unless partitioning increases the partition count, it is not beneficial:+ && increases_partition_count+ && !preserve_partial_aggregate_partitioning;+// When `repartition_file_scans` is set, attempt to increase// parallelism at the source.//@@ -1258,15 +1245,59 @@ pub fn ensure_distribution(}Distribution::HashPartitioned(exprs)| Distribution::KeyPartitioned(exprs) => {+ let child_partitions =+ child.plan.output_partitioning().partition_count();+ let distribution_satisfied = child+ .plan+ .output_partitioning()+ .satisfaction(+ &requirement,+ child.plan.equivalence_properties(),+ allow_subset_satisfy_partitioning,+ )+ .is_satisfied();+ let range_satisfied_for_aggregate =+ aggregate_can_reuse_range_partitioning(&plan)+ && range_partitioning_satisfies_key_partitioning(+ child.plan.output_partitioning(),+ exprs,+ child.plan.equivalence_properties(),+ allow_subset_satisfy_partitioning,+ );++ let partitioning_satisfied =+ distribution_satisfied || range_satisfied_for_aggregate;++ // When subset satisfaction is enabled, preserve an+ // already-satisfying partitioning. Otherwise, hash+ // repartition may also increase parallelism.+ let needs_hash_repartition = if allow_subset_satisfy_partitioning {+ !partitioning_satisfied+ } else {+ !partitioning_satisfied || target_partitions > child_partitions+ };+ let should_add_hash_repartition =+ hash_necessary && needs_hash_repartition;+// See https://github.com/apache/datafusion/issues/18341#issuecomment-3503238325 for background// When inserting hash is necessary to satisfy hash requirement, insert hash repartition.- if hash_necessary {- child = add_hash_on_top(- child,- exprs.to_vec(),- target_partitions,- allow_subset_satisfy_partitioning,- )?;+ if should_add_hash_repartition {+ // When there is an existing ordering, we preserve ordering during+ // repartition. This will be rolled back in the future if any of the+ // following conditions is true:+ // - Preserving ordering is not helpful in terms of satisfying ordering+ // requirements.+ // - Usage of order preserving variants is not desirable (per the flag+ // `config.optimizer.prefer_existing_sort`).+ let partitioning = Distribution::KeyPartitioned(exprs.to_vec())+ .create_partitioning(target_partitions);+ let repartition = RepartitionExec::try_new(+ Arc::clone(&child.plan),+ partitioning,+ )?+ .with_preserve_order();+ let plan = Arc::new(repartition) as _;+ child = DistributionContext::new(plan, true, vec![child]);}}Distribution::UnspecifiedDistribution => {diff --git a/datafusion/physical-optimizer/src/sanity_checker.rs b/datafusion/physical-optimizer/src/sanity_checker.rsindex 40c6245d89..936bc8271a 100644--- a/datafusion/physical-optimizer/src/sanity_checker.rs+++ b/datafusion/physical-optimizer/src/sanity_checker.rs@@ -24,6 +24,7 @@use std::sync::Arc;use datafusion_common::Result;+use datafusion_physical_expr::Distribution;use datafusion_physical_plan::ExecutionPlan;use datafusion_common::config::{ConfigOptions, OptimizerOptions};@@ -35,6 +36,9 @@ use datafusion_physical_plan::joins::SymmetricHashJoinExec;use datafusion_physical_plan::{ExecutionPlanProperties, get_plan_string};use crate::PhysicalOptimizerRule;+use crate::utils::{+ aggregate_can_reuse_range_partitioning, range_partitioning_satisfies_key_partitioning,+};use datafusion_physical_expr_common::sort_expr::format_physical_sort_requirement_list;use itertools::izip;@@ -136,6 +140,10 @@ fn is_prunable(join: &SymmetricHashJoinExec) -> bool {/// Ensures that the plan is pipeline friendly and the order and/// distribution requirements from its children are satisfied.+#[expect(+ deprecated,+ reason = "HashPartitioned is accepted during the KeyPartitioned migration"+)]pub fn check_plan_sanity(plan: &Arc<dyn ExecutionPlan>,optimizer_options: &OptimizerOptions,@@ -162,11 +170,26 @@ pub fn check_plan_sanity(}}- if !child+ let child_satisfies_distribution = child.output_partitioning().satisfaction(&dist_req, child_eq_props, true)- .is_satisfied()- {+ .is_satisfied();+ let range_satisfies_aggregate_distribution =+ aggregate_can_reuse_range_partitioning(plan)+ && match &dist_req {+ Distribution::HashPartitioned(exprs)+ | Distribution::KeyPartitioned(exprs) => {+ range_partitioning_satisfies_key_partitioning(+ child.output_partitioning(),+ exprs,+ child_eq_props,+ true,+ )+ }+ _ => false,+ };++ if !(child_satisfies_distribution || range_satisfies_aggregate_distribution) {let plan_str = get_plan_string(plan);return plan_err!("Plan: {:?} does not satisfy distribution requirements: {}. Child-{} output partitioning: {}",diff --git a/datafusion/physical-optimizer/src/utils.rs b/datafusion/physical-optimizer/src/utils.rsindex 04229e1cc2..e7a19ca0b0 100644--- a/datafusion/physical-optimizer/src/utils.rs+++ b/datafusion/physical-optimizer/src/utils.rs@@ -18,7 +18,11 @@use std::sync::Arc;use datafusion_common::Result;-use datafusion_physical_expr::{Distribution, LexOrdering, LexRequirement};+use datafusion_physical_expr::{+ Distribution, EquivalenceProperties, LexOrdering, LexRequirement, Partitioning,+ PhysicalExpr, physical_exprs_equal,+};+use datafusion_physical_plan::aggregates::{AggregateExec, AggregateMode};use datafusion_physical_plan::coalesce_partitions::CoalescePartitionsExec;use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};use datafusion_physical_plan::repartition::RepartitionExec;@@ -158,6 +162,78 @@ pub fn is_repartition(plan: &Arc<dyn ExecutionPlan>) -> bool {plan.is::<RepartitionExec>()}+/// TODO: remove once Range generally satisfies KeyPartitioned requirements+/// through Partitioning::satisfaction.+/// See <https://github.com/apache/datafusion/issues/23266>.+///+/// Checks whether range partitioning satisfies a key partitioning requirement.+/// This is intentionally separate from general partitioning satisfaction while+/// range reuse is rolled out operator by operator.+pub(crate) fn range_partitioning_satisfies_key_partitioning(+ partitioning: &Partitioning,+ required_exprs: &[Arc<dyn PhysicalExpr>],+ eq_properties: &EquivalenceProperties,+ allow_subset: bool,+) -> bool {+ match partitioning {+ Partitioning::Range(range) => {+ let partition_exprs = range+ .ordering()+ .iter()+ .map(|sort_expr| Arc::clone(&sort_expr.expr))+ .collect::<Vec<_>>();++ if partition_exprs.is_empty() || required_exprs.is_empty() {+ return false;+ }++ let eq_group = eq_properties.eq_group();+ let normalized_partition_exprs = partition_exprs+ .iter()+ .map(|expr| eq_group.normalize_expr(Arc::clone(expr)))+ .collect::<Vec<_>>();+ let normalized_required_exprs = required_exprs+ .iter()+ .map(|expr| eq_group.normalize_expr(Arc::clone(expr)))+ .collect::<Vec<_>>();++ if physical_exprs_equal(+ &normalized_required_exprs,+ &normalized_partition_exprs,+ ) {+ return true;+ }++ allow_subset+ && normalized_partition_exprs.len() < normalized_required_exprs.len()+ && normalized_partition_exprs.iter().all(|partition_expr| {+ normalized_required_exprs+ .iter()+ .any(|required_expr| partition_expr.eq(required_expr))+ })+ }+ _ => false,+ }+}++/// TODO: remove once Range generally satisfies KeyPartitioned requirements+/// through Partitioning::satisfaction.+/// See <https://github.com/apache/datafusion/issues/23266>.+///+/// Checks whether an aggregate can reuse range partitioning to satisfy its key+/// partitioning requirement.+pub(crate) fn aggregate_can_reuse_range_partitioning(+ plan: &Arc<dyn ExecutionPlan>,+) -> bool {+ plan.downcast_ref::<AggregateExec>()+ .is_some_and(|aggregate| {+ matches!(+ aggregate.mode(),+ AggregateMode::FinalPartitioned | AggregateMode::SinglePartitioned+ ) && !aggregate.group_expr().has_grouping_set()+ })+}+/// Checks whether the given operator is a limit;/// i.e. either a [`LocalLimitExec`] or a [`GlobalLimitExec`].pub fn is_limit(plan: &Arc<dyn ExecutionPlan>) -> bool {diff --git a/datafusion/sqllogictest/test_files/range_partitioning.slt b/datafusion/sqllogictest/test_files/range_partitioning.sltindex 2b7a2cfdf4..7904f92310 100644--- a/datafusion/sqllogictest/test_files/range_partitioning.slt+++ b/datafusion/sqllogictest/test_files/range_partitioning.slt@@ -28,19 +28,22 @@ set datafusion.explain.physical_plan_only = true;########### TEST 1: Aggregate on Range Partition Column-# Scanning range_key preserves source Range partitioning metadata.-# Planning still inserts Hash repartitioning today; later optimizer PRs can-# use this baseline to show when the repartition is removed.+# With subset threshold met and preserve-file disabled, Range([range_key])+# satisfies the aggregate key and avoids repartitioning.##########+statement ok+set datafusion.optimizer.subset_repartition_threshold = 4;++statement ok+set datafusion.optimizer.preserve_file_partitions = 0;+query TTEXPLAIN SELECT range_key, SUM(value) FROM range_partitioned GROUP BY range_key;----physical_plan-01)AggregateExec: mode=FinalPartitioned, gby=[range_key@0 as range_key], aggr=[sum(range_partitioned.value)]-02)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4-03)----AggregateExec: mode=Partial, gby=[range_key@0 as range_key], aggr=[sum(range_partitioned.value)]-04)------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-0.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-1.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-2.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-3.csv]]}, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+01)AggregateExec: mode=SinglePartitioned, gby=[range_key@0 as range_key], aggr=[sum(range_partitioned.value)]+02)--DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-0.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-1.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-2.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-3.csv]]}, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=falsequery IISELECT range_key, SUM(value) FROM range_partitioned GROUP BY range_key ORDER BY range_key;@@ -57,11 +60,16 @@ SELECT range_key, SUM(value) FROM range_partitioned GROUP BY range_key ORDER BY########### TEST 2: Aggregate on Non-Range Column-# Projecting away range_key means the scan output no longer contains the-# expression needed to describe range partitioning, so it reports-# UnknownPartitioning with the same partition count.+# With subset threshold met and preserve-file disabled, grouping on a non-range+# key cannot reuse Range([range_key]) and requires hash repartitioning.##########+statement ok+set datafusion.optimizer.subset_repartition_threshold = 4;++statement ok+set datafusion.optimizer.preserve_file_partitions = 0;+query TTEXPLAIN SELECT non_range_key, SUM(value) FROM range_partitioned GROUP BY non_range_key;----@@ -79,7 +87,182 @@ SELECT non_range_key, SUM(value) FROM range_partitioned GROUP BY non_range_key O##########-# TEST 3: Join on Range Partition Column+# TEST 3: Aggregate Reuses Range Subset Partitioning+# With subset threshold met and preserve-file disabled, Range([range_key])+# satisfies grouping by (range_key, non_range_key).+##########++statement ok+set datafusion.optimizer.subset_repartition_threshold = 4;++statement ok+set datafusion.optimizer.preserve_file_partitions = 0;++query TT+EXPLAIN SELECT range_key, non_range_key, SUM(value) FROM range_partitioned GROUP BY range_key, non_range_key;+----+physical_plan+01)AggregateExec: mode=SinglePartitioned, gby=[range_key@0 as range_key, non_range_key@1 as non_range_key], aggr=[sum(range_partitioned.value)]+02)--DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-0.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-1.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-2.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-3.csv]]}, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false++query III+SELECT range_key, non_range_key, SUM(value) FROM range_partitioned GROUP BY range_key, non_range_key ORDER BY range_key, non_range_key;+----+1 1 10+5 2 50+10 1 100+15 2 150+20 1 200+25 2 250+30 1 300+35 2 350+++##########+# TEST 4: Exact Range Aggregate Below Subset Threshold+# Even when subset satisfaction is disabled, exact Range([range_key])+# satisfies GROUP BY range_key when repartitioning would not increase+# partition count.+##########++statement ok+set datafusion.execution.target_partitions = 4;++statement ok+set datafusion.optimizer.subset_repartition_threshold = 5;++statement ok+set datafusion.optimizer.preserve_file_partitions = 0;++query TT+EXPLAIN SELECT range_key, SUM(value) FROM range_partitioned GROUP BY range_key;+----+physical_plan+01)AggregateExec: mode=SinglePartitioned, gby=[range_key@0 as range_key], aggr=[sum(range_partitioned.value)]+02)--DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-0.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-1.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-2.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-3.csv]]}, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+++##########+# TEST 5: Range Subset Aggregate Rehashes Below Subset Threshold+# Range([range_key]) is only a subset of GROUP BY (range_key, non_range_key),+# so it should not satisfy the aggregate key when subset satisfaction is+# disabled.+##########++statement ok+set datafusion.execution.target_partitions = 4;++statement ok+set datafusion.optimizer.subset_repartition_threshold = 5;++statement ok+set datafusion.optimizer.preserve_file_partitions = 0;++query TT+EXPLAIN SELECT range_key, non_range_key, SUM(value) FROM range_partitioned GROUP BY range_key, non_range_key;+----+physical_plan+01)AggregateExec: mode=FinalPartitioned, gby=[range_key@0 as range_key, non_range_key@1 as non_range_key], aggr=[sum(range_partitioned.value)]+02)--RepartitionExec: partitioning=Hash([range_key@0, non_range_key@1], 4), input_partitions=4+03)----AggregateExec: mode=Partial, gby=[range_key@0 as range_key, non_range_key@1 as non_range_key], aggr=[sum(range_partitioned.value)]+04)------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-0.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-1.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-2.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-3.csv]]}, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+++##########+# TEST 6: Aggregate Rehashes Below Subset Threshold+# With subset threshold 5 and only 4 input partitions, planning repartitions+# to increase parallelism instead of reusing Range partitioning.+##########++statement ok+set datafusion.execution.target_partitions = 5;++statement ok+set datafusion.optimizer.subset_repartition_threshold = 5;++statement ok+set datafusion.optimizer.preserve_file_partitions = 0;++query TT+EXPLAIN SELECT range_key, SUM(value) FROM range_partitioned GROUP BY range_key;+----+physical_plan+01)AggregateExec: mode=FinalPartitioned, gby=[range_key@0 as range_key], aggr=[sum(range_partitioned.value)]+02)--RepartitionExec: partitioning=Hash([range_key@0], 5), input_partitions=5+03)----AggregateExec: mode=Partial, gby=[range_key@0 as range_key], aggr=[sum(range_partitioned.value)]+04)------RepartitionExec: partitioning=RoundRobinBatch(5), input_partitions=4+05)--------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-0.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-1.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-2.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-3.csv]]}, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false++statement ok+set datafusion.execution.target_partitions = 4;++statement ok+reset datafusion.optimizer.subset_repartition_threshold;+++##########+# TEST 7: Aggregate Preserves Range When Preserve File Threshold Met+# With preserve-file threshold 1 and 4 input partitions, Range is preserved+# even though target_partitions is 5.+##########++statement ok+set datafusion.execution.target_partitions = 5;++statement ok+set datafusion.optimizer.subset_repartition_threshold = 4;++statement ok+set datafusion.optimizer.preserve_file_partitions = 1;++query TT+EXPLAIN SELECT range_key, SUM(value) FROM range_partitioned GROUP BY range_key;+----+physical_plan+01)AggregateExec: mode=SinglePartitioned, gby=[range_key@0 as range_key], aggr=[sum(range_partitioned.value)]+02)--DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-0.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-1.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-2.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-3.csv]]}, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false++statement ok+set datafusion.execution.target_partitions = 4;++statement ok+reset datafusion.optimizer.preserve_file_partitions;+++##########+# TEST 8: Aggregate Rehashes When Preserve File Threshold Not Met+# With preserve-file threshold 5 and only 4 input partitions, planning can+# repartition to increase parallelism.+##########++statement ok+set datafusion.execution.target_partitions = 5;++statement ok+set datafusion.optimizer.subset_repartition_threshold = 4;++statement ok+set datafusion.optimizer.preserve_file_partitions = 5;++query TT+EXPLAIN SELECT range_key, SUM(value) FROM range_partitioned GROUP BY range_key;+----+physical_plan+01)AggregateExec: mode=FinalPartitioned, gby=[range_key@0 as range_key], aggr=[sum(range_partitioned.value)]+02)--RepartitionExec: partitioning=Hash([range_key@0], 5), input_partitions=5+03)----AggregateExec: mode=Partial, gby=[range_key@0 as range_key], aggr=[sum(range_partitioned.value)]+04)------RepartitionExec: partitioning=RoundRobinBatch(5), input_partitions=4+05)--------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-0.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-1.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-2.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-3.csv]]}, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false++statement ok+set datafusion.execution.target_partitions = 4;++statement ok+reset datafusion.optimizer.preserve_file_partitions;+++##########+# TEST 9: Join on Range Partition Column# Both inputs expose Range partitioning on range_key. Join planning currently# reaches the unsupported Range output-partitioning path; later optimizer PRs# can replace this baseline with a successful plan and result test.@@ -91,7 +274,7 @@ FROM range_partitioned lJOIN range_partitioned r ON l.range_key = r.range_key;##########-# TEST 4: Union of Range Partitioned Inputs+# TEST 10: Union of Range Partitioned Inputs# Each input exposes Range partitioning on range_key. This records current# UNION ALL behavior before later PRs decide whether compatible range inputs can# preserve Range partitioning across the union.