Apache DataFusion SQL Query Engine
feat: reuse range partitioning for date_trunc aggregation (#26449)
Ben Bellick8aad40bbb95f
diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rsindex 295bace596..e5aeab72d4 100644--- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs+++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs@@ -28,9 +28,12 @@ use crate::physical_optimizer::test_utils::{sort_preserving_merge_exec, union_exec,};-use arrow::array::{Int64Array, RecordBatch, UInt8Array, UInt64Array};+use arrow::array::{+ Int64Array, RecordBatch, TimestampNanosecondArray, UInt8Array, UInt64Array,+};use arrow::compute::SortOptions;-use arrow_schema::{DataType, Field, Schema, SchemaRef};+use arrow_schema::{DataType, Field, Schema, SchemaRef, TimeUnit};+use datafusion::assert_batches_sorted_eq;use datafusion::config::ConfigOptions;use datafusion::datasource::MemTable;use datafusion::datasource::file_format::file_compression_type::FileCompressionType;@@ -75,7 +78,7 @@ use datafusion_physical_plan::operator_statistics::{use datafusion_physical_expr::{AcrossPartitions, ConstExpr, Distribution, Partitioning, RangePartitioning,- SplitPoint,+ ScalarFunctionExpr, SplitPoint,};use datafusion_physical_plan::coalesce_partitions::CoalescePartitionsExec;use datafusion_physical_plan::execution_plan::ExecutionPlan;@@ -959,6 +962,233 @@ fn range_aggregate_reuses_range_partitioning() -> Result<()> {Ok(())}+fn timestamp_range_input(split: i64) -> Result<Arc<dyn ExecutionPlan>> {+ let schema = Arc::new(Schema::new(vec![Field::new(+ "timestamp",+ DataType::Timestamp(TimeUnit::Nanosecond, None),+ false,+ )]));+ let hour = 3_600_000_000_000;+ let (before_split, after_split): (Vec<_>, Vec<_>) =+ [0, 1, hour, hour + 1, 2 * hour, 2 * hour + 1]+ .into_iter()+ .partition(|timestamp| *timestamp < split);+ let partitions = [before_split, after_split]+ .into_iter()+ .map(|values| -> Result<_> {+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(TimestampNanosecondArray::from(values))],+ )?;+ Ok(vec![batch])+ })+ .collect::<Result<Vec<_>>>()?;+ let range = RangePartitioning::try_new(+ [PhysicalSortExpr::new_default(col("timestamp", &schema)?)].into(),+ vec![SplitPoint::new(vec![ScalarValue::TimestampNanosecond(+ Some(split),+ None,+ )])],+ )?;+ let source = MemorySourceConfig::try_new_exec(&partitions, schema, None)?;+ Ok(Arc::new(+ source+ .as_ref()+ .clone()+ .with_partitioning(Partitioning::Range(range)),+ ))+}++fn date_trunc_expr(+ input: &dyn ExecutionPlan,+ precision: &str,+) -> Result<Arc<dyn PhysicalExpr>> {+ Ok(Arc::new(ScalarFunctionExpr::new(+ "date_trunc",+ datafusion_functions::datetime::date_trunc(),+ vec![lit(precision), col("timestamp", &input.schema())?],+ Field::new(+ "bucket",+ DataType::Timestamp(TimeUnit::Nanosecond, None),+ false,+ )+ .into(),+ Arc::new(ConfigOptions::default()),+ )))+}++/// Builds separate Partial and Final stages without projected range metadata.+/// This exercises checking group locality through the partial stage's input.+fn partial_final_date_trunc_count(+ input: Arc<dyn ExecutionPlan>,+ precision: &str,+) -> Result<Arc<dyn ExecutionPlan>> {+ let input_schema = input.schema();+ let group_by = PhysicalGroupBy::new_single(vec![(+ date_trunc_expr(input.as_ref(), precision)?,+ "bucket".to_string(),+ )]);+ let count = Arc::new(+ AggregateExprBuilder::new(count_udaf(), vec![col("timestamp", &input_schema)?])+ .schema(Arc::clone(&input_schema))+ .alias("count")+ .build()?,+ );+ let partial = Arc::new(AggregateExec::try_new(+ AggregateMode::Partial,+ group_by.clone(),+ vec![Arc::clone(&count)],+ vec![None],+ input,+ Arc::clone(&input_schema),+ )?);+ assert!(matches!(+ partial.properties().output_partitioning(),+ Partitioning::UnknownPartitioning(2)+ ));++ Ok(Arc::new(AggregateExec::try_new(+ AggregateMode::FinalPartitioned,+ group_by.as_final(),+ vec![count],+ vec![None],+ partial,+ input_schema,+ )?))+}++#[tokio::test]+async fn aligned_hour_groups_avoid_shuffle_through_partial_aggregate() -> Result<()> {+ let hour = 3_600_000_000_000;+ let input = timestamp_range_input(hour)?;+ let aggregate = partial_final_date_trunc_count(input, "hour")?;+ let config = TestConfig::default()+ .with_query_execution_partitions(2)+ .config;+ let optimized = EnsureRequirements::new().optimize(aggregate, &config)?;+ SanityCheckPlan::new().optimize(Arc::clone(&optimized), &config)?;++ assert_plan!(+ optimized,+ @r"+ AggregateExec: mode=FinalPartitioned, gby=[bucket@0 as bucket], aggr=[count]+ AggregateExec: mode=Partial, gby=[date_trunc(hour, timestamp@0) as bucket], aggr=[count]+ DataSourceExec: partitions=2, partition_sizes=[1, 1]+ "+ );++ let batches = collect(optimized, SessionContext::new().task_ctx()).await?;+ assert_batches_sorted_eq!(+ [+ "+---------------------+-------+",+ "| bucket | count |",+ "+---------------------+-------+",+ "| 1970-01-01T00:00:00 | 2 |",+ "| 1970-01-01T01:00:00 | 2 |",+ "| 1970-01-01T02:00:00 | 2 |",+ "+---------------------+-------+",+ ],+ &batches+ );+ Ok(())+}++#[tokio::test]+async fn crossing_day_groups_keep_shuffle_through_partial_aggregate() -> Result<()> {+ let hour = 3_600_000_000_000;+ let input = timestamp_range_input(hour)?;+ let aggregate = partial_final_date_trunc_count(input, "day")?;+ let config = TestConfig::default()+ .with_query_execution_partitions(2)+ .config;+ let optimized = EnsureRequirements::new().optimize(aggregate, &config)?;+ SanityCheckPlan::new().optimize(Arc::clone(&optimized), &config)?;++ assert_plan!(+ optimized,+ @r"+ AggregateExec: mode=FinalPartitioned, gby=[bucket@0 as bucket], aggr=[count]+ RepartitionExec: partitioning=Hash([bucket@0], 2), input_partitions=2+ AggregateExec: mode=Partial, gby=[date_trunc(day, timestamp@0) as bucket], aggr=[count]+ DataSourceExec: partitions=2, partition_sizes=[1, 1]+ "+ );++ let batches = collect(optimized, SessionContext::new().task_ctx()).await?;+ assert_batches_sorted_eq!(+ [+ "+---------------------+-------+",+ "| bucket | count |",+ "+---------------------+-------+",+ "| 1970-01-01T00:00:00 | 6 |",+ "+---------------------+-------+",+ ],+ &batches+ );+ Ok(())+}++#[test]+fn range_transform_aggregate_optimization_is_idempotent() -> Result<()> {+ let hour = 3_600_000_000_000;+ let config = TestConfig::default()+ .with_query_execution_partitions(2)+ .config;++ for precision in ["hour", "day"] {+ let input = timestamp_range_input(hour)?;+ let aggregate = partial_final_date_trunc_count(input, precision)?;+ let optimized = EnsureRequirements::new().optimize(aggregate, &config)?;+ SanityCheckPlan::new().optimize(Arc::clone(&optimized), &config)?;+ let reoptimized =+ EnsureRequirements::new().optimize(Arc::clone(&optimized), &config)?;+ SanityCheckPlan::new().optimize(Arc::clone(&reoptimized), &config)?;+ assert_plan!(optimized, reoptimized);+ }+ Ok(())+}++#[tokio::test]+async fn range_transform_locality_does_not_satisfy_join_distribution() -> Result<()> {+ let hour = 3_600_000_000_000;+ let left = timestamp_range_input(hour)?;+ let right = timestamp_range_input(2 * hour)?;+ let left_key = date_trunc_expr(left.as_ref(), "hour")?;+ let right_key = date_trunc_expr(right.as_ref(), "hour")?;+ for (input, key) in [(&left, &left_key), (&right, &right_key)] {+ assert!(input.output_partitioning().keeps_keys_local(+ &[Arc::clone(key)],+ input.equivalence_properties(),+ false,+ ));+ }+ let join =+ hash_join_exec(left, right, &vec![(left_key, right_key)], &JoinType::Inner);+ let config = TestConfig::default()+ .with_query_execution_partitions(2)+ .config;+ let optimized = EnsureRequirements::new().optimize(join, &config)?;+ SanityCheckPlan::new().optimize(Arc::clone(&optimized), &config)?;+ assert_plan!(+ optimized,+ @r"+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(date_trunc(hour, timestamp@0), date_trunc(hour, timestamp@0))]+ RepartitionExec: partitioning=Hash([date_trunc(hour, timestamp@0)], 2), input_partitions=2+ DataSourceExec: partitions=2, partition_sizes=[1, 1]+ RepartitionExec: partitioning=Hash([date_trunc(hour, timestamp@0)], 2), input_partitions=2+ DataSourceExec: partitions=2, partition_sizes=[1, 1]+ "+ );++ let batches = collect(optimized, SessionContext::new().task_ctx()).await?;+ assert_eq!(+ batches.iter().map(RecordBatch::num_rows).sum::<usize>(),+ 3 * 2 * 2,+ "three hour buckets, each joining two left rows with two right rows"+ );+ Ok(())+}+#[test]fn range_grouping_set_aggregate_rehashes_with_grouping_id() -> Result<()> {let input = parquet_exec_with_output_partitioning(range_partitioning(diff --git a/datafusion/expr/src/udf.rs b/datafusion/expr/src/udf.rsindex d937a54029..f900564d45 100644--- a/datafusion/expr/src/udf.rs+++ b/datafusion/expr/src/udf.rs@@ -364,6 +364,15 @@ impl ScalarUDF {self.inner.output_ordering(inputs)}+ /// See [`ScalarUDFImpl::supports_range_key_locality_analysis`] for more details.+ pub fn supports_range_key_locality_analysis(+ &self,+ argument_types: &[DataType],+ ) -> bool {+ self.inner+ .supports_range_key_locality_analysis(argument_types)+ }+pub fn preserves_lex_ordering(&self, inputs: &[ExprProperties]) -> Result<bool> {self.inner.preserves_lex_ordering(inputs)}@@ -983,6 +992,29 @@ pub trait ScalarUDFImpl: Debug + DynEq + DynHash + Send + Sync + Any {Ok(inner(inputs))}+ /// Returns whether this function and argument-type combination has been+ /// audited for range-partitioning locality analysis.+ ///+ /// Range-boundary analysis has subtle correctness requirements, so the+ /// policy is default-deny. Functions may opt in only after individual audit.+ /// At minimum, an opted-in combination must:+ ///+ /// - have same-direction [`SortProperties::Ordered`] metadata over the+ /// admitted non-null domain when all other inputs are singleton;+ /// - preserve nullness exactly, so successful evaluation returns null if and+ /// only if the range input is null.+ ///+ /// Callers may then prove key locality by evaluating values on both sides of+ /// each concrete range split. Ordered metadata alone is insufficient because+ /// a non-strict transform can collapse a split boundary.+ ///+ /// This does not claim a concrete partition layout or cross-input+ /// co-partitioning compatibility. Boundary evaluation errors cause the+ /// analysis to return `false`. This capability is not transported across FFI.+ fn supports_range_key_locality_analysis(&self, _argument_types: &[DataType]) -> bool {+ false+ }+/// Returns true if the function preserves lexicographical ordering based on/// the input ordering.///@@ -1205,6 +1237,11 @@ impl ScalarUDFImpl for AliasedScalarUDFImpl {self.inner.output_ordering(inputs)}+ fn supports_range_key_locality_analysis(&self, argument_types: &[DataType]) -> bool {+ self.inner+ .supports_range_key_locality_analysis(argument_types)+ }+fn preserves_lex_ordering(&self, inputs: &[ExprProperties]) -> Result<bool> {self.inner.preserves_lex_ordering(inputs)}@@ -1287,6 +1324,12 @@ mod tests {assert_eq!(b.partial_cmp(&o), Some(Ordering::Less));}+ #[test]+ fn range_key_locality_analysis_is_disabled_by_default() {+ let function = test_func("range_function", "a");+ assert!(!function.supports_range_key_locality_analysis(&[DataType::Int64]));+ }+fn test_func(name: &'static str, parameter: &'static str) -> ScalarUDF {ScalarUDF::from(TestScalarUDFImpl {name,diff --git a/datafusion/functions/src/datetime/date_trunc.rs b/datafusion/functions/src/datetime/date_trunc.rsindex 3d304989af..89fd40603f 100644--- a/datafusion/functions/src/datetime/date_trunc.rs+++ b/datafusion/functions/src/datetime/date_trunc.rs@@ -571,6 +571,18 @@ impl ScalarUDFImpl for DateTruncFunc {Ok(SortProperties::Unordered)}}++ fn supports_range_key_locality_analysis(&self, argument_types: &[DataType]) -> bool {+ // Across an America/Goose_Bay transition, ascending epoch seconds+ // 562129259 < 562129260 truncate to 562129200 > 562125600.+ // Keep the audited domain to timezone-less timestamp columns.+ let [precision, timestamp] = argument_types else {+ return false;+ };++ precision.is_string() && matches!(timestamp, Timestamp(_, None))+ }+fn documentation(&self) -> Option<&Documentation> {self.doc()}@@ -1339,7 +1351,10 @@ mod tests {};use arrow::array::cast::as_primitive_array;- use arrow::array::types::{ArrowTimestampType, TimestampNanosecondType};+ use arrow::array::types::{+ ArrowTimestampType, TimestampMicrosecondType, TimestampMillisecondType,+ TimestampNanosecondType, TimestampSecondType,+ };use arrow::array::{Array, PrimitiveArray, TimestampMicrosecondArray, TimestampMillisecondArray,TimestampNanosecondArray, TimestampSecondArray,@@ -1394,6 +1409,91 @@ mod tests {);}+ #[test]+ fn range_key_locality_analysis_requires_timezone_less_timestamp() {+ let date_trunc = DateTruncFunc::new();++ assert!(date_trunc.supports_range_key_locality_analysis(&[+ DataType::Utf8,+ DataType::Timestamp(TimeUnit::Nanosecond, None),+ ]));+ assert!(!date_trunc.supports_range_key_locality_analysis(&[+ DataType::Utf8,+ DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())),+ ]));+ assert!(!date_trunc.supports_range_key_locality_analysis(&[+ DataType::Int64,+ DataType::Timestamp(TimeUnit::Nanosecond, None),+ ]));+ assert!(!date_trunc.supports_range_key_locality_analysis(&[DataType::Utf8]));+ }++ fn invoke_date_trunc(+ granularity: &str,+ timestamp: ColumnarValue,+ timestamp_type: DataType,+ number_rows: usize,+ ) -> datafusion_common::Result<ColumnarValue> {+ DateTruncFunc::new().invoke_with_args(ScalarFunctionArgs {+ args: vec![ScalarValue::from(granularity).into(), timestamp],+ arg_fields: vec![+ Field::new("granularity", DataType::Utf8, false).into(),+ Field::new("timestamp", timestamp_type.clone(), true).into(),+ ],+ number_rows,+ return_field: Field::new("date_trunc", timestamp_type, true).into(),+ config_options: Arc::new(ConfigOptions::default()),+ })+ }++ fn assert_timezone_less_date_trunc_preserves_nullness<T: ArrowTimestampType>() {+ let input = PrimitiveArray::<T>::from_iter([Some(0), None]);+ let timestamp_type = DataType::Timestamp(T::UNIT, None);++ for granularity in [+ "microsecond",+ "millisecond",+ "second",+ "minute",+ "hour",+ "day",+ "week",+ "month",+ "quarter",+ "year",+ ] {+ let output = invoke_date_trunc(+ granularity,+ ColumnarValue::Array(Arc::new(input.clone())),+ timestamp_type.clone(),+ input.len(),+ )+ .unwrap();+ let ColumnarValue::Array(output) = output else {+ panic!("expected array output")+ };++ assert_eq!(+ input.nulls(),+ output.nulls(),+ "{granularity} must preserve nullness for {:?}",+ T::UNIT+ );+ }+ }++ /// Verifies the exact null contract required by+ /// [`ScalarUDFImpl::supports_range_key_locality_analysis`] across every+ /// supported granularity and timezone-less timestamp unit. Ordering metadata+ /// and concrete range-split separation are tested separately.+ #[test]+ fn timezone_less_date_trunc_preserves_nullness() {+ assert_timezone_less_date_trunc_preserves_nullness::<TimestampSecondType>();+ assert_timezone_less_date_trunc_preserves_nullness::<TimestampMillisecondType>();+ assert_timezone_less_date_trunc_preserves_nullness::<TimestampMicrosecondType>();+ assert_timezone_less_date_trunc_preserves_nullness::<TimestampNanosecondType>();+ }+#[test]fn date_trunc_test() {let cases = vec![diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rsindex 1c2d970ccf..ed824e5712 100644--- a/datafusion/physical-expr/src/equivalence/properties/mod.rs+++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs@@ -39,6 +39,7 @@ use crate::{PhysicalSortRequirement,};+use arrow::compute::SortOptions;use arrow::datatypes::SchemaRef;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};use datafusion_common::{Constraint, Constraints, HashMap, Result, plan_err};@@ -1408,6 +1409,29 @@ impl EquivalenceProperties {.unwrap_or_else(|_| ExprProperties::new_unknown())}+ /// Returns whether `transformed_expr` reports the same ordering direction+ /// when `source_expr` is ordered and all other inputs are singleton.+ pub(crate) fn is_same_direction_ordered_transform(+ &self,+ transformed_expr: &Arc<dyn PhysicalExpr>,+ source_expr: &Arc<dyn PhysicalExpr>,+ ) -> bool {+ if transformed_expr.eq(source_expr) {+ return false;+ }++ let options = SortOptions::default();+ let dependencies = Dependencies::new(std::iter::once(PhysicalSortExpr::new(+ Arc::clone(source_expr),+ options,+ )));+ matches!(+ get_expr_properties(transformed_expr, &dependencies, &self.schema)+ .map(|properties| properties.sort_properties),+ Ok(SortProperties::Ordered(actual)) if actual == options+ )+ }+/// Transforms this `EquivalenceProperties` by mapping columns in the/// original schema to columns in the new schema by index.pub fn with_new_schema(mut self, schema: SchemaRef) -> Result<Self> {diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rsindex 336d09b987..15d7849ebe 100644--- a/datafusion/physical-expr/src/partitioning.rs+++ b/datafusion/physical-expr/src/partitioning.rs@@ -17,13 +17,17 @@//! [`Partitioning`] and [`Distribution`] for `ExecutionPlans`+use crate::expressions::{Literal, UnKnownColumn};+use crate::simplifier::const_evaluator::create_dummy_batch;use crate::{- EquivalenceProperties, PhysicalExpr, equivalence::ProjectionMapping,- expressions::UnKnownColumn, physical_exprs_contains, physical_exprs_equal,+ EquivalenceProperties, PhysicalExpr, ScalarFunctionExpr,+ equivalence::ProjectionMapping, physical_exprs_contains, physical_exprs_equal,};use arrow::datatypes::Schema;pub use datafusion_common::SplitPoint;-use datafusion_common::{Result, plan_err, validate_range_split_points};+use datafusion_common::tree_node::{Transformed, TreeNode};+use datafusion_common::{Result, ScalarValue, plan_err, validate_range_split_points};+use datafusion_expr::ColumnarValue;use datafusion_physical_expr_common::physical_expr::format_physical_expr_list;use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};#[cfg(feature = "proto")]@@ -532,6 +536,196 @@ fn normalize_exprs(.collect()}+fn evaluate_with_range_value(+ expr: &Arc<dyn PhysicalExpr>,+ range_key: &Arc<dyn PhysicalExpr>,+ value: &ScalarValue,+) -> Option<ScalarValue> {+ let literal: Arc<dyn PhysicalExpr> = Arc::new(Literal::new(value.clone()));+ let rewritten = Arc::clone(expr)+ .transform(|node| {+ if node.eq(range_key) {+ Ok(Transformed::yes(Arc::clone(&literal)))+ } else {+ Ok(Transformed::no(node))+ }+ })+ .ok()?;+ if !rewritten.transformed {+ return None;+ }++ match rewritten.data.evaluate(create_dummy_batch().ok()?).ok()? {+ ColumnarValue::Scalar(value) => Some(value),+ ColumnarValue::Array(array) => ScalarValue::try_from_array(&array, 0).ok(),+ }+}++fn checked_adjacent_timestamp(+ value: &ScalarValue,+ is_successor: bool,+) -> Option<ScalarValue> {+ let offset = if is_successor { 1 } else { -1 };+ match value {+ ScalarValue::TimestampSecond(Some(value), None) => value+ .checked_add(offset)+ .map(|value| ScalarValue::TimestampSecond(Some(value), None)),+ ScalarValue::TimestampMillisecond(Some(value), None) => value+ .checked_add(offset)+ .map(|value| ScalarValue::TimestampMillisecond(Some(value), None)),+ ScalarValue::TimestampMicrosecond(Some(value), None) => value+ .checked_add(offset)+ .map(|value| ScalarValue::TimestampMicrosecond(Some(value), None)),+ ScalarValue::TimestampNanosecond(Some(value), None) => value+ .checked_add(offset)+ .map(|value| ScalarValue::TimestampNanosecond(Some(value), None)),+ _ => None,+ }+}++/// Determines whether a nondecreasing transform keeps equal output keys local.+///+/// For one ascending range key, split points `x₁, ..., xₙ` define:+///+/// ```text+/// P₀ = (-∞, x₁)+/// Pᵢ = [xᵢ, xᵢ₊₁) for 1 ≤ i < n+/// Pₙ = [xₙ, +∞)+/// ```+///+/// Aggregation can reuse these partitions after applying `f` only if no output+/// key occurs in two partitions. In other words, the partition images must be+/// disjoint: `f(Pᵢ) ∩ f(Pⱼ) = ∅` for `i ≠ j`.+///+/// Monotonicity alone is insufficient. Ordered metadata gives only+///+/// ```text+/// f(predecessor(xᵢ)) ≤ f(xᵢ)+/// ```+///+/// but locality requires strict separation at every split:+///+/// ```text+/// f(predecessor(xᵢ)) < f(xᵢ)+/// ```+///+/// Since ordering already supplies the non-strict inequality, the analysis+/// proves strictness by explicitly checking that the two outputs are not equal.+/// Otherwise, the same transformed key could occur in both adjacent partitions,+/// so the transform would not preserve key locality. For example, an hourly+/// truncation maps timestamps immediately before and at a `01:30` split to the+/// same `01:00` bucket. A split at `01:00` is safe because its predecessor maps+/// to `00:00`, while the split itself maps to `01:00`.+///+/// `predecessor(xᵢ)` is the greatest representable timestamp below the split.+/// Therefore, for every `x < xᵢ` and `y ≥ xᵢ`, ordering and the strict boundary+/// check give+///+/// ```text+/// f(x) ≤ f(predecessor(xᵢ)) < f(xᵢ) ≤ f(y)+/// ```+///+/// Thus no transformed key can occur on both sides of the split. The successor of+/// an ascending split is already in the same partition as `xᵢ`, so checking it+/// would not say anything about groups crossing that split.+///+/// The argument is the same for descending ranges, but `successor(xᵢ)` must be+/// used instead. Null preservation is also required so the transform cannot+/// introduce a null group outside the source partition selected by null order.+fn range_splits_separate_transformed_keys(+ function: &Arc<dyn PhysicalExpr>,+ range_key: &Arc<dyn PhysicalExpr>,+ range: &RangePartitioning,+ schema: &Schema,+) -> bool {+ let Ok(range_type) = range_key.data_type(schema) else {+ return false;+ };+ let sort_options = range.ordering().first().options;+ if validate_range_split_points(range.split_points(), &[sort_options]).is_err() {+ return false;+ }+ range.split_points().iter().all(|split_point| {+ let Some(split_value) = split_point.values().first() else {+ return false;+ };+ if split_value.data_type() != range_type {+ return false;+ }+ let Some(adjacent) =+ checked_adjacent_timestamp(split_value, sort_options.descending)+ else {+ return false;+ };+ let Some(at_split) = evaluate_with_range_value(function, range_key, split_value)+ else {+ return false;+ };+ let Some(across_split) =+ evaluate_with_range_value(function, range_key, &adjacent)+ else {+ return false;+ };++ !at_split.is_null()+ && !across_split.is_null()+ && at_split.data_type() == across_split.data_type()+ && at_split != across_split+ })+}++fn range_transform_proves_key_locality(+ candidate: &Arc<dyn PhysicalExpr>,+ range_key: &Arc<dyn PhysicalExpr>,+ range: &RangePartitioning,+ eq_properties: &EquivalenceProperties,+) -> bool {+ let Some(function) = candidate.downcast_ref::<ScalarFunctionExpr>() else {+ return false;+ };+ let schema = eq_properties.schema();+ if datafusion_physical_expr_common::physical_expr::is_volatile(candidate)+ || !function+ .supports_range_key_locality_analysis(schema)+ .unwrap_or(false)+ || !eq_properties.is_same_direction_ordered_transform(candidate, range_key)+ {+ return false;+ }++ let mut range_argument_count = 0;+ for argument in function.args() {+ if eq_properties.eq_group().exprs_equal(argument, range_key) {+ range_argument_count += 1;+ } else if !argument.is::<Literal>() {+ return false;+ }+ }++ range_argument_count == 1+ && range_splits_separate_transformed_keys(candidate, range_key, range, schema)+}++fn range_partitioning_keeps_keys_local(+ range: &RangePartitioning,+ required_exprs: &[Arc<dyn PhysicalExpr>],+ eq_properties: &EquivalenceProperties,+ allow_subset: bool,+) -> bool {+ // Currently, transform locality analysis only supports single-key ranges.+ // Extending the proof to compound range keys is tracked in #24644.+ if range.ordering().len() != 1 || required_exprs.is_empty() {+ return false;+ }++ let range_key = &range.ordering()[0].expr;+ let has_key_locality_proof = required_exprs.iter().any(|required| {+ range_transform_proves_key_locality(required, range_key, range, eq_properties)+ });++ has_key_locality_proof && (required_exprs.len() == 1 || allow_subset)+}+/// Represents how a [`Partitioning`] satisfies a [`Distribution`] requirement.#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub enum PartitioningSatisfaction {@@ -646,6 +840,41 @@ impl Partitioning {}}+ /// Returns true when it can prove that equal tuples of `required_exprs`+ /// cannot occur in different partitions.+ ///+ /// Unlike [`Self::satisfaction`], this does not claim a concrete partition+ /// layout. This weaker guarantee is sufficient for independent keyed+ /// operations such as aggregation, but not for co-partitioning multiple+ /// inputs. Transform analysis is currently limited to single-key range+ /// partitioning.+ pub fn keeps_keys_local(+ &self,+ required_exprs: &[Arc<dyn PhysicalExpr>],+ eq_properties: &EquivalenceProperties,+ allow_subset: bool,+ ) -> bool {+ let required = Distribution::KeyPartitioned(required_exprs.to_vec());+ if self+ .satisfaction(&required, eq_properties, allow_subset)+ .is_satisfied()+ {+ return true;+ }++ match self {+ Self::Range(range) => range_partitioning_keeps_keys_local(+ range,+ required_exprs,+ eq_properties,+ allow_subset,+ ),+ Self::RoundRobinBatch(_)+ | Self::Hash(_, _)+ | Self::UnknownPartitioning(_) => false,+ }+ }+fn key_satisfaction(partition_exprs: &[Arc<dyn PhysicalExpr>],required_exprs: &[Arc<dyn PhysicalExpr>],@@ -991,7 +1220,8 @@ mod tests {use crate::projection::ProjectionTargets;use arrow::compute::SortOptions;- use arrow::datatypes::{DataType, Field, Schema, SchemaRef};+ use arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit};+ use datafusion_common::config::ConfigOptions;use datafusion_common::{Result, ScalarValue};struct PartitioningTestFixture {@@ -1090,6 +1320,17 @@ mod tests {) -> Partitioning {Partitioning::Range(self.range(indices, split_points))}++ fn range_partitioning_with_ordering(+ &self,+ ordering: LexOrdering,+ split_points: Vec<SplitPoint>,+ ) -> Partitioning {+ Partitioning::Range(+ RangePartitioning::try_new(ordering, split_points)+ .expect("test range partitioning should be valid"),+ )+ }}fn assert_satisfaction(@@ -1112,6 +1353,625 @@ mod tests {);}+ fn assert_key_locality(+ desc: &str,+ partitioning: &Partitioning,+ required_exprs: &[Arc<dyn PhysicalExpr>],+ eq_properties: &EquivalenceProperties,+ expected_with_subset: bool,+ expected_without_subset: bool,+ ) {+ assert_eq!(+ partitioning.keeps_keys_local(required_exprs, eq_properties, true),+ expected_with_subset,+ "Failed for {desc} with subset enabled"+ );+ assert_eq!(+ partitioning.keeps_keys_local(required_exprs, eq_properties, false),+ expected_without_subset,+ "Failed for {desc} with subset disabled"+ );+ }++ fn date_trunc_of(+ timestamp: Arc<dyn PhysicalExpr>,+ precision: &str,+ timezone: Option<&str>,+ ) -> Arc<dyn PhysicalExpr> {+ let timezone = timezone.map(Into::into);+ Arc::new(ScalarFunctionExpr::new(+ "date_trunc",+ datafusion_functions::datetime::date_trunc(),+ vec![+ Arc::new(Literal::new(ScalarValue::Utf8(Some(precision.into())))),+ timestamp,+ ],+ Field::new(+ "date_trunc",+ DataType::Timestamp(TimeUnit::Nanosecond, timezone),+ true,+ )+ .into(),+ Arc::new(ConfigOptions::default()),+ ))+ }++ fn timestamp_split(value: Option<i64>, timezone: Option<&str>) -> SplitPoint {+ SplitPoint::new(vec![ScalarValue::TimestampNanosecond(+ value,+ timezone.map(Into::into),+ )])+ }++ /// Assigns sample rows to source ranges and evaluates whole arrays,+ /// independently of the optimizer's boundary-adjacency proof.+ fn date_trunc_partition_images_are_disjoint<+ T: arrow::array::types::ArrowTimestampType,+ >(+ function: &Arc<dyn PhysicalExpr>,+ splits: &[i64],+ options: SortOptions,+ schema: &SchemaRef,+ ) -> Result<bool> {+ use arrow::array::{PrimitiveArray, RecordBatch};+ use datafusion_common::HashSet;++ let mut partitions = vec![vec![]; splits.len() + 1];+ let rows = splits+ .iter()+ .flat_map(|split| [Some(split - 1), Some(*split), Some(split + 1)])+ .chain([None, None]);+ for value in rows {+ let index = match value {+ None if options.nulls_first => 0,+ None => splits.len(),+ Some(value) => splits+ .iter()+ .filter(|split| {+ if options.descending {+ value <= **split+ } else {+ value >= **split+ }+ })+ .count(),+ };+ partitions[index].push(value);+ }+ let images = partitions+ .into_iter()+ .map(|values| {+ let input = Arc::new(PrimitiveArray::<T>::from_iter(values));+ let batch = RecordBatch::try_new(Arc::clone(schema), vec![input])?;+ let output = function.evaluate(&batch)?.into_array(batch.num_rows())?;+ (0..output.len())+ .map(|i| ScalarValue::try_from_array(&output, i))+ .collect::<Result<HashSet<_>>>()+ })+ .collect::<Result<Vec<_>>>()?;++ Ok(images.iter().enumerate().all(|(i, left)| {+ images+ .iter()+ .skip(i + 1)+ .all(|right| left.is_disjoint(right))+ }))+ }++ fn assert_date_trunc_partition_images<T: arrow::array::types::ArrowTimestampType>()+ -> Result<()> {+ let ticks_per_second = match T::UNIT {+ TimeUnit::Second => 1,+ TimeUnit::Millisecond => 1_000,+ TimeUnit::Microsecond => 1_000_000,+ TimeUnit::Nanosecond => 1_000_000_000,+ };+ let hour = 3600 * ticks_per_second;+ let data_type = DataType::Timestamp(T::UNIT, None);+ let schema =+ Arc::new(Schema::new(vec![Field::new("ts", data_type.clone(), true)]));+ let timestamp: Arc<dyn PhysicalExpr> = Arc::new(Column::new("ts", 0));+ let properties = EquivalenceProperties::new(Arc::clone(&schema));++ let mut cases = vec![+ (+ "aligned ascending hours",+ "hour",+ false,+ vec![-hour, 0, hour],+ true,+ ),+ (+ "unaligned ascending hours",+ "hour",+ false,+ vec![-hour + 1, 1, hour + 1],+ false,+ ),+ (+ "days cross hourly splits",+ "day",+ false,+ vec![-hour, 0, hour],+ false,+ ),+ (+ "descending splits inside hour buckets",+ "hour",+ true,+ vec![-hour, 0, hour],+ false,+ ),+ (+ "descending splits between hour buckets",+ "hour",+ true,+ vec![-hour - 1, -1, hour - 1],+ true,+ ),+ ];+ for (precision, divisor) in [("millisecond", 1_000), ("microsecond", 1_000_000)] {+ let width = (ticks_per_second / divisor).max(1);+ cases.extend([+ (+ "aligned ascending subsecond splits",+ precision,+ false,+ vec![-width, 0, width],+ true,+ ),+ // In particular, microsecond input with millisecond precision+ // must reject split -999: both adjacent array values map to -1000.+ (+ "offset ascending subsecond split",+ precision,+ false,+ vec![-width + 1],+ width == 1,+ ),+ (+ "descending splits at subsecond bucket starts",+ precision,+ true,+ vec![-width, 0, width],+ width == 1,+ ),+ (+ "descending splits at subsecond bucket ends",+ precision,+ true,+ vec![-width - 1, -1, width - 1],+ true,+ ),+ ]);+ }+ for (description, precision, descending, source_splits, expect_disjoint) in cases+ {+ for nulls_first in [false, true] {+ let options = SortOptions::new(descending, nulls_first);+ let mut splits = source_splits.clone();+ if descending {+ splits.reverse();+ }+ let range = Partitioning::Range(RangePartitioning::try_new(+ [PhysicalSortExpr::new(Arc::clone(×tamp), options)].into(),+ splits+ .iter()+ .map(|value| {+ SplitPoint::new(vec![ScalarValue::new_timestamp::<T>(+ Some(*value),+ None,+ )])+ })+ .collect(),+ )?);+ let function: Arc<dyn PhysicalExpr> = Arc::new(ScalarFunctionExpr::new(+ "date_trunc",+ datafusion_functions::datetime::date_trunc(),+ vec![+ Arc::new(Literal::new(ScalarValue::from(precision))),+ Arc::clone(×tamp),+ ],+ Field::new("bucket", data_type.clone(), true).into(),+ Arc::new(ConfigOptions::default()),+ ));+ assert_eq!(+ range.keeps_keys_local(&[Arc::clone(&function)], &properties, false),+ expect_disjoint,+ "optimizer: {description}, {precision}, {:?}, {options:?}, splits={splits:?}",+ T::UNIT+ );+ assert_eq!(+ date_trunc_partition_images_are_disjoint::<T>(+ &function, &splits, options, &schema,+ )?,+ expect_disjoint,+ "array evaluation: {description}, {precision}, {:?}, {options:?}, splits={splits:?}",+ T::UNIT+ );+ }+ }+ Ok(())+ }++ #[test]+ fn date_trunc_partition_images_are_disjoint_only_for_safe_splits() -> Result<()> {+ use arrow::array::types::{+ TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType,+ TimestampSecondType,+ };+ assert_date_trunc_partition_images::<TimestampSecondType>()?;+ assert_date_trunc_partition_images::<TimestampMillisecondType>()?;+ assert_date_trunc_partition_images::<TimestampMicrosecondType>()?;+ assert_date_trunc_partition_images::<TimestampNanosecondType>()+ }++ #[test]+ fn range_transform_requires_literal_parameters() -> Result<()> {+ use crate::expressions::CastExpr;+ let fixture = PartitioningTestFixture::new(vec![(+ "ts",+ DataType::Timestamp(TimeUnit::Nanosecond, None),+ )])?;+ let function = date_trunc_of(fixture.col(0), "hour", None);+ let scalar_but_not_literal: Arc<dyn PhysicalExpr> = Arc::new(CastExpr::new(+ Arc::new(Literal::new(ScalarValue::from("hour"))),+ DataType::Utf8View,+ None,+ ));+ let function =+ function.with_new_children(vec![scalar_but_not_literal, fixture.col(0)])?;+ let range = fixture.range_partitioning([0], vec![timestamp_split(Some(0), None)]);+ assert!(!range.keeps_keys_local(&[function], &fixture.eq_properties, false));+ Ok(())+ }++ #[test]+ fn range_transform_establishes_key_locality_only() -> Result<()> {+ let fixture = PartitioningTestFixture::new(vec![+ ("timestamp", DataType::Timestamp(TimeUnit::Nanosecond, None)),+ ("tag", DataType::Utf8),+ ])?;+ let hour = 1_704_070_800_000_000_000;+ let aligned =+ fixture.range_partitioning([0], vec![timestamp_split(Some(hour), None)]);+ let trunc_hour = date_trunc_of(fixture.col(0), "hour", None);++ let exact_key_exprs = vec![Arc::clone(&trunc_hour)];+ assert_eq!(+ aligned.satisfaction(+ &Distribution::KeyPartitioned(exact_key_exprs.clone()),+ &fixture.eq_properties,+ false,+ ),+ PartitioningSatisfaction::NotSatisfied+ );+ assert_key_locality(+ "aligned transformed key",+ &aligned,+ &exact_key_exprs,+ &fixture.eq_properties,+ true,+ true,+ );++ assert_key_locality(+ "additional grouping keys require subset permission",+ &aligned,+ &[trunc_hour, fixture.col(1)],+ &fixture.eq_properties,+ true,+ false,+ );++ let trunc_day = date_trunc_of(fixture.col(0), "day", None);+ assert_key_locality(+ "an hour split straddles a day bucket",+ &aligned,+ &[trunc_day],+ &fixture.eq_properties,+ false,+ false,+ );++ let two_aligned = fixture.range_partitioning(+ [0],+ vec![+ timestamp_split(Some(hour), None),+ timestamp_split(Some(hour + 3_600_000_000_000), None),+ ],+ );+ assert_key_locality(+ "every split must be aligned",+ &two_aligned,+ &exact_key_exprs,+ &fixture.eq_properties,+ true,+ true,+ );++ let unaligned =+ fixture.range_partitioning([0], vec![timestamp_split(Some(hour + 1), None)]);+ assert_key_locality(+ "an unaligned split straddles an hour bucket",+ &unaligned,+ &exact_key_exprs,+ &fixture.eq_properties,+ false,+ false,+ );+ Ok(())+ }++ #[test]+ fn range_transform_rejects_compound_range_keys() -> Result<()> {+ let fixture = PartitioningTestFixture::new(vec![+ ("key", DataType::Utf8),+ ("timestamp", DataType::Timestamp(TimeUnit::Nanosecond, None)),+ ])?;+ let split = SplitPoint::new(vec![+ ScalarValue::Utf8(Some("m".into())),+ ScalarValue::TimestampNanosecond(Some(1_704_070_800_000_000_000), None),+ ]);+ let range = fixture.range_partitioning([0, 1], vec![split]);++ // Start with single-key ranges; compound transforms can follow once+ // tuple-boundary safety is established.+ assert!(!range.keeps_keys_local(+ &[fixture.col(0), date_trunc_of(fixture.col(1), "hour", None),],+ &fixture.eq_properties,+ false,+ ));+ Ok(())+ }++ #[test]+ #[expect(+ deprecated,+ reason = "test intentionally constructs malformed range splits"+ )]+ fn range_transform_rejects_malformed_split_points() -> Result<()> {+ let fixture = PartitioningTestFixture::new(vec![(+ "timestamp",+ DataType::Timestamp(TimeUnit::Nanosecond, None),+ )])?;+ let required = vec![date_trunc_of(fixture.col(0), "hour", None)];+ let hour = 3_600_000_000_000;+ let timestamp = |value| ScalarValue::TimestampNanosecond(Some(value), None);++ for (description, descending, splits) in [+ (+ "reversed ascending splits",+ false,+ vec![vec![timestamp(2 * hour)], vec![timestamp(hour)]],+ ),+ (+ "reversed descending splits",+ true,+ vec![vec![timestamp(hour - 1)], vec![timestamp(2 * hour - 1)]],+ ),+ (+ "duplicate ascending splits",+ false,+ vec![vec![timestamp(hour)], vec![timestamp(hour)]],+ ),+ (+ "duplicate descending splits",+ true,+ vec![vec![timestamp(hour - 1)], vec![timestamp(hour - 1)]],+ ),+ (+ "overwide split tuple",+ false,+ vec![vec![timestamp(hour), ScalarValue::Int64(Some(0))]],+ ),+ ("empty split tuple", false, vec![vec![]]),+ ] {+ let range = Partitioning::Range(RangePartitioning::new(+ [PhysicalSortExpr::new(+ fixture.col(0),+ SortOptions::new(descending, false),+ )]+ .into(),+ splits.into_iter().map(SplitPoint::new).collect(),+ ));+ assert_key_locality(+ description,+ &range,+ &required,+ &fixture.eq_properties,+ false,+ false,+ );+ }+ Ok(())+ }++ #[test]+ fn range_transform_rejects_null_or_minimum_timestamp_splits() -> Result<()> {+ let fixture = PartitioningTestFixture::new(vec![(+ "timestamp",+ DataType::Timestamp(TimeUnit::Nanosecond, None),+ )])?;+ let required = vec![date_trunc_of(fixture.col(0), "hour", None)];++ for (description, split) in [+ ("null", timestamp_split(None, None)),+ (+ "minimum has no predecessor",+ timestamp_split(Some(i64::MIN), None),+ ),+ ] {+ assert_key_locality(+ description,+ &fixture.range_partitioning([0], vec![split]),+ &required,+ &fixture.eq_properties,+ false,+ false,+ );+ }+ Ok(())+ }++ #[test]+ #[expect(+ deprecated,+ reason = "test intentionally constructs a malformed range split"+ )]+ fn range_transform_rejects_split_type_mismatch() -> Result<()> {+ let fixture = PartitioningTestFixture::new(vec![(+ "timestamp",+ DataType::Timestamp(TimeUnit::Nanosecond, None),+ )])?;+ let required = vec![date_trunc_of(fixture.col(0), "hour", None)];+ let mismatched_split = Partitioning::Range(RangePartitioning::new(+ fixture.range_ordering([0]),+ vec![SplitPoint::new(vec![ScalarValue::Int64(Some(10))])],+ ));+ assert_key_locality(+ "split scalar type must match the range key",+ &mismatched_split,+ &required,+ &fixture.eq_properties,+ false,+ false,+ );+ Ok(())+ }++ #[test]+ fn range_transform_rejects_boundary_evaluation_errors() -> Result<()> {+ let fixture = PartitioningTestFixture::new(vec![(+ "timestamp",+ DataType::Timestamp(TimeUnit::Nanosecond, None),+ )])?;+ let invalid_precision =+ vec![date_trunc_of(fixture.col(0), "not-a-precision", None)];+ let aligned = fixture.range_partitioning(+ [0],+ vec![timestamp_split(Some(1_704_070_800_000_000_000), None)],+ );+ assert_key_locality(+ "split evaluation errors fail closed",+ &aligned,+ &invalid_precision,+ &fixture.eq_properties,+ false,+ false,+ );+ Ok(())+ }++ #[test]+ fn range_transform_rejects_adjacent_timestamp_evaluation_errors() -> Result<()> {+ let fixture = PartitioningTestFixture::new(vec![(+ "timestamp",+ DataType::Timestamp(TimeUnit::Nanosecond, None),+ )])?;+ let nanoseconds_per_hour = 3_600_000_000_000;+ let first_representable_hour =+ i64::MIN / nanoseconds_per_hour * nanoseconds_per_hour;+ let split =+ ScalarValue::TimestampNanosecond(Some(first_representable_hour), None);+ let predecessor =+ ScalarValue::TimestampNanosecond(Some(first_representable_hour - 1), None);+ let timestamp = fixture.col(0);+ let trunc_hour = date_trunc_of(Arc::clone(×tamp), "hour", None);++ assert_eq!(+ evaluate_with_range_value(&trunc_hour, ×tamp, &split),+ Some(split.clone()),+ );+ assert_eq!(+ evaluate_with_range_value(&trunc_hour, ×tamp, &predecessor),+ None,+ );++ let range = fixture.range_partitioning([0], vec![SplitPoint::new(vec![split])]);+ assert_key_locality(+ "predecessor truncation underflows even though split truncation succeeds",+ &range,+ &[trunc_hour],+ &fixture.eq_properties,+ false,+ false,+ );+ Ok(())+ }++ #[test]+ fn range_transform_rejects_timezone_annotated_timestamps() -> Result<()> {+ let fixture = PartitioningTestFixture::new(vec![(+ "timestamp",+ DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())),+ )])?;+ let trunc_hour = date_trunc_of(fixture.col(0), "hour", Some("UTC"));+ let range = fixture.range_partitioning(+ [0],+ vec![timestamp_split(+ Some(1_704_070_800_000_000_000),+ Some("UTC"),+ )],+ );+ assert_key_locality(+ "timezone-aware date_trunc is not audited",+ &range,+ &[trunc_hour],+ &fixture.eq_properties,+ false,+ false,+ );+ Ok(())+ }++ #[test]+ fn descending_range_uses_the_open_side_successor() -> Result<()> {+ let fixture = PartitioningTestFixture::new(vec![(+ "timestamp",+ DataType::Timestamp(TimeUnit::Nanosecond, None),+ )])?;+ let hour = 1_704_070_800_000_000_000;+ let ordering: LexOrdering =+ [fixture.range_sort_expr(0, SortOptions::new(true, false))].into();+ let required = vec![date_trunc_of(fixture.col(0), "hour", None)];++ assert_key_locality(+ "the successor of an hour boundary is in the same bucket",+ &fixture.range_partitioning_with_ordering(+ ordering.clone(),+ vec![timestamp_split(Some(hour), None)],+ ),+ &required,+ &fixture.eq_properties,+ false,+ false,+ );+ assert_key_locality(+ "the successor of the final nanosecond starts a new bucket",+ &fixture.range_partitioning_with_ordering(+ ordering.clone(),+ vec![timestamp_split(Some(hour + 3_600_000_000_000 - 1), None)],+ ),+ &required,+ &fixture.eq_properties,+ true,+ true,+ );+ assert_key_locality(+ "the maximum timestamp has no successor",+ &fixture.range_partitioning_with_ordering(+ ordering,+ vec![timestamp_split(Some(i64::MAX), None)],+ ),+ &required,+ &fixture.eq_properties,+ false,+ false,+ );+ Ok(())+ }+#[test]#[expect(deprecated,diff --git a/datafusion/physical-expr/src/scalar_function.rs b/datafusion/physical-expr/src/scalar_function.rsindex f7bd5469cd..1d9bc2f3fd 100644--- a/datafusion/physical-expr/src/scalar_function.rs+++ b/datafusion/physical-expr/src/scalar_function.rs@@ -158,6 +158,25 @@ impl ScalarFunctionExpr {&self.config_options}+ /// Returns whether this function expression's argument types have been+ /// explicitly audited as candidates for range-key locality analysis.+ ///+ /// See [`ScalarUDFImpl::supports_range_key_locality_analysis`] for the safety+ /// requirements.+ pub(crate) fn supports_range_key_locality_analysis(+ &self,+ input_schema: &Schema,+ ) -> Result<bool> {+ let argument_types = self+ .args+ .iter()+ .map(|argument| argument.data_type(input_schema))+ .collect::<Result<Vec<_>>>()?;+ Ok(self+ .fun+ .supports_range_key_locality_analysis(&argument_types))+ }+/// Given an arbitrary PhysicalExpr attempt to downcast it to a ScalarFunctionExpr/// and verify that its inner function is of type T./// If the downcast fails, or the function is not of type T, returns `None`.diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rsindex 49776d8843..168eef838d 100644--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs@@ -915,6 +915,20 @@ fn partial_aggregate_output_satisfies_final_partitioning(.is_satisfied()}+fn partial_aggregate_input_keeps_groups_local(+ plan: &Arc<dyn ExecutionPlan>,+ allow_subset_satisfy_partitioning: bool,+) -> bool {+ plan.downcast_ref::<AggregateExec>()+ .is_some_and(|aggregate| {+ aggregate.mode() == &AggregateMode::Partial+ && aggregate.input_keeps_groups_local(+ aggregate.input().as_ref(),+ allow_subset_satisfy_partitioning,+ )+ })+}+/// Adds a [`SortPreservingMergeExec`] or a [`CoalescePartitionsExec`] operator/// on top of the given plan node to satisfy a single partition requirement/// while preserving ordering constraints.@@ -1686,10 +1700,13 @@ pub fn ensure_distribution_with_stats(let preserve_partial_aggregate_partitioning =preserve_file_partition_threshold_met- && partial_aggregate_output_satisfies_final_partitioning(+ && (partial_aggregate_output_satisfies_final_partitioning(+ &plan,+ allow_subset_satisfy_partitioning,+ ) || partial_aggregate_input_keeps_groups_local(&plan,allow_subset_satisfy_partitioning,- );+ ));let add_roundrobin = enable_round_robin// Operator benefits from partitioning (e.g. filter):@@ -1721,6 +1738,9 @@ pub fn ensure_distribution_with_stats(Distribution::HashPartitioned(_) | Distribution::KeyPartitioned(_) => {let child_partitions =child.plan.output_partitioning().partition_count();+ // An aggregate can avoid repartitioning if all rows for each group+ // are already in one input partition, even when the ordinary+ // distribution check fails (#25344). This does not prove join co-partitioning.let partitioning_satisfied = input_distributions.child_satisfaction(child_idx,@@ -1728,7 +1748,15 @@ pub fn ensure_distribution_with_stats(ChildSatisfactionOptions::new().with_allow_subset(allow_subset_satisfy_partitioning),)?- .is_satisfied();+ .is_satisfied()+ || plan.downcast_ref::<AggregateExec>().is_some_and(+ |aggregate| {+ aggregate.input_keeps_groups_local(+ child.plan.as_ref(),+ allow_subset_satisfy_partitioning,+ )+ },+ );let preserve_satisfying_file_partitioning =preserve_file_partition_threshold_met&& !requires_grouping_iddiff --git a/datafusion/physical-optimizer/src/sanity_checker.rs b/datafusion/physical-optimizer/src/sanity_checker.rsindex 713213b706..746746761b 100644--- a/datafusion/physical-optimizer/src/sanity_checker.rs+++ b/datafusion/physical-optimizer/src/sanity_checker.rs@@ -30,6 +30,7 @@ use datafusion_common::config::{ConfigOptions, OptimizerOptions};use datafusion_common::plan_err;use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};use datafusion_physical_expr::intervals::utils::{check_support, is_datatype_supported};+use datafusion_physical_plan::aggregates::AggregateExec;use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType, InvariantLevel,};@@ -167,14 +168,17 @@ pub fn check_plan_sanity(}}- if !input_distributions- .child_satisfaction(- idx,- child.as_ref(),- ChildSatisfactionOptions::new().with_allow_subset(true),- )?- .is_satisfied()- {+ let distribution_satisfaction = input_distributions.child_satisfaction(+ idx,+ child.as_ref(),+ ChildSatisfactionOptions::new().with_allow_subset(true),+ )?;+ let accepts_partition_local_groups = plan+ .downcast_ref::<AggregateExec>()+ .is_some_and(|aggregate| {+ aggregate.input_keeps_groups_local(child.as_ref(), true)+ });+ if !distribution_satisfaction.is_satisfied() && !accepts_partition_local_groups {let plan_str = get_plan_string(plan);return plan_err!("Plan: {:?} does not satisfy distribution requirements: {}. Child-{} output partitioning: {}",diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rsindex 7957c804d5..226ce3d271 100644--- a/datafusion/physical-plan/src/aggregates/mod.rs+++ b/datafusion/physical-plan/src/aggregates/mod.rs@@ -1654,6 +1654,52 @@ impl AggregateExec {&self.input}+ /// Whether a candidate input keeps each group in one partition.+ ///+ /// Unlike co-partitioning, aggregation only needs equality groups to be+ /// local. A final stage can inherit that guarantee from a partial stage+ /// even when projecting the grouping expressions loses range metadata.+ /// This check does not require the two stages to be combined.+ pub fn input_keeps_groups_local(+ &self,+ input: &dyn ExecutionPlan,+ allow_subset: bool,+ ) -> bool {+ if !matches!(+ self.mode,+ AggregateMode::Partial+ | AggregateMode::SinglePartitioned+ | AggregateMode::FinalPartitioned+ ) || self.group_by().is_empty()+ || self.group_by().has_grouping_set()+ {+ return false;+ }++ if input.output_partitioning().keeps_keys_local(+ &self.group_by().input_exprs(),+ input.equivalence_properties(),+ allow_subset,+ ) {+ return true;+ }++ self.mode == AggregateMode::FinalPartitioned+ && input+ .downcast_ref::<AggregateExec>()+ .is_some_and(|partial| {+ partial.mode == AggregateMode::Partial+ && datafusion_physical_expr::physical_exprs_equal(+ &partial.output_group_expr(),+ &self.group_by().input_exprs(),+ )+ && partial.input_keeps_groups_local(+ partial.input.as_ref(),+ allow_subset,+ )+ })+ }+/// Get the input schema before any aggregates are appliedpub fn input_schema(&self) -> SchemaRef {Arc::clone(&self.input_schema)diff --git a/datafusion/sqllogictest/src/test_context/range_partitioning.rs b/datafusion/sqllogictest/src/test_context/range_partitioning.rsindex 2cb4f57e29..4989294146 100644--- a/datafusion/sqllogictest/src/test_context/range_partitioning.rs+++ b/datafusion/sqllogictest/src/test_context/range_partitioning.rs@@ -423,9 +423,9 @@ type TimeBinRow = (/// - partition 1: `[2024-01-01 01:00, 02:00)`////// Files are range-partitioned on `timestamp` and sorted on `(key, timestamp)`.-/// Because `date_bin(60 seconds, timestamp)` does not straddle the hour split,-/// grouping by `(key, time_bin)` is partition-disjoint. Today's planner still-/// inserts a hash shuffle; the test pins that plan so a follow-up can remove it.+/// The hour split permits partition-local `date_trunc('hour')` aggregation,+/// but `date_trunc('day')` straddles it. Aligned `date_bin` remains a negative+/// control because the function is not admitted to range analysis.pub(super) fn register_range_sorted_time_bin_table(ctx: &SessionContext) {let schema = Arc::new(Schema::new(vec![Field::new("key", DataType::Utf8, false),diff --git a/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt b/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.sltindex 400bbab1a3..e8c656450b 100644--- a/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt+++ b/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt@@ -30,11 +30,16 @@# 2. Two file_groups, so the two 60-minute streams run in parallel## Improvement opportunity:-# date_bin(60s) is monotonic in timestamp and the hour split is aligned to bin-# boundaries, so (key, time_bin) is partition-disjoint. Aggregation could be a-# single streaming SinglePartitioned step with no hash shuffle.+# In this fixture, date_bin(60s) is monotonic in timestamp and the hour split+# is aligned to bin boundaries, so (key, time_bin) is partition-disjoint.+# Aggregation could be a single streaming SinglePartitioned step with no+# hash shuffle.#-# Today's plan still hash-repartitions:+# This optimization exists for timezone-less date_trunc when every split+# keeps groups partition-local. Extending it to date_bin is future work+# requiring a safety audit, including its null behavior (#25344).+#+# Today's date_bin plan still hash-repartitions:# Partial AggregateExec (group_clustering_mode=Full)# -> RepartitionExec Hash([key, date_bin(...)])# -> FinalPartitioned AggregateExec (group_clustering_mode=Full)@@ -82,11 +87,11 @@ physical_plan DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion########### TEST 2: Filtered time-bin aggregation.# GROUP BY keys are (key, date_bin(timestamp)). Input is sorted on those keys-# (date_bin is monotonic in timestamp) and range-partitioned so bins do not-# overlap across the two 60-minute streams.-#-# Today this is still Partial + hash RepartitionExec + Final, even though-# group_clustering_mode=Full is already recognized.+# and the split aligns with the bins in this data, so the groups are+# partition-local. Timezone-less date_trunc can already avoid the shuffle;+# extending this optimization to date_bin awaits its safety audit (#25344).+# Until then, Partial + hash RepartitionExec + Final is retained even though+# group_clustering_mode=Full is recognized.##########query TT@@ -117,8 +122,9 @@ k2 2024-01-01T01:30:00 30k2 2024-01-01T01:45:00 5##########-# TEST 3: Same aggregation without the col4 filter. The scan still has two-# 60-minute file groups, and today's plan still hash-repartitions.+# TEST 3: Same aggregation without the col4 filter. The groups remain+# partition-local, but date_bin retains the hash shuffle pending its+# safety audit (#25344).##########query TT@@ -146,6 +152,97 @@ k2 2024-01-01T00:30:00 7k2 2024-01-01T01:30:00 30k2 2024-01-01T01:45:00 5+##########+# TEST 4: date_trunc('hour') is audited and the split is aligned, so each+# group is partition-local and aggregation needs no hash shuffle.+##########++query TT+EXPLAIN SELECT key, date_trunc('hour', timestamp) AS time_bin, sum(value)+FROM range_sorted_time_bin+WHERE col4 = 'a'+GROUP BY key, time_bin;+----+physical_plan+01)ProjectionExec: expr=[key@0 as key, date_trunc(Utf8("hour"),range_sorted_time_bin.timestamp)@1 as time_bin, sum(range_sorted_time_bin.value)@2 as sum(range_sorted_time_bin.value)]+02)--AggregateExec: mode=SinglePartitioned, gby=[key@0 as key, date_trunc(hour, timestamp@1) as date_trunc(Utf8("hour"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full+03)----FilterExec: col4@1 = a, projection=[key@0, timestamp@2, value@3]+04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[key, col4, timestamp, value], output_ordering=[key@0 ASC, timestamp@2 ASC], output_partitioning=Range([timestamp@2 ASC], [(1704070800000000000)], 2), file_type=parquet, predicate=col4@4 = a, pruning_predicate=col4_null_count@2 != row_count@3 AND col4_min@0 <= a AND a <= col4_max@1, required_guarantees=[col4 in (a)]++query TPI+SELECT key, date_trunc('hour', timestamp) AS time_bin, sum(value)+FROM range_sorted_time_bin+WHERE col4 = 'a'+GROUP BY key, time_bin+ORDER BY key, time_bin;+----+k1 2024-01-01T00:00:00 3+k1 2024-01-01T01:00:00 30+k2 2024-01-01T00:00:00 7+k2 2024-01-01T01:00:00 35++##########+# TEST 5: date_trunc('day') straddles the hour split, so the hash shuffle+# remains.+##########++query TT+EXPLAIN SELECT key, date_trunc('day', timestamp) AS time_bin, sum(value)+FROM range_sorted_time_bin+WHERE col4 = 'a'+GROUP BY key, time_bin;+----+physical_plan+01)ProjectionExec: expr=[key@0 as key, date_trunc(Utf8("day"),range_sorted_time_bin.timestamp)@1 as time_bin, sum(range_sorted_time_bin.value)@2 as sum(range_sorted_time_bin.value)]+02)--AggregateExec: mode=FinalPartitioned, gby=[key@0 as key, date_trunc(Utf8("day"),range_sorted_time_bin.timestamp)@1 as date_trunc(Utf8("day"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full+03)----RepartitionExec: partitioning=Hash([key@0, date_trunc(Utf8("day"),range_sorted_time_bin.timestamp)@1], 2), input_partitions=2, preserve_order=true, sort_exprs=key@0 ASC, date_trunc(Utf8("day"),range_sorted_time_bin.timestamp)@1 ASC+04)------AggregateExec: mode=Partial, gby=[key@0 as key, date_trunc(day, timestamp@1) as date_trunc(Utf8("day"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full+05)--------FilterExec: col4@1 = a, projection=[key@0, timestamp@2, value@3]+06)----------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[key, col4, timestamp, value], output_ordering=[key@0 ASC, timestamp@2 ASC], output_partitioning=Range([timestamp@2 ASC], [(1704070800000000000)], 2), file_type=parquet, predicate=col4@4 = a, pruning_predicate=col4_null_count@2 != row_count@3 AND col4_min@0 <= a AND a <= col4_max@1, required_guarantees=[col4 in (a)]++query TPI+SELECT key, date_trunc('day', timestamp) AS time_bin, sum(value)+FROM range_sorted_time_bin+WHERE col4 = 'a'+GROUP BY key, time_bin+ORDER BY key, time_bin;+----+k1 2024-01-01T00:00:00 33+k2 2024-01-01T00:00:00 42++##########+# TEST 6: Grouping sets can omit the range transform. The per-key subtotal+# crosses the hour split, so it still needs a hash shuffle.+##########++query TT+EXPLAIN SELECT key, date_trunc('hour', timestamp) AS time_bin, sum(value)+FROM range_sorted_time_bin+WHERE col4 = 'a'+GROUP BY GROUPING SETS ((key, time_bin), (key));+----+physical_plan+01)ProjectionExec: expr=[key@0 as key, date_trunc(Utf8("hour"),range_sorted_time_bin.timestamp)@1 as time_bin, sum(range_sorted_time_bin.value)@3 as sum(range_sorted_time_bin.value)]+02)--AggregateExec: mode=FinalPartitioned, gby=[key@0 as key, date_trunc(Utf8("hour"),range_sorted_time_bin.timestamp)@1 as date_trunc(Utf8("hour"),range_sorted_time_bin.timestamp), __grouping_id@2 as __grouping_id], aggr=[sum(range_sorted_time_bin.value)]+03)----RepartitionExec: partitioning=Hash([key@0, date_trunc(Utf8("hour"),range_sorted_time_bin.timestamp)@1, __grouping_id@2], 2), input_partitions=2+04)------AggregateExec: mode=Partial, gby=[(key@0 as key, date_trunc(hour, timestamp@1) as date_trunc(Utf8("hour"),range_sorted_time_bin.timestamp)), (key@0 as key, NULL as date_trunc(Utf8("hour"),range_sorted_time_bin.timestamp))], aggr=[sum(range_sorted_time_bin.value)]+05)--------FilterExec: col4@1 = a, projection=[key@0, timestamp@2, value@3]+06)----------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[key, col4, timestamp, value], output_ordering=[key@0 ASC, timestamp@2 ASC], output_partitioning=Range([timestamp@2 ASC], [(1704070800000000000)], 2), file_type=parquet, predicate=col4@4 = a, pruning_predicate=col4_null_count@2 != row_count@3 AND col4_min@0 <= a AND a <= col4_max@1, required_guarantees=[col4 in (a)]++query TPI+SELECT key, date_trunc('hour', timestamp) AS time_bin, sum(value)+FROM range_sorted_time_bin+WHERE col4 = 'a'+GROUP BY GROUPING SETS ((key, time_bin), (key))+ORDER BY key, time_bin NULLS LAST;+----+k1 2024-01-01T00:00:00 3+k1 2024-01-01T01:00:00 30+k1 NULL 33+k2 2024-01-01T00:00:00 7+k2 2024-01-01T01:00:00 35+k2 NULL 42+########### CLEANUP##########