Apache DataFusion SQL Query Engine
feat: reuse range partitioning for audited date_bin aggregation (#26835)
Ben Bellickf1ecc4ad2ee5
diff --git a/datafusion/expr/src/udf.rs b/datafusion/expr/src/udf.rsindex 65c1341b6a..c3c9e461e3 100644--- a/datafusion/expr/src/udf.rs+++ b/datafusion/expr/src/udf.rs@@ -355,9 +355,10 @@ impl ScalarUDF {pub fn supports_range_key_locality_analysis(&self,argument_types: &[DataType],+ literal_arguments: &[Option<&ScalarValue>],) -> bool {self.inner- .supports_range_key_locality_analysis(argument_types)+ .supports_range_key_locality_analysis(argument_types, literal_arguments)}pub fn preserves_lex_ordering(&self, inputs: &[ExprProperties]) -> Result<bool> {@@ -979,8 +980,8 @@ pub trait ScalarUDFImpl: Debug + DynEq + DynHash + Send + Sync + Any {Ok(inner(inputs))}- /// Returns whether this function and argument-type combination has been- /// audited for range-partitioning locality analysis.+ /// Returns whether this function has been audited for range-key locality+ /// analysis with these argument types and literal values.////// Range-boundary analysis has subtle correctness requirements, so the/// policy is default-deny. Functions may opt in only after individual audit.@@ -998,7 +999,15 @@ pub trait ScalarUDFImpl: Debug + DynEq + DynHash + Send + Sync + Any {/// This does not claim a concrete partition layout or cross-input/// co-partitioning compatibility. Boundary evaluation errors cause the/// analysis to return `false`. This capability is not transported across FFI.- fn supports_range_key_locality_analysis(&self, _argument_types: &[DataType]) -> bool {+ ///+ /// `literal_arguments` has one entry per argument. `None` means the argument+ /// is not a literal; `Some(value)` supplies its value, which may itself be+ /// NULL. Non-literal expressions are not evaluated to obtain these values.+ fn supports_range_key_locality_analysis(+ &self,+ _argument_types: &[DataType],+ _literal_arguments: &[Option<&ScalarValue>],+ ) -> bool {false}@@ -1224,9 +1233,13 @@ impl ScalarUDFImpl for AliasedScalarUDFImpl {self.inner.output_ordering(inputs)}- fn supports_range_key_locality_analysis(&self, argument_types: &[DataType]) -> bool {+ fn supports_range_key_locality_analysis(+ &self,+ argument_types: &[DataType],+ literal_arguments: &[Option<&ScalarValue>],+ ) -> bool {self.inner- .supports_range_key_locality_analysis(argument_types)+ .supports_range_key_locality_analysis(argument_types, literal_arguments)}fn preserves_lex_ordering(&self, inputs: &[ExprProperties]) -> Result<bool> {@@ -1314,7 +1327,10 @@ mod tests {#[test]fn range_key_locality_analysis_is_disabled_by_default() {let function = test_func("range_function", "a");- assert!(!function.supports_range_key_locality_analysis(&[DataType::Int64]));+ assert!(!function.supports_range_key_locality_analysis(+ &[DataType::Int64],+ &[Some(&ScalarValue::Int64(Some(1)))],+ ));}fn test_func(name: &'static str, parameter: &'static str) -> ScalarUDF {diff --git a/datafusion/functions/src/datetime/date_bin.rs b/datafusion/functions/src/datetime/date_bin.rsindex 98a05d9212..3f72f444f0 100644--- a/datafusion/functions/src/datetime/date_bin.rs+++ b/datafusion/functions/src/datetime/date_bin.rs@@ -276,6 +276,34 @@ impl ScalarUDFImpl for DateBinFunc {}}+ fn supports_range_key_locality_analysis(+ &self,+ argument_types: &[DataType],+ literal_arguments: &[Option<&ScalarValue>],+ ) -> bool {+ // With the epoch origin, a positive fixed-duration stride rounds down+ // monotonically. As in date_bin_impl, each day is 24 hours.+ // Timestamp binning preserves NULLs or returns an error.+ match (argument_types, literal_arguments) {+ (+ [+ DataType::Interval(MonthDayNano),+ Timestamp(Nanosecond, None),+ ],+ [Some(ScalarValue::IntervalMonthDayNano(Some(stride))), None],+ ) => {+ stride.months == 0+ && TimeDelta::try_days(i64::from(stride.days))+ .and_then(|days| {+ days.checked_add(&Duration::nanoseconds(stride.nanoseconds))+ })+ .and_then(|stride| stride.num_nanoseconds())+ .is_some_and(|stride| stride > 0)+ }+ _ => false,+ }+ }+fn documentation(&self) -> Option<&Documentation> {self.doc()}@@ -978,6 +1006,148 @@ mod tests {);}+ #[test]+ fn range_key_locality_requires_positive_fixed_duration_stride() {+ use arrow::datatypes::DataType::Timestamp;+ use arrow::datatypes::TimeUnit::Nanosecond;++ let function = DateBinFunc::new();+ let timestamp_type = Timestamp(Nanosecond, None);+ let cases = [+ (+ "one nanosecond",+ ScalarValue::new_interval_mdn(0, 0, 1),+ true,+ ),+ (+ "one hour",+ ScalarValue::new_interval_mdn(0, 0, 3_600_000_000_000),+ true,+ ),+ (+ "largest nanosecond stride",+ ScalarValue::new_interval_mdn(0, 0, i64::MAX),+ true,+ ),+ (+ "NULL stride",+ ScalarValue::IntervalMonthDayNano(None),+ false,+ ),+ ("zero stride", ScalarValue::new_interval_mdn(0, 0, 0), false),+ (+ "negative stride",+ ScalarValue::new_interval_mdn(0, 0, -1),+ false,+ ),+ (+ "month stride",+ ScalarValue::new_interval_mdn(1, 0, 0),+ false,+ ),+ ("day stride", ScalarValue::new_interval_mdn(0, 1, 0), true),+ (+ "mixed day and nanosecond stride",+ ScalarValue::new_interval_mdn(0, 1, 1),+ true,+ ),+ (+ "day minus one nanosecond",+ ScalarValue::new_interval_mdn(0, 1, -1),+ true,+ ),+ (+ "negative day plus one nanosecond",+ ScalarValue::new_interval_mdn(0, -1, 1),+ false,+ ),+ (+ "combined zero stride",+ ScalarValue::new_interval_mdn(0, 1, -86_400_000_000_000),+ false,+ ),+ (+ "mixed month and day stride",+ ScalarValue::new_interval_mdn(1, 1, 0),+ false,+ ),+ (+ "combined stride overflows nanoseconds",+ ScalarValue::new_interval_mdn(0, 1, i64::MAX),+ false,+ ),+ (+ "day-time stride",+ ScalarValue::new_interval_dt(0, 1000),+ false,+ ),+ ];+ for (description, stride, expected) in cases {+ assert_eq!(+ function.supports_range_key_locality_analysis(+ &[stride.data_type(), timestamp_type.clone()],+ &[Some(&stride), None],+ ),+ expected,+ "{description}",+ );+ }+ }++ #[test]+ fn range_key_locality_rejects_unsupported_source_types() {+ use arrow::datatypes::DataType::{Time64, Timestamp};+ use arrow::datatypes::TimeUnit::{Microsecond, Millisecond, Nanosecond, Second};++ let function = DateBinFunc::new();+ let stride = ScalarValue::new_interval_mdn(0, 0, 3_600_000_000_000);+ for source_type in [+ Timestamp(Second, None),+ Timestamp(Millisecond, None),+ Timestamp(Microsecond, None),+ Timestamp(Nanosecond, Some("UTC".into())),+ Time64(Nanosecond),+ ] {+ assert!(+ !function.supports_range_key_locality_analysis(+ &[stride.data_type(), source_type.clone()],+ &[Some(&stride), None],+ ),+ "source type {source_type:?}",+ );+ }+ }++ #[test]+ fn range_key_locality_rejects_non_literal_stride() {+ use arrow::datatypes::IntervalUnit::MonthDayNano;++ let function = DateBinFunc::new();+ let argument_types = [+ DataType::Interval(MonthDayNano),+ DataType::Timestamp(TimeUnit::Nanosecond, None),+ ];+ assert!(+ !function+ .supports_range_key_locality_analysis(&argument_types, &[None, None])+ );+ }++ #[test]+ fn range_key_locality_rejects_explicit_origin() {+ let function = DateBinFunc::new();+ let stride = ScalarValue::new_interval_mdn(0, 0, 3_600_000_000_000);+ let epoch = ScalarValue::TimestampNanosecond(Some(0), None);+ assert!(!function.supports_range_key_locality_analysis(+ &[+ stride.data_type(),+ DataType::Timestamp(TimeUnit::Nanosecond, None),+ epoch.data_type(),+ ],+ &[Some(&stride), None, Some(&epoch)],+ ));+ }+#[test]fn output_ordering_requires_constant_stride() {use arrow::compute::SortOptions;diff --git a/datafusion/functions/src/datetime/date_trunc.rs b/datafusion/functions/src/datetime/date_trunc.rsindex 0ca7270816..d5cf186bc1 100644--- a/datafusion/functions/src/datetime/date_trunc.rs+++ b/datafusion/functions/src/datetime/date_trunc.rs@@ -559,7 +559,11 @@ impl ScalarUDFImpl for DateTruncFunc {}}- fn supports_range_key_locality_analysis(&self, argument_types: &[DataType]) -> bool {+ fn supports_range_key_locality_analysis(+ &self,+ argument_types: &[DataType],+ _literal_arguments: &[Option<&ScalarValue>],+ ) -> bool {// Across an America/Goose_Bay transition, ascending epoch seconds// 562129259 < 562129260 truncate to 562129200 > 562125600.// Keep the audited domain to timezone-less timestamp columns.@@ -1400,19 +1404,30 @@ mod tests {fn range_key_locality_analysis_requires_timezone_less_timestamp() {let date_trunc = DateTruncFunc::new();- assert!(date_trunc.supports_range_key_locality_analysis(&[- DataType::Utf8,- DataType::Timestamp(TimeUnit::Nanosecond, None),- ]));- assert!(!date_trunc.supports_range_key_locality_analysis(&[- DataType::Utf8,- DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())),- ]));- assert!(!date_trunc.supports_range_key_locality_analysis(&[- DataType::Int64,- DataType::Timestamp(TimeUnit::Nanosecond, None),- ]));- assert!(!date_trunc.supports_range_key_locality_analysis(&[DataType::Utf8]));+ assert!(date_trunc.supports_range_key_locality_analysis(+ &[+ DataType::Utf8,+ DataType::Timestamp(TimeUnit::Nanosecond, None),+ ],+ &[None, None],+ ));+ assert!(!date_trunc.supports_range_key_locality_analysis(+ &[+ DataType::Utf8,+ DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())),+ ],+ &[None, None],+ ));+ assert!(!date_trunc.supports_range_key_locality_analysis(+ &[+ DataType::Int64,+ DataType::Timestamp(TimeUnit::Nanosecond, None),+ ],+ &[None, None],+ ));+ assert!(+ !date_trunc.supports_range_key_locality_analysis(&[DataType::Utf8], &[None])+ );}fn invoke_date_trunc(diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rsindex ad1f56b03a..e5177d39be 100644--- a/datafusion/physical-expr/src/partitioning.rs+++ b/datafusion/physical-expr/src/partitioning.rs@@ -1383,6 +1383,117 @@ mod tests {))}+ /// Builds `date_bin(stride, timestamp)` with a nanosecond-only literal stride+ /// and the default epoch origin. This constructs the expression without evaluating it.+ fn date_bin_of(+ timestamp: Arc<dyn PhysicalExpr>,+ stride_nanos: i64,+ ) -> Arc<dyn PhysicalExpr> {+ Arc::new(ScalarFunctionExpr::new(+ "date_bin",+ datafusion_functions::datetime::date_bin(),+ vec![+ Arc::new(Literal::new(ScalarValue::new_interval_mdn(+ 0,+ 0,+ stride_nanos,+ ))),+ timestamp,+ ],+ Field::new(+ "date_bin",+ DataType::Timestamp(TimeUnit::Nanosecond, None),+ true,+ )+ .into(),+ Arc::new(ConfigOptions::default()),+ ))+ }++ #[test]+ fn date_bin_requires_disjoint_bins_at_every_split() -> Result<()> {+ let fixture = PartitioningTestFixture::new(vec![(+ "ts",+ DataType::Timestamp(TimeUnit::Nanosecond, None),+ )])?;+ let hour = 3_600_000_000_000;+ let function = date_bin_of(fixture.col(0), hour);+ // Ascending compares split - 1 ns with split. Descending compares split+ // with split + 1 ns, so aligned descending splits end a bin rather than start it.+ let cases = [+ ("ascending aligned", false, vec![-hour, 0, hour], true),+ ("ascending crossing", false, vec![-hour, 1, hour], false),+ (+ "descending aligned",+ true,+ vec![hour - 1, -1, -hour - 1],+ true,+ ),+ ("descending crossing", true, vec![hour, 0, -hour], false),+ ];+ for (description, descending, splits, expected) in cases {+ for nulls_first in [false, true] {+ let range = fixture.range_partitioning_with_ordering(+ [fixture+ .range_sort_expr(0, SortOptions::new(descending, nulls_first))]+ .into(),+ splits+ .iter()+ .map(|value| timestamp_split(Some(*value), None))+ .collect(),+ );+ assert_eq!(+ range.keeps_keys_local(+ &[Arc::clone(&function)],+ &fixture.eq_properties,+ false,+ ),+ expected,+ "{description}, nulls_first={nulls_first}",+ );+ }+ }+ Ok(())+ }++ #[test]+ fn date_bin_rejects_adjacent_timestamp_binning_errors() -> Result<()> {+ let fixture = PartitioningTestFixture::new(vec![(+ "ts",+ DataType::Timestamp(TimeUnit::Nanosecond, None),+ )])?;+ let hour = 3_600_000_000_000;+ // The hour containing i64::MIN starts outside the timestamp range.+ let nanos_to_next_hour = hour - i64::MIN.rem_euclid(hour);+ let first_representable_hour = i64::MIN + nanos_to_next_hour;+ let at_split =+ ScalarValue::TimestampNanosecond(Some(first_representable_hour), None);+ let before_split =+ ScalarValue::TimestampNanosecond(Some(first_representable_hour - 1), None);+ let timestamp = fixture.col(0);+ let function = date_bin_of(Arc::clone(×tamp), hour);++ let bin_at_split = evaluate_with_range_value(&function, ×tamp, &at_split);+ assert_eq!(bin_at_split, Some(at_split));++ let bin_before_split =+ evaluate_with_range_value(&function, ×tamp, &before_split);+ assert!(+ bin_before_split.is_none(),+ "binning 1 ns before the split must fail"+ );++ let range = fixture.range_partitioning(+ [0],+ vec![timestamp_split(Some(first_representable_hour), None)],+ );+ assert!(+ !range.keeps_keys_local(&[function], &fixture.eq_properties, false),+ "a boundary evaluation error must reject the optimization",+ );+ Ok(())+ }+fn timestamp_split(value: Option<i64>, timezone: Option<&str>) -> SplitPoint {SplitPoint::new(vec![ScalarValue::TimestampNanosecond(value,diff --git a/datafusion/physical-expr/src/scalar_function.rs b/datafusion/physical-expr/src/scalar_function.rsindex 17dda66269..f43d7e79f1 100644--- a/datafusion/physical-expr/src/scalar_function.rs+++ b/datafusion/physical-expr/src/scalar_function.rs@@ -145,8 +145,8 @@ impl ScalarFunctionExpr {&self.config_options}- /// Returns whether this function expression's argument types have been- /// explicitly audited as candidates for range-key locality analysis.+ /// Returns whether this function expression's argument types and literal+ /// values have been audited as candidates for range-key locality analysis.////// See [`ScalarUDFImpl::supports_range_key_locality_analysis`] for the safety/// requirements.@@ -159,9 +159,14 @@ impl ScalarFunctionExpr {.iter().map(|argument| argument.data_type(input_schema)).collect::<Result<Vec<_>>>()?;+ let literal_arguments = self+ .args+ .iter()+ .map(|argument| argument.downcast_ref::<Literal>().map(Literal::value))+ .collect::<Vec<_>>();Ok(self.fun- .supports_range_key_locality_analysis(&argument_types))+ .supports_range_key_locality_analysis(&argument_types, &literal_arguments))}/// Given an arbitrary PhysicalExpr attempt to downcast it to a ScalarFunctionExpr@@ -370,7 +375,7 @@ impl PhysicalExpr for ScalarFunctionExpr {mod tests {use super::*;use crate::expressions::Column;- use arrow::datatypes::{Field, TimeUnit};+ use arrow::datatypes::{Field, IntervalUnit, TimeUnit};use datafusion_expr::sort_properties::SortProperties;use datafusion_expr::{ScalarUDFImpl, Signature};use datafusion_physical_expr_common::physical_expr::is_volatile;@@ -402,6 +407,111 @@ mod tests {}}+ /// Builds `date_bin(stride, ts)` and checks whether locality analysis is allowed.+ fn date_bin_supports_locality_with_stride(+ function: Arc<ScalarUDF>,+ stride: Arc<dyn PhysicalExpr>,+ ) -> Result<bool> {+ let schema = Schema::new(vec![+ Field::new(+ "stride",+ DataType::Interval(IntervalUnit::MonthDayNano),+ true,+ ),+ Field::new("ts", DataType::Timestamp(TimeUnit::Nanosecond, None), true),+ ]);+ let expr = ScalarFunctionExpr::try_new(+ function,+ vec![stride, Arc::new(Column::new("ts", 1))],+ &schema,+ Arc::new(ConfigOptions::default()),+ )?;+ expr.supports_range_key_locality_analysis(&schema)+ }++ #[test]+ fn range_key_locality_analysis_receives_literal_values() -> Result<()> {+ let function = datafusion_functions::datetime::date_bin();+ let hour = Arc::new(Literal::new(ScalarValue::new_interval_mdn(+ 0,+ 0,+ 3_600_000_000_000,+ )));+ let month = Arc::new(Literal::new(ScalarValue::new_interval_mdn(1, 0, 0)));++ assert!(date_bin_supports_locality_with_stride(+ Arc::clone(&function),+ hour+ )?);+ assert!(!date_bin_supports_locality_with_stride(function, month)?);+ Ok(())+ }++ #[test]+ fn range_key_locality_analysis_preserves_aliases() -> Result<()> {+ let date_bin = datafusion_functions::datetime::date_bin();+ let function =+ Arc::new(date_bin.as_ref().clone().with_aliases(["date_bin_alias"]));+ let hour = Arc::new(Literal::new(ScalarValue::new_interval_mdn(+ 0,+ 0,+ 3_600_000_000_000,+ )));+ let month = Arc::new(Literal::new(ScalarValue::new_interval_mdn(1, 0, 0)));++ assert!(date_bin_supports_locality_with_stride(+ Arc::clone(&function),+ hour+ )?);+ assert!(!date_bin_supports_locality_with_stride(function, month)?);+ Ok(())+ }++ #[test]+ fn range_key_locality_analysis_rejects_non_literal_parameters() -> Result<()> {+ use crate::expressions::CastExpr;++ let function = datafusion_functions::datetime::date_bin();+ let column = Arc::new(Column::new("stride", 0));+ let hour = Arc::new(Literal::new(ScalarValue::new_interval_mdn(+ 0,+ 0,+ 3_600_000_000_000,+ )));+ let cast = Arc::new(CastExpr::new(+ hour,+ DataType::Interval(IntervalUnit::MonthDayNano),+ None,+ ));++ assert!(!date_bin_supports_locality_with_stride(+ Arc::clone(&function),+ column+ )?);+ assert!(!date_bin_supports_locality_with_stride(function, cast)?);+ Ok(())+ }++ #[test]+ fn range_key_locality_analysis_accepts_date_trunc() -> Result<()> {+ let schema = Schema::new(vec![Field::new(+ "ts",+ DataType::Timestamp(TimeUnit::Nanosecond, None),+ true,+ )]);+ let expr = ScalarFunctionExpr::try_new(+ datafusion_functions::datetime::date_trunc(),+ vec![+ Arc::new(Literal::new(ScalarValue::from("hour"))),+ Arc::new(Column::new("ts", 0)),+ ],+ &schema,+ Arc::new(ConfigOptions::default()),+ )?;+ assert!(expr.supports_range_key_locality_analysis(&schema)?);+ Ok(())+ }+fn default_bounds_expr(return_type: DataType) -> ScalarFunctionExpr {ScalarFunctionExpr::try_new(Arc::new(ScalarUDF::from(MockScalarUDF {diff --git a/datafusion/sqllogictest/src/test_context.rs b/datafusion/sqllogictest/src/test_context.rsindex 0b5ada362e..88bf015d1e 100644--- a/datafusion/sqllogictest/src/test_context.rs+++ b/datafusion/sqllogictest/src/test_context.rs@@ -42,7 +42,8 @@ use datafusion_spark::SessionStateBuilderSpark;use crate::is_spark_path;use range_partitioning::{- register_range_partitioned_table, register_range_sorted_time_bin_table,+ register_range_partitioned_table, register_range_sorted_day_bin_table,+ register_range_sorted_time_bin_table,};use async_trait::async_trait;@@ -218,6 +219,7 @@ impl TestContext {"range_sorted_time_bin_agg.slt" => {info!("Registering range-sorted time-bin table");register_range_sorted_time_bin_table(test_ctx.session_ctx());+ register_range_sorted_day_bin_table(test_ctx.session_ctx());}"metadata.slt" | "arrow_field.slt" => {info!("Registering metadata table tables");diff --git a/datafusion/sqllogictest/src/test_context/range_partitioning.rs b/datafusion/sqllogictest/src/test_context/range_partitioning.rsindex e11dcf8040..3cc7af5261 100644--- a/datafusion/sqllogictest/src/test_context/range_partitioning.rs+++ b/datafusion/sqllogictest/src/test_context/range_partitioning.rs@@ -410,9 +410,8 @@ type TimeBinRow = (/// - partition 1: `[2024-01-01 01:00, 02:00)`////// Files are range-partitioned on `timestamp` and sorted on `(key, timestamp)`.-/// The hour split permits partition-local `date_trunc('hour')` aggregation,-/// but `date_trunc('day')` straddles it. Aligned `date_bin` remains a negative-/// control because the function is not admitted to range analysis.+/// The hour split permits partition-local `date_trunc('hour')` and aligned+/// `date_bin` aggregation, but daily bins straddle it.pub(super) fn register_range_sorted_time_bin_table(ctx: &SessionContext) {let schema = Arc::new(Schema::new(vec![Field::new("key", DataType::Utf8, false),@@ -479,6 +478,61 @@ pub(super) fn register_range_sorted_time_bin_table(ctx: &SessionContext) {);}+/// Registers `range_sorted_day_bin` with two timestamp ranges split at midnight.+pub(super) fn register_range_sorted_day_bin_table(ctx: &SessionContext) {+ let schema = Arc::new(Schema::new(vec![+ Field::new(+ "timestamp",+ DataType::Timestamp(TimeUnit::Nanosecond, None),+ false,+ ),+ Field::new("value", DataType::Int64, false),+ ]));+ let partitions = [+ (+ vec![time_bin_ts(23 * 60, 0), time_bin_ts(23 * 60 + 30, 0)],+ vec![1, 2],+ ),+ (+ vec![time_bin_ts(24 * 60, 0), time_bin_ts(24 * 60 + 30, 0)],+ vec![10, 20],+ ),+ ];+ let batches = partitions+ .into_iter()+ .map(|(timestamps, values)| {+ RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(TimestampNanosecondArray::from(timestamps)) as ArrayRef,+ Arc::new(Int64Array::from(values)) as ArrayRef,+ ],+ )+ .expect("daily time-bin batch should be valid")+ })+ .collect();+ let output_partitioning = Partitioning::Range(+ RangePartitioning::try_new(+ vec![col("timestamp").sort(true, true)],+ vec![SplitPoint::new(vec![ScalarValue::TimestampNanosecond(+ Some(time_bin_ts(24 * 60, 0)),+ None,+ )])],+ )+ .expect("daily time-bin range partitioning should be valid"),+ );+ register_parquet_listing_table(+ ctx,+ "range_sorted_day_bin",+ Path::new(env!("CARGO_MANIFEST_DIR"))+ .join("test_files/scratch_range_partitioning/range_sorted_day_bin"),+ schema,+ batches,+ output_partitioning,+ Some(vec![vec![col("timestamp").sort(true, true)]]),+ );+}+fn time_bin_batch(schema: SchemaRef, rows: &[TimeBinRow]) -> RecordBatch {RecordBatch::try_new(schema,diff --git a/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt b/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.sltindex 0aacd3344b..20d186f766 100644--- a/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt+++ b/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt@@ -16,20 +16,12 @@# 1. Range([timestamp]) and output_ordering=[key, timestamp]# 2. Two file_groups, so the two 60-minute streams run in parallel#-# Improvement opportunity:-# In this fixture, date_bin(60s) is monotonic in timestamp and the hour split-# is aligned to bin boundaries, so (key, time_bin) is partition-disjoint.-# Aggregation could be a single streaming SinglePartitioned step with no-# hash shuffle.-#-# This optimization exists for timezone-less date_trunc when every split-# keeps groups partition-local. Extending it to date_bin is future work-# requiring a safety audit, including its null behavior (#25344).-#-# Today's date_bin plan still hash-repartitions:-# Partial AggregateExec (group_clustering_mode=Full)-# -> RepartitionExec Hash([key, date_bin(...)])-# -> FinalPartitioned AggregateExec (group_clustering_mode=Full)+# Both date_bin(60s) and date_trunc('hour') keep groups local at the hour+# split, so aggregation uses SinglePartitioned without a hash shuffle.+# date_bin admission is limited to timezone-less nanosecond timestamps,+# a positive fixed-duration IntervalMonthDayNano literal stride with no months,+# and the default epoch origin.+# Crossing bins and expressions outside that scope retain the hash shuffle.statement okset datafusion.explain.physical_plan_only = true;@@ -75,10 +67,7 @@ physical_plan DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion# TEST 2: Filtered time-bin aggregation.# GROUP BY keys are (key, date_bin(timestamp)). Input is sorted on those keys# and the split aligns with the bins in this data, so the groups are-# partition-local. Timezone-less date_trunc can already avoid the shuffle;-# extending this optimization to date_bin awaits its safety audit (#25344).-# Until then, Partial + hash RepartitionExec + Final is retained even though-# group_clustering_mode=Full is recognized.+# partition-local. The audited date_bin expression avoids the hash shuffle.##########query TT@@ -89,11 +78,9 @@ GROUP BY key, time_bin;----physical_plan01)ProjectionExec: expr=[key@0 as key, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)@1 as time_bin, sum(range_sorted_time_bin.value)@2 as sum(range_sorted_time_bin.value)]-02)--AggregateExec: mode=FinalPartitioned, gby=[key@0 as key, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)@1 as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full-03)----RepartitionExec: partitioning=Hash([key@0, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)@1], 2), input_partitions=2, preserve_order=true, sort_exprs=key@0 ASC, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)@1 ASC-04)------AggregateExec: mode=Partial, gby=[key@0 as key, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, timestamp@1) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full-05)--------FilterExec: col4@1 = a, projection=[key@0, timestamp@2, value@3]-06)----------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[key, col4, timestamp, value], output_ordering=[key@0 ASC, timestamp@2 ASC], output_partitioning=Range([timestamp@2 ASC], [(1704070800000000000)], 2), file_type=parquet, predicate=col4@4 = a, pruning_predicate=col4_null_count@2 != row_count@3 AND col4_min@0 <= a AND a <= col4_max@1, required_guarantees=[col4 in (a)]+02)--AggregateExec: mode=SinglePartitioned, gby=[key@0 as key, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, timestamp@1) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full+03)----FilterExec: col4@1 = a, projection=[key@0, timestamp@2, value@3]+04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[key, col4, timestamp, value], output_ordering=[key@0 ASC, timestamp@2 ASC], output_partitioning=Range([timestamp@2 ASC], [(1704070800000000000)], 2), file_type=parquet, predicate=col4@4 = a, pruning_predicate=col4_null_count@2 != row_count@3 AND col4_min@0 <= a AND a <= col4_max@1, required_guarantees=[col4 in (a)]query TPISELECT key, date_bin(INTERVAL '60 seconds', timestamp) AS time_bin, sum(value)@@ -110,8 +97,7 @@ k2 2024-01-01T01:45:00 5########### TEST 3: Same aggregation without the col4 filter. The groups remain-# partition-local, but date_bin retains the hash shuffle pending its-# safety audit (#25344).+# partition-local, so date_bin still avoids the hash shuffle.##########query TT@@ -121,10 +107,8 @@ GROUP BY key, time_bin;----physical_plan01)ProjectionExec: expr=[key@0 as key, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)@1 as time_bin, sum(range_sorted_time_bin.value)@2 as sum(range_sorted_time_bin.value)]-02)--AggregateExec: mode=FinalPartitioned, gby=[key@0 as key, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)@1 as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full-03)----RepartitionExec: partitioning=Hash([key@0, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)@1], 2), input_partitions=2, preserve_order=true, sort_exprs=key@0 ASC, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)@1 ASC-04)------AggregateExec: mode=Partial, gby=[key@0 as key, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, timestamp@1) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full-05)--------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[key, timestamp, value], output_ordering=[key@0 ASC, timestamp@1 ASC], output_partitioning=Range([timestamp@1 ASC], [(1704070800000000000)], 2), file_type=parquet+02)--AggregateExec: mode=SinglePartitioned, gby=[key@0 as key, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, timestamp@1) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full+03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[key, timestamp, value], output_ordering=[key@0 ASC, timestamp@1 ASC], output_partitioning=Range([timestamp@1 ASC], [(1704070800000000000)], 2), file_type=parquetquery TPISELECT key, date_bin(INTERVAL '60 seconds', timestamp) AS time_bin, sum(value)@@ -139,6 +123,119 @@ k2 2024-01-01T00:30:00 7k2 2024-01-01T01:30:00 30k2 2024-01-01T01:45:00 5+##########+# The 90-minute bin [00:00, 01:30) crosses the scan's 01:00 partition split.+# Keep the hash shuffle even though the date_bin parameters are admitted.+##########++query TT+EXPLAIN SELECT key, date_bin(INTERVAL '90 minutes', timestamp) AS time_bin, sum(value)+FROM range_sorted_time_bin+WHERE col4 = 'a'+GROUP BY key, time_bin;+----+physical_plan+01)ProjectionExec: expr=[key@0 as key, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5400000000000 }"),range_sorted_time_bin.timestamp)@1 as time_bin, sum(range_sorted_time_bin.value)@2 as sum(range_sorted_time_bin.value)]+02)--AggregateExec: mode=FinalPartitioned, gby=[key@0 as key, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5400000000000 }"),range_sorted_time_bin.timestamp)@1 as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5400000000000 }"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full+03)----RepartitionExec: partitioning=Hash([key@0, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5400000000000 }"),range_sorted_time_bin.timestamp)@1], 2), input_partitions=2, preserve_order=true, sort_exprs=key@0 ASC, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5400000000000 }"),range_sorted_time_bin.timestamp)@1 ASC+04)------AggregateExec: mode=Partial, gby=[key@0 as key, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5400000000000 }, timestamp@1) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5400000000000 }"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full+05)--------FilterExec: col4@1 = a, projection=[key@0, timestamp@2, value@3]+06)----------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[key, col4, timestamp, value], output_ordering=[key@0 ASC, timestamp@2 ASC], output_partitioning=Range([timestamp@2 ASC], [(1704070800000000000)], 2), file_type=parquet, predicate=col4@4 = a, pruning_predicate=col4_null_count@2 != row_count@3 AND col4_min@0 <= a AND a <= col4_max@1, required_guarantees=[col4 in (a)]++query TPI+SELECT key, date_bin(INTERVAL '90 minutes', timestamp) AS time_bin, sum(value)+FROM range_sorted_time_bin+WHERE col4 = 'a'+GROUP BY key, time_bin+ORDER BY key, time_bin;+----+k1 2024-01-01T00:00:00 33+k2 2024-01-01T00:00:00 7+k2 2024-01-01T01:30:00 35++##########+# INTERVAL '1 day' uses a day component rather than nanoseconds. Daily bins+# stay local across the 2024-01-02 midnight split, so no hash shuffle is needed.+##########++query TT+EXPLAIN SELECT date_bin(INTERVAL '1 day', timestamp) AS day_bin, sum(value)+FROM range_sorted_day_bin+GROUP BY day_bin;+----+physical_plan+01)ProjectionExec: expr=[date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 1, nanoseconds: 0 }"),range_sorted_day_bin.timestamp)@0 as day_bin, sum(range_sorted_day_bin.value)@1 as sum(range_sorted_day_bin.value)]+02)--AggregateExec: mode=SinglePartitioned, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 1, nanoseconds: 0 }, timestamp@0) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 1, nanoseconds: 0 }"),range_sorted_day_bin.timestamp)], aggr=[sum(range_sorted_day_bin.value)], group_clustering_mode=Full+03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_day_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_day_bin/part-1.parquet]]}, projection=[timestamp, value], output_ordering=[timestamp@0 ASC], output_partitioning=Range([timestamp@0 ASC], [(1704153600000000000)], 2), file_type=parquet++query PI+SELECT date_bin(INTERVAL '1 day', timestamp) AS day_bin, sum(value)+FROM range_sorted_day_bin+GROUP BY day_bin+ORDER BY day_bin;+----+2024-01-01T00:00:00 3+2024-01-02T00:00:00 30++##########+# The same daily bin crosses the original table's 01:00 split. The hash+# shuffle must remain so values from both partitions are combined.+##########++query TT+EXPLAIN SELECT date_bin(INTERVAL '1 day', timestamp) AS day_bin, sum(value)+FROM range_sorted_time_bin+GROUP BY day_bin;+----+physical_plan+01)ProjectionExec: expr=[date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 1, nanoseconds: 0 }"),range_sorted_time_bin.timestamp)@0 as day_bin, sum(range_sorted_time_bin.value)@1 as sum(range_sorted_time_bin.value)]+02)--AggregateExec: mode=FinalPartitioned, gby=[date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 1, nanoseconds: 0 }"),range_sorted_time_bin.timestamp)@0 as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 1, nanoseconds: 0 }"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)]+03)----RepartitionExec: partitioning=Hash([date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 1, nanoseconds: 0 }"),range_sorted_time_bin.timestamp)@0], 2), input_partitions=2+04)------AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 1, nanoseconds: 0 }, timestamp@0) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 1, nanoseconds: 0 }"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)]+05)--------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[timestamp, value], output_partitioning=Range([timestamp@0 ASC], [(1704070800000000000)], 2), file_type=parquet++query PI+SELECT date_bin(INTERVAL '1 day', timestamp) AS day_bin, sum(value)+FROM range_sorted_time_bin+GROUP BY day_bin+ORDER BY day_bin;+----+2024-01-01T00:00:00 174++##########+# This PR enables shuffle removal only for two-argument date_bin calls.+# A three-argument call therefore retains the shuffle, even when its+# explicit epoch origin produces identical bins and would be safe.+# See #26913 for explicit-origin support.+##########++query TT+EXPLAIN SELECT key, date_bin(INTERVAL '60 seconds', timestamp, TIMESTAMP '1970-01-01 00:00:00') AS time_bin, sum(value)+FROM range_sorted_time_bin+WHERE col4 = 'a'+GROUP BY key, time_bin;+----+physical_plan+01)ProjectionExec: expr=[key@0 as key, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp,Utf8("1970-01-01 00:00:00"))@1 as time_bin, sum(range_sorted_time_bin.value)@2 as sum(range_sorted_time_bin.value)]+02)--AggregateExec: mode=FinalPartitioned, gby=[key@0 as key, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp,Utf8("1970-01-01 00:00:00"))@1 as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp,Utf8("1970-01-01 00:00:00"))], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full+03)----RepartitionExec: partitioning=Hash([key@0, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp,Utf8("1970-01-01 00:00:00"))@1], 2), input_partitions=2, preserve_order=true, sort_exprs=key@0 ASC, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp,Utf8("1970-01-01 00:00:00"))@1 ASC+04)------AggregateExec: mode=Partial, gby=[key@0 as key, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, timestamp@1, 0) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp,Utf8("1970-01-01 00:00:00"))], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full+05)--------FilterExec: col4@1 = a, projection=[key@0, timestamp@2, value@3]+06)----------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[key, col4, timestamp, value], output_ordering=[key@0 ASC, timestamp@2 ASC], output_partitioning=Range([timestamp@2 ASC], [(1704070800000000000)], 2), file_type=parquet, predicate=col4@4 = a, pruning_predicate=col4_null_count@2 != row_count@3 AND col4_min@0 <= a AND a <= col4_max@1, required_guarantees=[col4 in (a)]++query TPI+SELECT key, date_bin(INTERVAL '60 seconds', timestamp, TIMESTAMP '1970-01-01 00:00:00') AS time_bin, sum(value)+FROM range_sorted_time_bin+WHERE col4 = 'a'+GROUP BY key, time_bin+ORDER BY key, time_bin;+----+k1 2024-01-01T00:00:00 3+k1 2024-01-01T01:00:00 30+k2 2024-01-01T00:30:00 7+k2 2024-01-01T01:30:00 30+k2 2024-01-01T01:45:00 5+########### TEST 4: date_trunc('hour') is audited and the split is aligned, so each# group is partition-local and aggregation needs no hash shuffle.