Skip to content

Apache DataFusion SQL Query Engine

feat: reuse range partitioning for audited date_bin aggregation (#26835)

Ben Bellickf1ecc4ad2ee5
diff --git a/datafusion/expr/src/udf.rs b/datafusion/expr/src/udf.rs
index 65c1341b6a..c3c9e461e3 100644
--- a/datafusion/expr/src/udf.rs
+++ b/datafusion/expr/src/udf.rs
@@ -355,9 +355,10 @@ impl ScalarUDF {
pub fn supports_range_key_locality_analysis(
&self,
argument_types: &[DataType],
+ literal_arguments: &[Option<&ScalarValue>],
) -> bool {
self.inner
- .supports_range_key_locality_analysis(argument_types)
+ .supports_range_key_locality_analysis(argument_types, literal_arguments)
}
pub fn preserves_lex_ordering(&self, inputs: &[ExprProperties]) -> Result<bool> {
@@ -979,8 +980,8 @@ pub trait ScalarUDFImpl: Debug + DynEq + DynHash + Send + Sync + Any {
Ok(inner(inputs))
}
- /// Returns whether this function and argument-type combination has been
- /// audited for range-partitioning locality analysis.
+ /// Returns whether this function has been audited for range-key locality
+ /// analysis with these argument types and literal values.
///
/// Range-boundary analysis has subtle correctness requirements, so the
/// policy is default-deny. Functions may opt in only after individual audit.
@@ -998,7 +999,15 @@ pub trait ScalarUDFImpl: Debug + DynEq + DynHash + Send + Sync + Any {
/// This does not claim a concrete partition layout or cross-input
/// co-partitioning compatibility. Boundary evaluation errors cause the
/// analysis to return `false`. This capability is not transported across FFI.
- fn supports_range_key_locality_analysis(&self, _argument_types: &[DataType]) -> bool {
+ ///
+ /// `literal_arguments` has one entry per argument. `None` means the argument
+ /// is not a literal; `Some(value)` supplies its value, which may itself be
+ /// NULL. Non-literal expressions are not evaluated to obtain these values.
+ fn supports_range_key_locality_analysis(
+ &self,
+ _argument_types: &[DataType],
+ _literal_arguments: &[Option<&ScalarValue>],
+ ) -> bool {
false
}
@@ -1224,9 +1233,13 @@ impl ScalarUDFImpl for AliasedScalarUDFImpl {
self.inner.output_ordering(inputs)
}
- fn supports_range_key_locality_analysis(&self, argument_types: &[DataType]) -> bool {
+ fn supports_range_key_locality_analysis(
+ &self,
+ argument_types: &[DataType],
+ literal_arguments: &[Option<&ScalarValue>],
+ ) -> bool {
self.inner
- .supports_range_key_locality_analysis(argument_types)
+ .supports_range_key_locality_analysis(argument_types, literal_arguments)
}
fn preserves_lex_ordering(&self, inputs: &[ExprProperties]) -> Result<bool> {
@@ -1314,7 +1327,10 @@ mod tests {
#[test]
fn range_key_locality_analysis_is_disabled_by_default() {
let function = test_func("range_function", "a");
- assert!(!function.supports_range_key_locality_analysis(&[DataType::Int64]));
+ assert!(!function.supports_range_key_locality_analysis(
+ &[DataType::Int64],
+ &[Some(&ScalarValue::Int64(Some(1)))],
+ ));
}
fn test_func(name: &'static str, parameter: &'static str) -> ScalarUDF {
diff --git a/datafusion/functions/src/datetime/date_bin.rs b/datafusion/functions/src/datetime/date_bin.rs
index 98a05d9212..3f72f444f0 100644
--- a/datafusion/functions/src/datetime/date_bin.rs
+++ b/datafusion/functions/src/datetime/date_bin.rs
@@ -276,6 +276,34 @@ impl ScalarUDFImpl for DateBinFunc {
}
}
+ fn supports_range_key_locality_analysis(
+ &self,
+ argument_types: &[DataType],
+ literal_arguments: &[Option<&ScalarValue>],
+ ) -> bool {
+ // With the epoch origin, a positive fixed-duration stride rounds down
+ // monotonically. As in date_bin_impl, each day is 24 hours.
+ // Timestamp binning preserves NULLs or returns an error.
+ match (argument_types, literal_arguments) {
+ (
+ [
+ DataType::Interval(MonthDayNano),
+ Timestamp(Nanosecond, None),
+ ],
+ [Some(ScalarValue::IntervalMonthDayNano(Some(stride))), None],
+ ) => {
+ stride.months == 0
+ && TimeDelta::try_days(i64::from(stride.days))
+ .and_then(|days| {
+ days.checked_add(&Duration::nanoseconds(stride.nanoseconds))
+ })
+ .and_then(|stride| stride.num_nanoseconds())
+ .is_some_and(|stride| stride > 0)
+ }
+ _ => false,
+ }
+ }
+
fn documentation(&self) -> Option<&Documentation> {
self.doc()
}
@@ -978,6 +1006,148 @@ mod tests {
);
}
+ #[test]
+ fn range_key_locality_requires_positive_fixed_duration_stride() {
+ use arrow::datatypes::DataType::Timestamp;
+ use arrow::datatypes::TimeUnit::Nanosecond;
+
+ let function = DateBinFunc::new();
+ let timestamp_type = Timestamp(Nanosecond, None);
+ let cases = [
+ (
+ "one nanosecond",
+ ScalarValue::new_interval_mdn(0, 0, 1),
+ true,
+ ),
+ (
+ "one hour",
+ ScalarValue::new_interval_mdn(0, 0, 3_600_000_000_000),
+ true,
+ ),
+ (
+ "largest nanosecond stride",
+ ScalarValue::new_interval_mdn(0, 0, i64::MAX),
+ true,
+ ),
+ (
+ "NULL stride",
+ ScalarValue::IntervalMonthDayNano(None),
+ false,
+ ),
+ ("zero stride", ScalarValue::new_interval_mdn(0, 0, 0), false),
+ (
+ "negative stride",
+ ScalarValue::new_interval_mdn(0, 0, -1),
+ false,
+ ),
+ (
+ "month stride",
+ ScalarValue::new_interval_mdn(1, 0, 0),
+ false,
+ ),
+ ("day stride", ScalarValue::new_interval_mdn(0, 1, 0), true),
+ (
+ "mixed day and nanosecond stride",
+ ScalarValue::new_interval_mdn(0, 1, 1),
+ true,
+ ),
+ (
+ "day minus one nanosecond",
+ ScalarValue::new_interval_mdn(0, 1, -1),
+ true,
+ ),
+ (
+ "negative day plus one nanosecond",
+ ScalarValue::new_interval_mdn(0, -1, 1),
+ false,
+ ),
+ (
+ "combined zero stride",
+ ScalarValue::new_interval_mdn(0, 1, -86_400_000_000_000),
+ false,
+ ),
+ (
+ "mixed month and day stride",
+ ScalarValue::new_interval_mdn(1, 1, 0),
+ false,
+ ),
+ (
+ "combined stride overflows nanoseconds",
+ ScalarValue::new_interval_mdn(0, 1, i64::MAX),
+ false,
+ ),
+ (
+ "day-time stride",
+ ScalarValue::new_interval_dt(0, 1000),
+ false,
+ ),
+ ];
+ for (description, stride, expected) in cases {
+ assert_eq!(
+ function.supports_range_key_locality_analysis(
+ &[stride.data_type(), timestamp_type.clone()],
+ &[Some(&stride), None],
+ ),
+ expected,
+ "{description}",
+ );
+ }
+ }
+
+ #[test]
+ fn range_key_locality_rejects_unsupported_source_types() {
+ use arrow::datatypes::DataType::{Time64, Timestamp};
+ use arrow::datatypes::TimeUnit::{Microsecond, Millisecond, Nanosecond, Second};
+
+ let function = DateBinFunc::new();
+ let stride = ScalarValue::new_interval_mdn(0, 0, 3_600_000_000_000);
+ for source_type in [
+ Timestamp(Second, None),
+ Timestamp(Millisecond, None),
+ Timestamp(Microsecond, None),
+ Timestamp(Nanosecond, Some("UTC".into())),
+ Time64(Nanosecond),
+ ] {
+ assert!(
+ !function.supports_range_key_locality_analysis(
+ &[stride.data_type(), source_type.clone()],
+ &[Some(&stride), None],
+ ),
+ "source type {source_type:?}",
+ );
+ }
+ }
+
+ #[test]
+ fn range_key_locality_rejects_non_literal_stride() {
+ use arrow::datatypes::IntervalUnit::MonthDayNano;
+
+ let function = DateBinFunc::new();
+ let argument_types = [
+ DataType::Interval(MonthDayNano),
+ DataType::Timestamp(TimeUnit::Nanosecond, None),
+ ];
+ assert!(
+ !function
+ .supports_range_key_locality_analysis(&argument_types, &[None, None])
+ );
+ }
+
+ #[test]
+ fn range_key_locality_rejects_explicit_origin() {
+ let function = DateBinFunc::new();
+ let stride = ScalarValue::new_interval_mdn(0, 0, 3_600_000_000_000);
+ let epoch = ScalarValue::TimestampNanosecond(Some(0), None);
+ assert!(!function.supports_range_key_locality_analysis(
+ &[
+ stride.data_type(),
+ DataType::Timestamp(TimeUnit::Nanosecond, None),
+ epoch.data_type(),
+ ],
+ &[Some(&stride), None, Some(&epoch)],
+ ));
+ }
+
#[test]
fn output_ordering_requires_constant_stride() {
use arrow::compute::SortOptions;
diff --git a/datafusion/functions/src/datetime/date_trunc.rs b/datafusion/functions/src/datetime/date_trunc.rs
index 0ca7270816..d5cf186bc1 100644
--- a/datafusion/functions/src/datetime/date_trunc.rs
+++ b/datafusion/functions/src/datetime/date_trunc.rs
@@ -559,7 +559,11 @@ impl ScalarUDFImpl for DateTruncFunc {
}
}
- fn supports_range_key_locality_analysis(&self, argument_types: &[DataType]) -> bool {
+ fn supports_range_key_locality_analysis(
+ &self,
+ argument_types: &[DataType],
+ _literal_arguments: &[Option<&ScalarValue>],
+ ) -> bool {
// Across an America/Goose_Bay transition, ascending epoch seconds
// 562129259 < 562129260 truncate to 562129200 > 562125600.
// Keep the audited domain to timezone-less timestamp columns.
@@ -1400,19 +1404,30 @@ mod tests {
fn range_key_locality_analysis_requires_timezone_less_timestamp() {
let date_trunc = DateTruncFunc::new();
- assert!(date_trunc.supports_range_key_locality_analysis(&[
- DataType::Utf8,
- DataType::Timestamp(TimeUnit::Nanosecond, None),
- ]));
- assert!(!date_trunc.supports_range_key_locality_analysis(&[
- DataType::Utf8,
- DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())),
- ]));
- assert!(!date_trunc.supports_range_key_locality_analysis(&[
- DataType::Int64,
- DataType::Timestamp(TimeUnit::Nanosecond, None),
- ]));
- assert!(!date_trunc.supports_range_key_locality_analysis(&[DataType::Utf8]));
+ assert!(date_trunc.supports_range_key_locality_analysis(
+ &[
+ DataType::Utf8,
+ DataType::Timestamp(TimeUnit::Nanosecond, None),
+ ],
+ &[None, None],
+ ));
+ assert!(!date_trunc.supports_range_key_locality_analysis(
+ &[
+ DataType::Utf8,
+ DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())),
+ ],
+ &[None, None],
+ ));
+ assert!(!date_trunc.supports_range_key_locality_analysis(
+ &[
+ DataType::Int64,
+ DataType::Timestamp(TimeUnit::Nanosecond, None),
+ ],
+ &[None, None],
+ ));
+ assert!(
+ !date_trunc.supports_range_key_locality_analysis(&[DataType::Utf8], &[None])
+ );
}
fn invoke_date_trunc(
diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rs
index ad1f56b03a..e5177d39be 100644
--- a/datafusion/physical-expr/src/partitioning.rs
+++ b/datafusion/physical-expr/src/partitioning.rs
@@ -1383,6 +1383,117 @@ mod tests {
))
}
+ /// Builds `date_bin(stride, timestamp)` with a nanosecond-only literal stride
+ /// and the default epoch origin. This constructs the expression without evaluating it.
+ fn date_bin_of(
+ timestamp: Arc<dyn PhysicalExpr>,
+ stride_nanos: i64,
+ ) -> Arc<dyn PhysicalExpr> {
+ Arc::new(ScalarFunctionExpr::new(
+ "date_bin",
+ datafusion_functions::datetime::date_bin(),
+ vec![
+ Arc::new(Literal::new(ScalarValue::new_interval_mdn(
+ 0,
+ 0,
+ stride_nanos,
+ ))),
+ timestamp,
+ ],
+ Field::new(
+ "date_bin",
+ DataType::Timestamp(TimeUnit::Nanosecond, None),
+ true,
+ )
+ .into(),
+ Arc::new(ConfigOptions::default()),
+ ))
+ }
+
+ #[test]
+ fn date_bin_requires_disjoint_bins_at_every_split() -> Result<()> {
+ let fixture = PartitioningTestFixture::new(vec![(
+ "ts",
+ DataType::Timestamp(TimeUnit::Nanosecond, None),
+ )])?;
+ let hour = 3_600_000_000_000;
+ let function = date_bin_of(fixture.col(0), hour);
+ // Ascending compares split - 1 ns with split. Descending compares split
+ // with split + 1 ns, so aligned descending splits end a bin rather than start it.
+ let cases = [
+ ("ascending aligned", false, vec![-hour, 0, hour], true),
+ ("ascending crossing", false, vec![-hour, 1, hour], false),
+ (
+ "descending aligned",
+ true,
+ vec![hour - 1, -1, -hour - 1],
+ true,
+ ),
+ ("descending crossing", true, vec![hour, 0, -hour], false),
+ ];
+ for (description, descending, splits, expected) in cases {
+ for nulls_first in [false, true] {
+ let range = fixture.range_partitioning_with_ordering(
+ [fixture
+ .range_sort_expr(0, SortOptions::new(descending, nulls_first))]
+ .into(),
+ splits
+ .iter()
+ .map(|value| timestamp_split(Some(*value), None))
+ .collect(),
+ );
+ assert_eq!(
+ range.keeps_keys_local(
+ &[Arc::clone(&function)],
+ &fixture.eq_properties,
+ false,
+ ),
+ expected,
+ "{description}, nulls_first={nulls_first}",
+ );
+ }
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn date_bin_rejects_adjacent_timestamp_binning_errors() -> Result<()> {
+ let fixture = PartitioningTestFixture::new(vec![(
+ "ts",
+ DataType::Timestamp(TimeUnit::Nanosecond, None),
+ )])?;
+ let hour = 3_600_000_000_000;
+ // The hour containing i64::MIN starts outside the timestamp range.
+ let nanos_to_next_hour = hour - i64::MIN.rem_euclid(hour);
+ let first_representable_hour = i64::MIN + nanos_to_next_hour;
+ let at_split =
+ ScalarValue::TimestampNanosecond(Some(first_representable_hour), None);
+ let before_split =
+ ScalarValue::TimestampNanosecond(Some(first_representable_hour - 1), None);
+ let timestamp = fixture.col(0);
+ let function = date_bin_of(Arc::clone(&timestamp), hour);
+
+ let bin_at_split = evaluate_with_range_value(&function, &timestamp, &at_split);
+ assert_eq!(bin_at_split, Some(at_split));
+
+ let bin_before_split =
+ evaluate_with_range_value(&function, &timestamp, &before_split);
+ assert!(
+ bin_before_split.is_none(),
+ "binning 1 ns before the split must fail"
+ );
+
+ let range = fixture.range_partitioning(
+ [0],
+ vec![timestamp_split(Some(first_representable_hour), None)],
+ );
+ assert!(
+ !range.keeps_keys_local(&[function], &fixture.eq_properties, false),
+ "a boundary evaluation error must reject the optimization",
+ );
+ Ok(())
+ }
+
fn timestamp_split(value: Option<i64>, timezone: Option<&str>) -> SplitPoint {
SplitPoint::new(vec![ScalarValue::TimestampNanosecond(
value,
diff --git a/datafusion/physical-expr/src/scalar_function.rs b/datafusion/physical-expr/src/scalar_function.rs
index 17dda66269..f43d7e79f1 100644
--- a/datafusion/physical-expr/src/scalar_function.rs
+++ b/datafusion/physical-expr/src/scalar_function.rs
@@ -145,8 +145,8 @@ impl ScalarFunctionExpr {
&self.config_options
}
- /// Returns whether this function expression's argument types have been
- /// explicitly audited as candidates for range-key locality analysis.
+ /// Returns whether this function expression's argument types and literal
+ /// values have been audited as candidates for range-key locality analysis.
///
/// See [`ScalarUDFImpl::supports_range_key_locality_analysis`] for the safety
/// requirements.
@@ -159,9 +159,14 @@ impl ScalarFunctionExpr {
.iter()
.map(|argument| argument.data_type(input_schema))
.collect::<Result<Vec<_>>>()?;
+ let literal_arguments = self
+ .args
+ .iter()
+ .map(|argument| argument.downcast_ref::<Literal>().map(Literal::value))
+ .collect::<Vec<_>>();
Ok(self
.fun
- .supports_range_key_locality_analysis(&argument_types))
+ .supports_range_key_locality_analysis(&argument_types, &literal_arguments))
}
/// Given an arbitrary PhysicalExpr attempt to downcast it to a ScalarFunctionExpr
@@ -370,7 +375,7 @@ impl PhysicalExpr for ScalarFunctionExpr {
mod tests {
use super::*;
use crate::expressions::Column;
- use arrow::datatypes::{Field, TimeUnit};
+ use arrow::datatypes::{Field, IntervalUnit, TimeUnit};
use datafusion_expr::sort_properties::SortProperties;
use datafusion_expr::{ScalarUDFImpl, Signature};
use datafusion_physical_expr_common::physical_expr::is_volatile;
@@ -402,6 +407,111 @@ mod tests {
}
}
+ /// Builds `date_bin(stride, ts)` and checks whether locality analysis is allowed.
+ fn date_bin_supports_locality_with_stride(
+ function: Arc<ScalarUDF>,
+ stride: Arc<dyn PhysicalExpr>,
+ ) -> Result<bool> {
+ let schema = Schema::new(vec![
+ Field::new(
+ "stride",
+ DataType::Interval(IntervalUnit::MonthDayNano),
+ true,
+ ),
+ Field::new("ts", DataType::Timestamp(TimeUnit::Nanosecond, None), true),
+ ]);
+ let expr = ScalarFunctionExpr::try_new(
+ function,
+ vec![stride, Arc::new(Column::new("ts", 1))],
+ &schema,
+ Arc::new(ConfigOptions::default()),
+ )?;
+ expr.supports_range_key_locality_analysis(&schema)
+ }
+
+ #[test]
+ fn range_key_locality_analysis_receives_literal_values() -> Result<()> {
+ let function = datafusion_functions::datetime::date_bin();
+ let hour = Arc::new(Literal::new(ScalarValue::new_interval_mdn(
+ 0,
+ 0,
+ 3_600_000_000_000,
+ )));
+ let month = Arc::new(Literal::new(ScalarValue::new_interval_mdn(1, 0, 0)));
+
+ assert!(date_bin_supports_locality_with_stride(
+ Arc::clone(&function),
+ hour
+ )?);
+ assert!(!date_bin_supports_locality_with_stride(function, month)?);
+ Ok(())
+ }
+
+ #[test]
+ fn range_key_locality_analysis_preserves_aliases() -> Result<()> {
+ let date_bin = datafusion_functions::datetime::date_bin();
+ let function =
+ Arc::new(date_bin.as_ref().clone().with_aliases(["date_bin_alias"]));
+ let hour = Arc::new(Literal::new(ScalarValue::new_interval_mdn(
+ 0,
+ 0,
+ 3_600_000_000_000,
+ )));
+ let month = Arc::new(Literal::new(ScalarValue::new_interval_mdn(1, 0, 0)));
+
+ assert!(date_bin_supports_locality_with_stride(
+ Arc::clone(&function),
+ hour
+ )?);
+ assert!(!date_bin_supports_locality_with_stride(function, month)?);
+ Ok(())
+ }
+
+ #[test]
+ fn range_key_locality_analysis_rejects_non_literal_parameters() -> Result<()> {
+ use crate::expressions::CastExpr;
+
+ let function = datafusion_functions::datetime::date_bin();
+ let column = Arc::new(Column::new("stride", 0));
+ let hour = Arc::new(Literal::new(ScalarValue::new_interval_mdn(
+ 0,
+ 0,
+ 3_600_000_000_000,
+ )));
+ let cast = Arc::new(CastExpr::new(
+ hour,
+ DataType::Interval(IntervalUnit::MonthDayNano),
+ None,
+ ));
+
+ assert!(!date_bin_supports_locality_with_stride(
+ Arc::clone(&function),
+ column
+ )?);
+ assert!(!date_bin_supports_locality_with_stride(function, cast)?);
+ Ok(())
+ }
+
+ #[test]
+ fn range_key_locality_analysis_accepts_date_trunc() -> Result<()> {
+ let schema = Schema::new(vec![Field::new(
+ "ts",
+ DataType::Timestamp(TimeUnit::Nanosecond, None),
+ true,
+ )]);
+ let expr = ScalarFunctionExpr::try_new(
+ datafusion_functions::datetime::date_trunc(),
+ vec![
+ Arc::new(Literal::new(ScalarValue::from("hour"))),
+ Arc::new(Column::new("ts", 0)),
+ ],
+ &schema,
+ Arc::new(ConfigOptions::default()),
+ )?;
+ assert!(expr.supports_range_key_locality_analysis(&schema)?);
+ Ok(())
+ }
+
fn default_bounds_expr(return_type: DataType) -> ScalarFunctionExpr {
ScalarFunctionExpr::try_new(
Arc::new(ScalarUDF::from(MockScalarUDF {
diff --git a/datafusion/sqllogictest/src/test_context.rs b/datafusion/sqllogictest/src/test_context.rs
index 0b5ada362e..88bf015d1e 100644
--- a/datafusion/sqllogictest/src/test_context.rs
+++ b/datafusion/sqllogictest/src/test_context.rs
@@ -42,7 +42,8 @@ use datafusion_spark::SessionStateBuilderSpark;
use crate::is_spark_path;
use range_partitioning::{
- register_range_partitioned_table, register_range_sorted_time_bin_table,
+ register_range_partitioned_table, register_range_sorted_day_bin_table,
+ register_range_sorted_time_bin_table,
};
use async_trait::async_trait;
@@ -218,6 +219,7 @@ impl TestContext {
"range_sorted_time_bin_agg.slt" => {
info!("Registering range-sorted time-bin table");
register_range_sorted_time_bin_table(test_ctx.session_ctx());
+ register_range_sorted_day_bin_table(test_ctx.session_ctx());
}
"metadata.slt" | "arrow_field.slt" => {
info!("Registering metadata table tables");
diff --git a/datafusion/sqllogictest/src/test_context/range_partitioning.rs b/datafusion/sqllogictest/src/test_context/range_partitioning.rs
index e11dcf8040..3cc7af5261 100644
--- a/datafusion/sqllogictest/src/test_context/range_partitioning.rs
+++ b/datafusion/sqllogictest/src/test_context/range_partitioning.rs
@@ -410,9 +410,8 @@ type TimeBinRow = (
/// - partition 1: `[2024-01-01 01:00, 02:00)`
///
/// Files are range-partitioned on `timestamp` and sorted on `(key, timestamp)`.
-/// The hour split permits partition-local `date_trunc('hour')` aggregation,
-/// but `date_trunc('day')` straddles it. Aligned `date_bin` remains a negative
-/// control because the function is not admitted to range analysis.
+/// The hour split permits partition-local `date_trunc('hour')` and aligned
+/// `date_bin` aggregation, but daily bins straddle it.
pub(super) fn register_range_sorted_time_bin_table(ctx: &SessionContext) {
let schema = Arc::new(Schema::new(vec![
Field::new("key", DataType::Utf8, false),
@@ -479,6 +478,61 @@ pub(super) fn register_range_sorted_time_bin_table(ctx: &SessionContext) {
);
}
+/// Registers `range_sorted_day_bin` with two timestamp ranges split at midnight.
+pub(super) fn register_range_sorted_day_bin_table(ctx: &SessionContext) {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new(
+ "timestamp",
+ DataType::Timestamp(TimeUnit::Nanosecond, None),
+ false,
+ ),
+ Field::new("value", DataType::Int64, false),
+ ]));
+ let partitions = [
+ (
+ vec![time_bin_ts(23 * 60, 0), time_bin_ts(23 * 60 + 30, 0)],
+ vec![1, 2],
+ ),
+ (
+ vec![time_bin_ts(24 * 60, 0), time_bin_ts(24 * 60 + 30, 0)],
+ vec![10, 20],
+ ),
+ ];
+ let batches = partitions
+ .into_iter()
+ .map(|(timestamps, values)| {
+ RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![
+ Arc::new(TimestampNanosecondArray::from(timestamps)) as ArrayRef,
+ Arc::new(Int64Array::from(values)) as ArrayRef,
+ ],
+ )
+ .expect("daily time-bin batch should be valid")
+ })
+ .collect();
+ let output_partitioning = Partitioning::Range(
+ RangePartitioning::try_new(
+ vec![col("timestamp").sort(true, true)],
+ vec![SplitPoint::new(vec![ScalarValue::TimestampNanosecond(
+ Some(time_bin_ts(24 * 60, 0)),
+ None,
+ )])],
+ )
+ .expect("daily time-bin range partitioning should be valid"),
+ );
+ register_parquet_listing_table(
+ ctx,
+ "range_sorted_day_bin",
+ Path::new(env!("CARGO_MANIFEST_DIR"))
+ .join("test_files/scratch_range_partitioning/range_sorted_day_bin"),
+ schema,
+ batches,
+ output_partitioning,
+ Some(vec![vec![col("timestamp").sort(true, true)]]),
+ );
+}
+
fn time_bin_batch(schema: SchemaRef, rows: &[TimeBinRow]) -> RecordBatch {
RecordBatch::try_new(
schema,
diff --git a/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt b/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt
index 0aacd3344b..20d186f766 100644
--- a/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt
+++ b/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt
@@ -16,20 +16,12 @@
# 1. Range([timestamp]) and output_ordering=[key, timestamp]
# 2. Two file_groups, so the two 60-minute streams run in parallel
#
-# Improvement opportunity:
-# In this fixture, date_bin(60s) is monotonic in timestamp and the hour split
-# is aligned to bin boundaries, so (key, time_bin) is partition-disjoint.
-# Aggregation could be a single streaming SinglePartitioned step with no
-# hash shuffle.
-#
-# This optimization exists for timezone-less date_trunc when every split
-# keeps groups partition-local. Extending it to date_bin is future work
-# requiring a safety audit, including its null behavior (#25344).
-#
-# Today's date_bin plan still hash-repartitions:
-# Partial AggregateExec (group_clustering_mode=Full)
-# -> RepartitionExec Hash([key, date_bin(...)])
-# -> FinalPartitioned AggregateExec (group_clustering_mode=Full)
+# Both date_bin(60s) and date_trunc('hour') keep groups local at the hour
+# split, so aggregation uses SinglePartitioned without a hash shuffle.
+# date_bin admission is limited to timezone-less nanosecond timestamps,
+# a positive fixed-duration IntervalMonthDayNano literal stride with no months,
+# and the default epoch origin.
+# Crossing bins and expressions outside that scope retain the hash shuffle.
statement ok
set datafusion.explain.physical_plan_only = true;
@@ -75,10 +67,7 @@ physical_plan DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion
# TEST 2: Filtered time-bin aggregation.
# GROUP BY keys are (key, date_bin(timestamp)). Input is sorted on those keys
# and the split aligns with the bins in this data, so the groups are
-# partition-local. Timezone-less date_trunc can already avoid the shuffle;
-# extending this optimization to date_bin awaits its safety audit (#25344).
-# Until then, Partial + hash RepartitionExec + Final is retained even though
-# group_clustering_mode=Full is recognized.
+# partition-local. The audited date_bin expression avoids the hash shuffle.
##########
query TT
@@ -89,11 +78,9 @@ GROUP BY key, time_bin;
----
physical_plan
01)ProjectionExec: expr=[key@0 as key, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)@1 as time_bin, sum(range_sorted_time_bin.value)@2 as sum(range_sorted_time_bin.value)]
-02)--AggregateExec: mode=FinalPartitioned, gby=[key@0 as key, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)@1 as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full
-03)----RepartitionExec: partitioning=Hash([key@0, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)@1], 2), input_partitions=2, preserve_order=true, sort_exprs=key@0 ASC, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)@1 ASC
-04)------AggregateExec: mode=Partial, gby=[key@0 as key, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, timestamp@1) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full
-05)--------FilterExec: col4@1 = a, projection=[key@0, timestamp@2, value@3]
-06)----------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[key, col4, timestamp, value], output_ordering=[key@0 ASC, timestamp@2 ASC], output_partitioning=Range([timestamp@2 ASC], [(1704070800000000000)], 2), file_type=parquet, predicate=col4@4 = a, pruning_predicate=col4_null_count@2 != row_count@3 AND col4_min@0 <= a AND a <= col4_max@1, required_guarantees=[col4 in (a)]
+02)--AggregateExec: mode=SinglePartitioned, gby=[key@0 as key, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, timestamp@1) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full
+03)----FilterExec: col4@1 = a, projection=[key@0, timestamp@2, value@3]
+04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[key, col4, timestamp, value], output_ordering=[key@0 ASC, timestamp@2 ASC], output_partitioning=Range([timestamp@2 ASC], [(1704070800000000000)], 2), file_type=parquet, predicate=col4@4 = a, pruning_predicate=col4_null_count@2 != row_count@3 AND col4_min@0 <= a AND a <= col4_max@1, required_guarantees=[col4 in (a)]
query TPI
SELECT key, date_bin(INTERVAL '60 seconds', timestamp) AS time_bin, sum(value)
@@ -110,8 +97,7 @@ k2 2024-01-01T01:45:00 5
##########
# TEST 3: Same aggregation without the col4 filter. The groups remain
-# partition-local, but date_bin retains the hash shuffle pending its
-# safety audit (#25344).
+# partition-local, so date_bin still avoids the hash shuffle.
##########
query TT
@@ -121,10 +107,8 @@ GROUP BY key, time_bin;
----
physical_plan
01)ProjectionExec: expr=[key@0 as key, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)@1 as time_bin, sum(range_sorted_time_bin.value)@2 as sum(range_sorted_time_bin.value)]
-02)--AggregateExec: mode=FinalPartitioned, gby=[key@0 as key, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)@1 as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full
-03)----RepartitionExec: partitioning=Hash([key@0, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)@1], 2), input_partitions=2, preserve_order=true, sort_exprs=key@0 ASC, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)@1 ASC
-04)------AggregateExec: mode=Partial, gby=[key@0 as key, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, timestamp@1) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full
-05)--------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[key, timestamp, value], output_ordering=[key@0 ASC, timestamp@1 ASC], output_partitioning=Range([timestamp@1 ASC], [(1704070800000000000)], 2), file_type=parquet
+02)--AggregateExec: mode=SinglePartitioned, gby=[key@0 as key, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, timestamp@1) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full
+03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[key, timestamp, value], output_ordering=[key@0 ASC, timestamp@1 ASC], output_partitioning=Range([timestamp@1 ASC], [(1704070800000000000)], 2), file_type=parquet
query TPI
SELECT key, date_bin(INTERVAL '60 seconds', timestamp) AS time_bin, sum(value)
@@ -139,6 +123,119 @@ k2 2024-01-01T00:30:00 7
k2 2024-01-01T01:30:00 30
k2 2024-01-01T01:45:00 5
+##########
+# The 90-minute bin [00:00, 01:30) crosses the scan's 01:00 partition split.
+# Keep the hash shuffle even though the date_bin parameters are admitted.
+##########
+
+query TT
+EXPLAIN SELECT key, date_bin(INTERVAL '90 minutes', timestamp) AS time_bin, sum(value)
+FROM range_sorted_time_bin
+WHERE col4 = 'a'
+GROUP BY key, time_bin;
+----
+physical_plan
+01)ProjectionExec: expr=[key@0 as key, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5400000000000 }"),range_sorted_time_bin.timestamp)@1 as time_bin, sum(range_sorted_time_bin.value)@2 as sum(range_sorted_time_bin.value)]
+02)--AggregateExec: mode=FinalPartitioned, gby=[key@0 as key, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5400000000000 }"),range_sorted_time_bin.timestamp)@1 as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5400000000000 }"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full
+03)----RepartitionExec: partitioning=Hash([key@0, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5400000000000 }"),range_sorted_time_bin.timestamp)@1], 2), input_partitions=2, preserve_order=true, sort_exprs=key@0 ASC, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5400000000000 }"),range_sorted_time_bin.timestamp)@1 ASC
+04)------AggregateExec: mode=Partial, gby=[key@0 as key, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5400000000000 }, timestamp@1) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5400000000000 }"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full
+05)--------FilterExec: col4@1 = a, projection=[key@0, timestamp@2, value@3]
+06)----------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[key, col4, timestamp, value], output_ordering=[key@0 ASC, timestamp@2 ASC], output_partitioning=Range([timestamp@2 ASC], [(1704070800000000000)], 2), file_type=parquet, predicate=col4@4 = a, pruning_predicate=col4_null_count@2 != row_count@3 AND col4_min@0 <= a AND a <= col4_max@1, required_guarantees=[col4 in (a)]
+
+query TPI
+SELECT key, date_bin(INTERVAL '90 minutes', timestamp) AS time_bin, sum(value)
+FROM range_sorted_time_bin
+WHERE col4 = 'a'
+GROUP BY key, time_bin
+ORDER BY key, time_bin;
+----
+k1 2024-01-01T00:00:00 33
+k2 2024-01-01T00:00:00 7
+k2 2024-01-01T01:30:00 35
+
+##########
+# INTERVAL '1 day' uses a day component rather than nanoseconds. Daily bins
+# stay local across the 2024-01-02 midnight split, so no hash shuffle is needed.
+##########
+
+query TT
+EXPLAIN SELECT date_bin(INTERVAL '1 day', timestamp) AS day_bin, sum(value)
+FROM range_sorted_day_bin
+GROUP BY day_bin;
+----
+physical_plan
+01)ProjectionExec: expr=[date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 1, nanoseconds: 0 }"),range_sorted_day_bin.timestamp)@0 as day_bin, sum(range_sorted_day_bin.value)@1 as sum(range_sorted_day_bin.value)]
+02)--AggregateExec: mode=SinglePartitioned, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 1, nanoseconds: 0 }, timestamp@0) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 1, nanoseconds: 0 }"),range_sorted_day_bin.timestamp)], aggr=[sum(range_sorted_day_bin.value)], group_clustering_mode=Full
+03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_day_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_day_bin/part-1.parquet]]}, projection=[timestamp, value], output_ordering=[timestamp@0 ASC], output_partitioning=Range([timestamp@0 ASC], [(1704153600000000000)], 2), file_type=parquet
+
+query PI
+SELECT date_bin(INTERVAL '1 day', timestamp) AS day_bin, sum(value)
+FROM range_sorted_day_bin
+GROUP BY day_bin
+ORDER BY day_bin;
+----
+2024-01-01T00:00:00 3
+2024-01-02T00:00:00 30
+
+##########
+# The same daily bin crosses the original table's 01:00 split. The hash
+# shuffle must remain so values from both partitions are combined.
+##########
+
+query TT
+EXPLAIN SELECT date_bin(INTERVAL '1 day', timestamp) AS day_bin, sum(value)
+FROM range_sorted_time_bin
+GROUP BY day_bin;
+----
+physical_plan
+01)ProjectionExec: expr=[date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 1, nanoseconds: 0 }"),range_sorted_time_bin.timestamp)@0 as day_bin, sum(range_sorted_time_bin.value)@1 as sum(range_sorted_time_bin.value)]
+02)--AggregateExec: mode=FinalPartitioned, gby=[date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 1, nanoseconds: 0 }"),range_sorted_time_bin.timestamp)@0 as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 1, nanoseconds: 0 }"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)]
+03)----RepartitionExec: partitioning=Hash([date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 1, nanoseconds: 0 }"),range_sorted_time_bin.timestamp)@0], 2), input_partitions=2
+04)------AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 1, nanoseconds: 0 }, timestamp@0) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 1, nanoseconds: 0 }"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)]
+05)--------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[timestamp, value], output_partitioning=Range([timestamp@0 ASC], [(1704070800000000000)], 2), file_type=parquet
+
+query PI
+SELECT date_bin(INTERVAL '1 day', timestamp) AS day_bin, sum(value)
+FROM range_sorted_time_bin
+GROUP BY day_bin
+ORDER BY day_bin;
+----
+2024-01-01T00:00:00 174
+
+##########
+# This PR enables shuffle removal only for two-argument date_bin calls.
+# A three-argument call therefore retains the shuffle, even when its
+# explicit epoch origin produces identical bins and would be safe.
+# See #26913 for explicit-origin support.
+##########
+
+query TT
+EXPLAIN SELECT key, date_bin(INTERVAL '60 seconds', timestamp, TIMESTAMP '1970-01-01 00:00:00') AS time_bin, sum(value)
+FROM range_sorted_time_bin
+WHERE col4 = 'a'
+GROUP BY key, time_bin;
+----
+physical_plan
+01)ProjectionExec: expr=[key@0 as key, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp,Utf8("1970-01-01 00:00:00"))@1 as time_bin, sum(range_sorted_time_bin.value)@2 as sum(range_sorted_time_bin.value)]
+02)--AggregateExec: mode=FinalPartitioned, gby=[key@0 as key, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp,Utf8("1970-01-01 00:00:00"))@1 as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp,Utf8("1970-01-01 00:00:00"))], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full
+03)----RepartitionExec: partitioning=Hash([key@0, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp,Utf8("1970-01-01 00:00:00"))@1], 2), input_partitions=2, preserve_order=true, sort_exprs=key@0 ASC, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp,Utf8("1970-01-01 00:00:00"))@1 ASC
+04)------AggregateExec: mode=Partial, gby=[key@0 as key, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, timestamp@1, 0) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp,Utf8("1970-01-01 00:00:00"))], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full
+05)--------FilterExec: col4@1 = a, projection=[key@0, timestamp@2, value@3]
+06)----------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[key, col4, timestamp, value], output_ordering=[key@0 ASC, timestamp@2 ASC], output_partitioning=Range([timestamp@2 ASC], [(1704070800000000000)], 2), file_type=parquet, predicate=col4@4 = a, pruning_predicate=col4_null_count@2 != row_count@3 AND col4_min@0 <= a AND a <= col4_max@1, required_guarantees=[col4 in (a)]
+
+query TPI
+SELECT key, date_bin(INTERVAL '60 seconds', timestamp, TIMESTAMP '1970-01-01 00:00:00') AS time_bin, sum(value)
+FROM range_sorted_time_bin
+WHERE col4 = 'a'
+GROUP BY key, time_bin
+ORDER BY key, time_bin;
+----
+k1 2024-01-01T00:00:00 3
+k1 2024-01-01T01:00:00 30
+k2 2024-01-01T00:30:00 7
+k2 2024-01-01T01:30:00 30
+k2 2024-01-01T01:45:00 5
+
##########
# TEST 4: date_trunc('hour') is audited and the split is aligned, so each
# group is partition-local and aggregation needs no hash shuffle.