Apache DataFusion SQL Query Engine
feat(datetime): support date_bin literal origins for range locality (#27135)
Ben Bellick14b3edd9a350
diff --git a/datafusion/functions/src/datetime/date_bin.rs b/datafusion/functions/src/datetime/date_bin.rsindex 3f72f444f0..976aa9aa2b 100644--- a/datafusion/functions/src/datetime/date_bin.rs+++ b/datafusion/functions/src/datetime/date_bin.rs@@ -281,7 +281,7 @@ impl ScalarUDFImpl for DateBinFunc {argument_types: &[DataType],literal_arguments: &[Option<&ScalarValue>],) -> bool {- // With the epoch origin, a positive fixed-duration stride rounds down+ // With a fixed origin, a positive fixed-duration stride rounds down// monotonically. As in date_bin_impl, each day is 24 hours.// Timestamp binning preserves NULLs or returns an error.match (argument_types, literal_arguments) {@@ -291,6 +291,18 @@ impl ScalarUDFImpl for DateBinFunc {Timestamp(Nanosecond, None),],[Some(ScalarValue::IntervalMonthDayNano(Some(stride))), None],+ )+ | (+ [+ DataType::Interval(MonthDayNano),+ Timestamp(Nanosecond, None),+ Timestamp(Nanosecond, None),+ ],+ [+ Some(ScalarValue::IntervalMonthDayNano(Some(stride))),+ None,+ Some(ScalarValue::TimestampNanosecond(Some(_), None)),+ ],) => {stride.months == 0&& TimeDelta::try_days(i64::from(stride.days))@@ -1013,6 +1025,7 @@ mod tests {let function = DateBinFunc::new();let timestamp_type = Timestamp(Nanosecond, None);+ let epoch = ScalarValue::TimestampNanosecond(Some(0), None);let cases = [("one nanosecond",@@ -1089,7 +1102,19 @@ mod tests {&[Some(&stride), None],),expected,- "{description}",+ "{description}, default origin",+ );+ assert_eq!(+ function.supports_range_key_locality_analysis(+ &[+ stride.data_type(),+ timestamp_type.clone(),+ epoch.data_type(),+ ],+ &[Some(&stride), None, Some(&epoch)],+ ),+ expected,+ "{description}, explicit origin",);}}@@ -1101,6 +1126,7 @@ mod tests {let function = DateBinFunc::new();let stride = ScalarValue::new_interval_mdn(0, 0, 3_600_000_000_000);+ let epoch = ScalarValue::TimestampNanosecond(Some(0), None);for source_type in [Timestamp(Second, None),Timestamp(Millisecond, None),@@ -1113,7 +1139,14 @@ mod tests {&[stride.data_type(), source_type.clone()],&[Some(&stride), None],),- "source type {source_type:?}",+ "source type {source_type:?}, default origin",+ );+ assert!(+ !function.supports_range_key_locality_analysis(+ &[stride.data_type(), source_type.clone(), epoch.data_type()],+ &[Some(&stride), None, Some(&epoch)],+ ),+ "source type {source_type:?}, explicit origin",);}}@@ -1131,23 +1164,54 @@ mod tests {!function.supports_range_key_locality_analysis(&argument_types, &[None, None]));+ let epoch = ScalarValue::TimestampNanosecond(Some(0), None);+ assert!(!function.supports_range_key_locality_analysis(+ &[+ argument_types[0].clone(),+ argument_types[1].clone(),+ epoch.data_type(),+ ],+ &[None, None, Some(&epoch)],+ ));}#[test]- fn range_key_locality_rejects_explicit_origin() {+ fn range_key_locality_rejects_non_literal_origin() {let function = DateBinFunc::new();let stride = ScalarValue::new_interval_mdn(0, 0, 3_600_000_000_000);- let epoch = ScalarValue::TimestampNanosecond(Some(0), None);+ let timestamp_type = DataType::Timestamp(TimeUnit::Nanosecond, None);assert!(!function.supports_range_key_locality_analysis(- &[- stride.data_type(),- DataType::Timestamp(TimeUnit::Nanosecond, None),- epoch.data_type(),- ],- &[Some(&stride), None, Some(&epoch)],+ &[stride.data_type(), timestamp_type.clone(), timestamp_type],+ &[Some(&stride), None, None],));}+ #[test]+ fn range_key_locality_rejects_unsupported_origins() {+ let function = DateBinFunc::new();+ let stride = ScalarValue::new_interval_mdn(0, 0, 3_600_000_000_000);+ for origin in [+ ScalarValue::TimestampNanosecond(None, None),+ ScalarValue::TimestampNanosecond(Some(0), Some("UTC".into())),+ ScalarValue::TimestampSecond(Some(0), None),+ ScalarValue::TimestampMillisecond(Some(0), None),+ ScalarValue::TimestampMicrosecond(Some(0), None),+ ScalarValue::Int64(Some(0)),+ ] {+ assert!(+ !function.supports_range_key_locality_analysis(+ &[+ stride.data_type(),+ DataType::Timestamp(TimeUnit::Nanosecond, None),+ origin.data_type(),+ ],+ &[Some(&stride), None, Some(&origin)],+ ),+ "origin {origin:?}",+ );+ }+ }+#[test]fn output_ordering_requires_constant_stride() {use arrow::compute::SortOptions;diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rsindex e5177d39be..e29e0c1c6b 100644--- a/datafusion/physical-expr/src/partitioning.rs+++ b/datafusion/physical-expr/src/partitioning.rs@@ -1383,23 +1383,31 @@ mod tests {))}- /// Builds `date_bin(stride, timestamp)` with a nanosecond-only literal stride- /// and the default epoch origin. This constructs the expression without evaluating it.+ /// Builds `date_bin` with a nanosecond-only literal stride and optional+ /// timezone-less nanosecond literal origin, without evaluating it.fn date_bin_of(timestamp: Arc<dyn PhysicalExpr>,stride_nanos: i64,+ origin_nanos: Option<i64>,) -> Arc<dyn PhysicalExpr> {+ let mut arguments: Vec<Arc<dyn PhysicalExpr>> = vec![+ Arc::new(Literal::new(ScalarValue::new_interval_mdn(+ 0,+ 0,+ stride_nanos,+ ))),+ timestamp,+ ];+ if let Some(origin_nanos) = origin_nanos {+ arguments.push(Arc::new(Literal::new(ScalarValue::TimestampNanosecond(+ Some(origin_nanos),+ None,+ ))));+ }Arc::new(ScalarFunctionExpr::new("date_bin",datafusion_functions::datetime::date_bin(),- vec![- Arc::new(Literal::new(ScalarValue::new_interval_mdn(- 0,- 0,- stride_nanos,- ))),- timestamp,- ],+ arguments,Field::new("date_bin",DataType::Timestamp(TimeUnit::Nanosecond, None),@@ -1417,7 +1425,7 @@ mod tests {DataType::Timestamp(TimeUnit::Nanosecond, None),)])?;let hour = 3_600_000_000_000;- let function = date_bin_of(fixture.col(0), hour);+ let function = date_bin_of(fixture.col(0), hour, None);// Ascending compares split - 1 ns with split. Descending compares split// with split + 1 ns, so aligned descending splits end a bin rather than start it.let cases = [@@ -1471,7 +1479,7 @@ mod tests {let before_split =ScalarValue::TimestampNanosecond(Some(first_representable_hour - 1), None);let timestamp = fixture.col(0);- let function = date_bin_of(Arc::clone(×tamp), hour);+ let function = date_bin_of(Arc::clone(×tamp), hour, None);let bin_at_split = evaluate_with_range_value(&function, ×tamp, &at_split);assert_eq!(bin_at_split, Some(at_split));@@ -1494,6 +1502,102 @@ mod tests {Ok(())}+ #[test]+ fn date_bin_literal_origin_requires_aligned_ascending_splits() -> Result<()> {+ let fixture = PartitioningTestFixture::new(vec![(+ "ts",+ DataType::Timestamp(TimeUnit::Nanosecond, None),+ )])?;+ let hour = 3_600_000_000_000;+ let half_hour = hour / 2;+ let time_bin = date_bin_of(fixture.col(0), hour, Some(half_hour));++ let aligned_range =+ fixture.range_partitioning([0], vec![timestamp_split(Some(half_hour), None)]);+ assert!(aligned_range.keeps_keys_local(+ &[Arc::clone(&time_bin)],+ &fixture.eq_properties,+ false,+ ));++ let crossing_range =+ fixture.range_partitioning([0], vec![timestamp_split(Some(hour), None)]);+ assert!(!crossing_range.keeps_keys_local(+ &[time_bin],+ &fixture.eq_properties,+ false,+ ));+ Ok(())+ }++ #[test]+ fn date_bin_literal_origin_requires_aligned_descending_splits() -> Result<()> {+ let fixture = PartitioningTestFixture::new(vec![(+ "ts",+ DataType::Timestamp(TimeUnit::Nanosecond, None),+ )])?;+ let hour = 3_600_000_000_000;+ let half_hour = hour / 2;+ let time_bin = date_bin_of(fixture.col(0), hour, Some(half_hour));+ let descending_ordering = LexOrdering::from([fixture.range_sort_expr(+ 0,+ SortOptions {+ descending: true,+ nulls_first: true,+ },+ )]);++ // A descending split belongs to the lower timestamp range. To separate+ // bins at 00:30, put the split at the last nanosecond before 00:30.+ let last_timestamp_before_half_hour = half_hour - 1;+ let aligned_range = fixture.range_partitioning_with_ordering(+ descending_ordering.clone(),+ vec![timestamp_split(Some(last_timestamp_before_half_hour), None)],+ );+ assert!(aligned_range.keeps_keys_local(+ &[Arc::clone(&time_bin)],+ &fixture.eq_properties,+ false,+ ));++ let last_timestamp_before_hour = hour - 1;+ let crossing_range = fixture.range_partitioning_with_ordering(+ descending_ordering,+ vec![timestamp_split(Some(last_timestamp_before_hour), None)],+ );+ assert!(!crossing_range.keeps_keys_local(+ &[time_bin],+ &fixture.eq_properties,+ false,+ ));+ Ok(())+ }++ #[test]+ fn date_bin_literal_origin_rejects_boundary_binning_errors() -> Result<()> {+ let fixture = PartitioningTestFixture::new(vec![(+ "ts",+ DataType::Timestamp(TimeUnit::Nanosecond, None),+ )])?;+ let timestamp = fixture.col(0);+ let function = date_bin_of(Arc::clone(×tamp), 2, Some(1));+ // A 2 ns stride with origin 1 bins to odd timestamps. i64::MIN is+ // even, so its bin is out of range, but i64::MIN + 1 is representable.+ let split = ScalarValue::TimestampNanosecond(Some(i64::MIN + 1), None);+ let before_split = ScalarValue::TimestampNanosecond(Some(i64::MIN), None);+ assert_eq!(+ evaluate_with_range_value(&function, ×tamp, &split),+ Some(split.clone()),+ );+ assert!(+ evaluate_with_range_value(&function, ×tamp, &before_split).is_none()+ );++ let range = fixture.range_partitioning([0], vec![SplitPoint::new(vec![split])]);+ assert!(!range.keeps_keys_local(&[function], &fixture.eq_properties, false));+ Ok(())+ }+fn timestamp_split(value: Option<i64>, timezone: Option<&str>) -> SplitPoint {SplitPoint::new(vec![ScalarValue::TimestampNanosecond(value,diff --git a/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt b/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.sltindex 20d186f766..874a2b6c79 100644--- a/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt+++ b/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt@@ -20,7 +20,8 @@# split, so aggregation uses SinglePartitioned without a hash shuffle.# date_bin admission is limited to timezone-less nanosecond timestamps,# a positive fixed-duration IntervalMonthDayNano literal stride with no months,-# and the default epoch origin.+# and either the default epoch origin or a non-null literal origin with the+# same timestamp type.# Crossing bins and expressions outside that scope retain the hash shuffle.statement ok@@ -203,10 +204,8 @@ ORDER BY day_bin;2024-01-01T00:00:00 174##########-# This PR enables shuffle removal only for two-argument date_bin calls.-# A three-argument call therefore retains the shuffle, even when its-# explicit epoch origin produces identical bins and would be safe.-# See #26913 for explicit-origin support.+# An explicit epoch origin produces the same aligned minute bins as the+# two-argument call and also avoids the hash shuffle.##########query TT@@ -217,11 +216,9 @@ GROUP BY key, time_bin;----physical_plan01)ProjectionExec: expr=[key@0 as key, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp,Utf8("1970-01-01 00:00:00"))@1 as time_bin, sum(range_sorted_time_bin.value)@2 as sum(range_sorted_time_bin.value)]-02)--AggregateExec: mode=FinalPartitioned, gby=[key@0 as key, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp,Utf8("1970-01-01 00:00:00"))@1 as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp,Utf8("1970-01-01 00:00:00"))], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full-03)----RepartitionExec: partitioning=Hash([key@0, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp,Utf8("1970-01-01 00:00:00"))@1], 2), input_partitions=2, preserve_order=true, sort_exprs=key@0 ASC, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp,Utf8("1970-01-01 00:00:00"))@1 ASC-04)------AggregateExec: mode=Partial, gby=[key@0 as key, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, timestamp@1, 0) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp,Utf8("1970-01-01 00:00:00"))], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full-05)--------FilterExec: col4@1 = a, projection=[key@0, timestamp@2, value@3]-06)----------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[key, col4, timestamp, value], output_ordering=[key@0 ASC, timestamp@2 ASC], output_partitioning=Range([timestamp@2 ASC], [(1704070800000000000)], 2), file_type=parquet, predicate=col4@4 = a, pruning_predicate=col4_null_count@2 != row_count@3 AND col4_min@0 <= a AND a <= col4_max@1, required_guarantees=[col4 in (a)]+02)--AggregateExec: mode=SinglePartitioned, gby=[key@0 as key, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, timestamp@1, 0) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"),range_sorted_time_bin.timestamp,Utf8("1970-01-01 00:00:00"))], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full+03)----FilterExec: col4@1 = a, projection=[key@0, timestamp@2, value@3]+04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[key, col4, timestamp, value], output_ordering=[key@0 ASC, timestamp@2 ASC], output_partitioning=Range([timestamp@2 ASC], [(1704070800000000000)], 2), file_type=parquet, predicate=col4@4 = a, pruning_predicate=col4_null_count@2 != row_count@3 AND col4_min@0 <= a AND a <= col4_max@1, required_guarantees=[col4 in (a)]query TPISELECT key, date_bin(INTERVAL '60 seconds', timestamp, TIMESTAMP '1970-01-01 00:00:00') AS time_bin, sum(value)@@ -236,6 +233,65 @@ k2 2024-01-01T00:30:00 7k2 2024-01-01T01:30:00 30k2 2024-01-01T01:45:00 5+##########+# With the default origin, 90-minute bins cross the 01:00 split. An explicit+# origin at 01:00 shifts the bins so the same split now separates them.+##########++query TT+EXPLAIN SELECT key, date_bin(INTERVAL '90 minutes', timestamp, TIMESTAMP '2010-01-01 01:00:00') AS time_bin, sum(value)+FROM range_sorted_time_bin+WHERE col4 = 'a'+GROUP BY key, time_bin;+----+physical_plan+01)ProjectionExec: expr=[key@0 as key, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5400000000000 }"),range_sorted_time_bin.timestamp,Utf8("2010-01-01 01:00:00"))@1 as time_bin, sum(range_sorted_time_bin.value)@2 as sum(range_sorted_time_bin.value)]+02)--AggregateExec: mode=SinglePartitioned, gby=[key@0 as key, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5400000000000 }, timestamp@1, 1262307600000000000) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5400000000000 }"),range_sorted_time_bin.timestamp,Utf8("2010-01-01 01:00:00"))], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full+03)----FilterExec: col4@1 = a, projection=[key@0, timestamp@2, value@3]+04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[key, col4, timestamp, value], output_ordering=[key@0 ASC, timestamp@2 ASC], output_partitioning=Range([timestamp@2 ASC], [(1704070800000000000)], 2), file_type=parquet, predicate=col4@4 = a, pruning_predicate=col4_null_count@2 != row_count@3 AND col4_min@0 <= a AND a <= col4_max@1, required_guarantees=[col4 in (a)]++query TPI+SELECT key, date_bin(INTERVAL '90 minutes', timestamp, TIMESTAMP '2010-01-01 01:00:00') AS time_bin, sum(value)+FROM range_sorted_time_bin+WHERE col4 = 'a'+GROUP BY key, time_bin+ORDER BY key, time_bin;+----+k1 2023-12-31T23:30:00 3+k1 2024-01-01T01:00:00 30+k2 2023-12-31T23:30:00 7+k2 2024-01-01T01:00:00 35++##########+# Moving the origin to 00:30 makes the 90-minute bin [00:30, 02:00) cross+# the 01:00 split. The shuffle must combine k2's values from both partitions.+##########++query TT+EXPLAIN SELECT key, date_bin(INTERVAL '90 minutes', timestamp, TIMESTAMP '2010-01-01 00:30:00') AS time_bin, sum(value)+FROM range_sorted_time_bin+WHERE col4 = 'a'+GROUP BY key, time_bin;+----+physical_plan+01)ProjectionExec: expr=[key@0 as key, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5400000000000 }"),range_sorted_time_bin.timestamp,Utf8("2010-01-01 00:30:00"))@1 as time_bin, sum(range_sorted_time_bin.value)@2 as sum(range_sorted_time_bin.value)]+02)--AggregateExec: mode=FinalPartitioned, gby=[key@0 as key, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5400000000000 }"),range_sorted_time_bin.timestamp,Utf8("2010-01-01 00:30:00"))@1 as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5400000000000 }"),range_sorted_time_bin.timestamp,Utf8("2010-01-01 00:30:00"))], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full+03)----RepartitionExec: partitioning=Hash([key@0, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5400000000000 }"),range_sorted_time_bin.timestamp,Utf8("2010-01-01 00:30:00"))@1], 2), input_partitions=2, preserve_order=true, sort_exprs=key@0 ASC, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5400000000000 }"),range_sorted_time_bin.timestamp,Utf8("2010-01-01 00:30:00"))@1 ASC+04)------AggregateExec: mode=Partial, gby=[key@0 as key, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5400000000000 }, timestamp@1, 1262305800000000000) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5400000000000 }"),range_sorted_time_bin.timestamp,Utf8("2010-01-01 00:30:00"))], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full+05)--------FilterExec: col4@1 = a, projection=[key@0, timestamp@2, value@3]+06)----------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[key, col4, timestamp, value], output_ordering=[key@0 ASC, timestamp@2 ASC], output_partitioning=Range([timestamp@2 ASC], [(1704070800000000000)], 2), file_type=parquet, predicate=col4@4 = a, pruning_predicate=col4_null_count@2 != row_count@3 AND col4_min@0 <= a AND a <= col4_max@1, required_guarantees=[col4 in (a)]++query TPI+SELECT key, date_bin(INTERVAL '90 minutes', timestamp, TIMESTAMP '2010-01-01 00:30:00') AS time_bin, sum(value)+FROM range_sorted_time_bin+WHERE col4 = 'a'+GROUP BY key, time_bin+ORDER BY key, time_bin;+----+k1 2023-12-31T23:00:00 3+k1 2024-01-01T00:30:00 30+k2 2024-01-01T00:30:00 42+########### TEST 4: date_trunc('hour') is audited and the split is aligned, so each# group is partition-local and aggregation needs no hash shuffle.