Apache DataFusion SQL Query Engine
Unify temporal field parsing with DatePart (#24906)
Kosta Tarasov6fc7baa7d0ae
diff --git a/datafusion/functions/src/datetime/date_part.rs b/datafusion/functions/src/datetime/date_part.rsindex e3f67db905..4bfadf7999 100644--- a/datafusion/functions/src/datetime/date_part.rs+++ b/datafusion/functions/src/datetime/date_part.rs@@ -21,7 +21,6 @@ use std::sync::Arc;use arrow::array::timezone::Tz;use arrow::array::{Array, ArrayRef, Float64Array, Int32Array, Int64Array};-use arrow::compute::kernels::cast_utils::IntervalUnit;use arrow::compute::{DatePart, binary, date_part};use arrow::datatypes::DataType::{Date32, Date64, Duration, Interval, Time32, Time64, Timestamp,@@ -216,37 +215,14 @@ impl ScalarUDFImpl for DatePartFunc {let part_trim = part_normalization(&part);- // using IntervalUnit here means we hand off all the work of supporting plurals (like "seconds")- // and synonyms ( like "ms,msec,msecond,millisecond") to Arrow- let arr = if let Ok(interval_unit) = IntervalUnit::from_str(part_trim) {- match interval_unit {- IntervalUnit::Year => date_part(array.as_ref(), DatePart::Year)?,- IntervalUnit::Month => date_part(array.as_ref(), DatePart::Month)?,- IntervalUnit::Week => date_part(array.as_ref(), DatePart::Week)?,- IntervalUnit::Day => date_part(array.as_ref(), DatePart::Day)?,- IntervalUnit::Hour => date_part(array.as_ref(), DatePart::Hour)?,- IntervalUnit::Minute => date_part(array.as_ref(), DatePart::Minute)?,- IntervalUnit::Second => seconds_as_i32(array.as_ref(), Second)?,- IntervalUnit::Millisecond => seconds_as_i32(array.as_ref(), Millisecond)?,- IntervalUnit::Microsecond => seconds_as_i32(array.as_ref(), Microsecond)?,- IntervalUnit::Nanosecond => seconds_ns(array.as_ref())?,- // century and decade are not supported by `DatePart`, although they are supported in postgres- _ => return exec_err!("Date part '{part}' not supported"),- }- } else {- // special cases that can be extracted (in postgres) but are not interval units- match part_trim.to_lowercase().as_str() {- "isoyear" => date_part(array.as_ref(), DatePart::YearISO)?,- "qtr" | "quarter" => date_part(array.as_ref(), DatePart::Quarter)?,- "doy" => date_part(array.as_ref(), DatePart::DayOfYear)?,- "dow" => date_part(array.as_ref(), DatePart::DayOfWeekSunday0)?,- "isodow" => {- // Postgres `isodow` is 1..=7 with Mon=1- date_part(array.as_ref(), DatePart::DayOfWeekMonday1)?- }- "epoch" => epoch(array.as_ref())?,- _ => return exec_err!("Date part '{part}' not supported"),- }+ let arr = match DatePart::from_str(part_trim) {+ Ok(DatePart::Second) => seconds_as_i32(array.as_ref(), Second)?,+ Ok(DatePart::Millisecond) => seconds_as_i32(array.as_ref(), Millisecond)?,+ Ok(DatePart::Microsecond) => seconds_as_i32(array.as_ref(), Microsecond)?,+ Ok(DatePart::Nanosecond) => seconds_ns(array.as_ref())?,+ Ok(part) => date_part(array.as_ref(), part)?,+ Err(_) if is_epoch(part_trim) => epoch(array.as_ref())?,+ Err(_) => return exec_err!("Date part '{part}' not supported"),};Ok(if is_scalar {@@ -256,7 +232,7 @@ impl ScalarUDFImpl for DatePartFunc {})}- // Only casting the year is supported since pruning other IntervalUnit is not possible+ // Only casting the year is supported since pruning other date parts is not possible// date_part(col, YEAR) = 2024 => col >= '2024-01-01' and col < '2025-01-01'// But for anything less than YEAR simplifying is not possible without specifying the bigger interval// date_part(col, MONTH) = 1 => col = '2023-01-01' or col = '2024-01-01' or ... or col = '3000-01-01'@@ -268,16 +244,16 @@ impl ScalarUDFImpl for DatePartFunc {) -> Result<PreimageResult> {let [part, col_expr] = take_function_args(self.name(), args)?;- // Get the interval unit from the part argument- let interval_unit = part+ // Get the date part from the part argument+ let date_part = part.as_literal().and_then(|sv| sv.try_as_str().flatten()).map(part_normalization)- .and_then(|s| IntervalUnit::from_str(s).ok());+ .and_then(|s| DatePart::from_str(s).ok());// only support extracting year- match interval_unit {- Some(IntervalUnit::Year) => (),+ match date_part {+ Some(DatePart::Year) => (),_ => return Ok(PreimageResult::None),}@@ -336,8 +312,8 @@ fn is_epoch(part: &str) -> bool {}fn is_nanosecond(part: &str) -> bool {- IntervalUnit::from_str(part_normalization(part))- .map(|p| matches!(p, IntervalUnit::Nanosecond))+ DatePart::from_str(part_normalization(part))+ .map(|p| matches!(p, DatePart::Nanosecond)).unwrap_or(false)}diff --git a/datafusion/functions/src/datetime/date_trunc.rs b/datafusion/functions/src/datetime/date_trunc.rsindex 926653c0d1..22c8af262a 100644--- a/datafusion/functions/src/datetime/date_trunc.rs+++ b/datafusion/functions/src/datetime/date_trunc.rs@@ -15,7 +15,6 @@// specific language governing permissions and limitations// under the License.-use std::fmt;use std::num::NonZeroI64;use std::ops::{Add, Sub};use std::str::FromStr;@@ -31,6 +30,7 @@ use arrow::array::types::{TimestampNanosecondType, TimestampSecondType,};use arrow::array::{Array, ArrayRef, PrimitiveArray};+use arrow::compute::DatePart;use arrow::datatypes::DataType::{self, Time32, Time64, Timestamp};use arrow::datatypes::TimeUnit::{self, Microsecond, Millisecond, Nanosecond, Second};use arrow::datatypes::{Field, FieldRef};@@ -51,74 +51,58 @@ use chrono::{DateTime, Datelike, Duration, LocalResult, NaiveDateTime, Offset, TimeDelta, Timelike,};-/// Represents the granularity for date truncation operations-#[derive(Debug, Clone, Copy, PartialEq, Eq)]-enum DateTruncGranularity {- Microsecond,- Millisecond,- Second,- Minute,- Hour,- Day,- Week,- Month,- Quarter,- Year,-}--impl DateTruncGranularity {- /// List of all supported granularity values- /// Cannot use HashMap here as it would require lazy_static or once_cell,- /// Rust does not support const HashMap yet.- const SUPPORTED_GRANULARITIES: &[&str] = &[- "microsecond",- "millisecond",- "second",- "minute",- "hour",- "day",- "week",- "month",- "quarter",- "year",- ];-- /// Parse a granularity string into a DateTruncGranularity enum- fn from_str(s: &str) -> Result<Self> {- // Using match for O(1) lookup - compiler optimizes this into a jump table or perfect hash- match s.to_lowercase().as_str() {- "microsecond" => Ok(Self::Microsecond),- "millisecond" => Ok(Self::Millisecond),- "second" => Ok(Self::Second),- "minute" => Ok(Self::Minute),- "hour" => Ok(Self::Hour),- "day" => Ok(Self::Day),- "week" => Ok(Self::Week),- "month" => Ok(Self::Month),- "quarter" => Ok(Self::Quarter),- "year" => Ok(Self::Year),- _ => {- let supported = Self::SUPPORTED_GRANULARITIES.join(", ");- exec_err!(- "Unsupported date_trunc granularity: '{s}'. Supported values are: {supported}"- )- }+fn parse_granularity(value: &str) -> Result<DatePart> {+ // DatePart also contains extraction-only fields such as day-of-week, which+ // are not valid truncation granularities.+ match DatePart::from_str(value) {+ Ok(granularity)+ if matches!(+ granularity,+ DatePart::Microsecond+ | DatePart::Millisecond+ | DatePart::Second+ | DatePart::Minute+ | DatePart::Hour+ | DatePart::Day+ | DatePart::Week+ | DatePart::Month+ | DatePart::Quarter+ | DatePart::Year+ ) =>+ {+ Ok(granularity)+ }+ _ => {+ exec_err!(+ "Unsupported date_trunc granularity: '{value}'. Supported granularities are: microsecond, millisecond, second, minute, hour, day, week, month, quarter, year"+ )}}+}++trait DateTruncGranularityExt {+ fn is_fine_granularity(&self) -> bool;+ fn is_fine_granularity_utc(&self) -> bool;+ fn valid_for_time(&self) -> bool;+}+impl DateTruncGranularityExt for DatePart {/// Returns true if this granularity can be handled with simple arithmetic/// (fine granularity: second, minute, millisecond, microsecond)fn is_fine_granularity(&self) -> bool {matches!(self,- Self::Second | Self::Minute | Self::Millisecond | Self::Microsecond+ DatePart::Second+ | DatePart::Minute+ | DatePart::Millisecond+ | DatePart::Microsecond)}/// Returns true if this granularity can be handled with simple arithmetic in UTC/// (hour and day in addition to fine granularities)fn is_fine_granularity_utc(&self) -> bool {- self.is_fine_granularity() || matches!(self, Self::Hour | Self::Day)+ self.is_fine_granularity() || matches!(self, DatePart::Hour | DatePart::Day)}/// Returns true if this granularity is valid for Time types@@ -126,33 +110,15 @@ impl DateTruncGranularity {fn valid_for_time(&self) -> bool {matches!(self,- Self::Hour- | Self::Minute- | Self::Second- | Self::Millisecond- | Self::Microsecond+ DatePart::Hour+ | DatePart::Minute+ | DatePart::Second+ | DatePart::Millisecond+ | DatePart::Microsecond)}}-impl fmt::Display for DateTruncGranularity {- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {- let value = match self {- Self::Microsecond => "microsecond",- Self::Millisecond => "millisecond",- Self::Second => "second",- Self::Minute => "minute",- Self::Hour => "hour",- Self::Day => "day",- Self::Week => "week",- Self::Month => "month",- Self::Quarter => "quarter",- Self::Year => "year",- };- f.write_str(value)- }-}-#[user_doc(doc_section(label = "Time and Date Functions"),description = "Truncates a timestamp or time value to a specified precision.",@@ -286,7 +252,7 @@ impl ScalarUDFImpl for DateTruncFunc {return exec_err!("Granularity of `date_trunc` must be non-null scalar Utf8");};- let granularity = DateTruncGranularity::from_str(&granularity_str)?;+ let granularity = parse_granularity(&granularity_str)?;// Check upfront if granularity is valid for Time typeslet is_time_type = matches!(array.data_type(), Time64(_) | Time32(_));@@ -299,7 +265,7 @@ impl ScalarUDFImpl for DateTruncFunc {fn process_array<T: ArrowTimestampType>(array: &dyn Array,- granularity: DateTruncGranularity,+ granularity: DatePart,tz_opt: Option<&Arc<str>>,) -> Result<ColumnarValue> {let parsed_tz = parse_tz(tz_opt)?;@@ -330,7 +296,7 @@ impl ScalarUDFImpl for DateTruncFunc {fn process_scalar<T: ArrowTimestampType>(v: Option<&i64>,- granularity: DateTruncGranularity,+ granularity: DatePart,tz_opt: Option<&Arc<str>>,) -> Result<ColumnarValue> {let parsed_tz = parse_tz(tz_opt)?;@@ -500,81 +466,78 @@ const SECS_PER_MINUTE: i32 = 60;const SECS_PER_HOUR: i32 = 60 * SECS_PER_MINUTE;/// Truncate time in nanoseconds to the specified granularity-fn truncate_time_nanos(value: i64, granularity: DateTruncGranularity) -> i64 {+fn truncate_time_nanos(value: i64, granularity: DatePart) -> i64 {match granularity {- DateTruncGranularity::Hour => value - (value % NANOS_PER_HOUR),- DateTruncGranularity::Minute => value - (value % NANOS_PER_MINUTE),- DateTruncGranularity::Second => value - (value % NANOS_PER_SECOND),- DateTruncGranularity::Millisecond => value - (value % NANOS_PER_MILLISECOND),- DateTruncGranularity::Microsecond => value - (value % NANOS_PER_MICROSECOND),+ DatePart::Hour => value - (value % NANOS_PER_HOUR),+ DatePart::Minute => value - (value % NANOS_PER_MINUTE),+ DatePart::Second => value - (value % NANOS_PER_SECOND),+ DatePart::Millisecond => value - (value % NANOS_PER_MILLISECOND),+ DatePart::Microsecond => value - (value % NANOS_PER_MICROSECOND),// Other granularities are not valid for time - should be caught earlier_ => value,}}/// Truncate time in microseconds to the specified granularity-fn truncate_time_micros(value: i64, granularity: DateTruncGranularity) -> i64 {+fn truncate_time_micros(value: i64, granularity: DatePart) -> i64 {match granularity {- DateTruncGranularity::Hour => value - (value % MICROS_PER_HOUR),- DateTruncGranularity::Minute => value - (value % MICROS_PER_MINUTE),- DateTruncGranularity::Second => value - (value % MICROS_PER_SECOND),- DateTruncGranularity::Millisecond => value - (value % MICROS_PER_MILLISECOND),- DateTruncGranularity::Microsecond => value, // Already at microsecond precision+ DatePart::Hour => value - (value % MICROS_PER_HOUR),+ DatePart::Minute => value - (value % MICROS_PER_MINUTE),+ DatePart::Second => value - (value % MICROS_PER_SECOND),+ DatePart::Millisecond => value - (value % MICROS_PER_MILLISECOND),+ DatePart::Microsecond => value, // Already at microsecond precision// Other granularities are not valid for time_ => value,}}/// Truncate time in milliseconds to the specified granularity-fn truncate_time_millis(value: i32, granularity: DateTruncGranularity) -> i32 {+fn truncate_time_millis(value: i32, granularity: DatePart) -> i32 {match granularity {- DateTruncGranularity::Hour => value - (value % MILLIS_PER_HOUR),- DateTruncGranularity::Minute => value - (value % MILLIS_PER_MINUTE),- DateTruncGranularity::Second => value - (value % MILLIS_PER_SECOND),- DateTruncGranularity::Millisecond => value, // Already at millisecond precision- DateTruncGranularity::Microsecond => value, // Can't truncate to finer precision+ DatePart::Hour => value - (value % MILLIS_PER_HOUR),+ DatePart::Minute => value - (value % MILLIS_PER_MINUTE),+ DatePart::Second => value - (value % MILLIS_PER_SECOND),+ DatePart::Millisecond => value, // Already at millisecond precision+ DatePart::Microsecond => value, // Can't truncate to finer precision// Other granularities are not valid for time_ => value,}}/// Truncate time in seconds to the specified granularity-fn truncate_time_secs(value: i32, granularity: DateTruncGranularity) -> i32 {+fn truncate_time_secs(value: i32, granularity: DatePart) -> i32 {match granularity {- DateTruncGranularity::Hour => value - (value % SECS_PER_HOUR),- DateTruncGranularity::Minute => value - (value % SECS_PER_MINUTE),- DateTruncGranularity::Second => value, // Already at second precision- DateTruncGranularity::Millisecond => value, // Can't truncate to finer precision- DateTruncGranularity::Microsecond => value, // Can't truncate to finer precision+ DatePart::Hour => value - (value % SECS_PER_HOUR),+ DatePart::Minute => value - (value % SECS_PER_MINUTE),+ DatePart::Second => value, // Already at second precision+ DatePart::Millisecond => value, // Can't truncate to finer precision+ DatePart::Microsecond => value, // Can't truncate to finer precision// Other granularities are not valid for time_ => value,}}-fn _date_trunc_coarse<T>(- granularity: DateTruncGranularity,- value: Option<T>,-) -> Result<Option<T>>+fn _date_trunc_coarse<T>(granularity: DatePart, value: Option<T>) -> Result<Option<T>>whereT: Datelike + Timelike + Sub<Duration, Output = T> + Copy,{let value = match granularity {- DateTruncGranularity::Millisecond => value,- DateTruncGranularity::Microsecond => value,- DateTruncGranularity::Second => value.and_then(|d| d.with_nanosecond(0)),- DateTruncGranularity::Minute => value+ DatePart::Millisecond => value,+ DatePart::Microsecond => value,+ DatePart::Second => value.and_then(|d| d.with_nanosecond(0)),+ DatePart::Minute => value.and_then(|d| d.with_nanosecond(0)).and_then(|d| d.with_second(0)),- DateTruncGranularity::Hour => value+ DatePart::Hour => value.and_then(|d| d.with_nanosecond(0)).and_then(|d| d.with_second(0)).and_then(|d| d.with_minute(0)),- DateTruncGranularity::Day => value+ DatePart::Day => value.and_then(|d| d.with_nanosecond(0)).and_then(|d| d.with_second(0)).and_then(|d| d.with_minute(0)).and_then(|d| d.with_hour(0)),- DateTruncGranularity::Week => value+ DatePart::Week => value.and_then(|d| d.with_nanosecond(0)).and_then(|d| d.with_second(0)).and_then(|d| d.with_minute(0))@@ -582,26 +545,27 @@ where.map(|d| {d - TimeDelta::try_seconds(60 * 60 * 24 * d.weekday() as i64).unwrap()}),- DateTruncGranularity::Month => value+ DatePart::Month => value.and_then(|d| d.with_nanosecond(0)).and_then(|d| d.with_second(0)).and_then(|d| d.with_minute(0)).and_then(|d| d.with_hour(0)).and_then(|d| d.with_day0(0)),- DateTruncGranularity::Quarter => value+ DatePart::Quarter => value.and_then(|d| d.with_nanosecond(0)).and_then(|d| d.with_second(0)).and_then(|d| d.with_minute(0)).and_then(|d| d.with_hour(0)).and_then(|d| d.with_day0(0)).and_then(|d| d.with_month(quarter_month(&d))),- DateTruncGranularity::Year => value+ DatePart::Year => value.and_then(|d| d.with_nanosecond(0)).and_then(|d| d.with_second(0)).and_then(|d| d.with_minute(0)).and_then(|d| d.with_hour(0)).and_then(|d| d.with_day0(0)).and_then(|d| d.with_month0(0)),+ _ => unreachable!("unsupported date_trunc granularity"),};Ok(value)}@@ -614,7 +578,7 @@ where}fn _date_trunc_coarse_with_tz(- granularity: DateTruncGranularity,+ granularity: DatePart,value: DateTime<Tz>,) -> Result<Option<i64>> {let local = value.naive_local();@@ -715,10 +679,7 @@ fn days_from_civil(year: i64, month: i64, day: i64) -> i64 {/// Returns `None` when the truncated timestamp is no longer representable as/// nanoseconds since the epoch, which the caller reports as an out of range/// error.-fn _date_trunc_coarse_without_tz(- granularity: DateTruncGranularity,- value: i64,-) -> Option<i64> {+fn _date_trunc_coarse_without_tz(granularity: DatePart, value: i64) -> Option<i64> {let truncate_to = |unit: i64| value.checked_sub(value.rem_euclid(unit));let days = || value.div_euclid(NANOS_PER_DAY);let nanos_from_days = |days: i64| days.checked_mul(NANOS_PER_DAY);@@ -726,31 +687,30 @@ fn _date_trunc_coarse_without_tz(match granularity {// Sub-second granularities are applied by the caller, which rescales// the nanoseconds to the time unit of the array.- DateTruncGranularity::Millisecond | DateTruncGranularity::Microsecond => {- Some(value)- }- DateTruncGranularity::Second => truncate_to(NANOS_PER_SECOND),- DateTruncGranularity::Minute => truncate_to(NANOS_PER_MINUTE),- DateTruncGranularity::Hour => truncate_to(NANOS_PER_HOUR),- DateTruncGranularity::Day => nanos_from_days(days()),- DateTruncGranularity::Week => {+ DatePart::Millisecond | DatePart::Microsecond => Some(value),+ DatePart::Second => truncate_to(NANOS_PER_SECOND),+ DatePart::Minute => truncate_to(NANOS_PER_MINUTE),+ DatePart::Hour => truncate_to(NANOS_PER_HOUR),+ DatePart::Day => nanos_from_days(days()),+ DatePart::Week => {let days = days();// `Weekday::num_days_from_monday` for the epoch (a Thursday) is 3.nanos_from_days(days - (days + 3).rem_euclid(7))}- DateTruncGranularity::Month => {+ DatePart::Month => {let days = days();let (_, _, day_of_month) = civil_from_days(days);nanos_from_days(days - (day_of_month - 1))}- DateTruncGranularity::Quarter => {+ DatePart::Quarter => {let (year, month, _) = civil_from_days(days());nanos_from_days(days_from_civil(year, 1 + 3 * ((month - 1) / 3), 1))}- DateTruncGranularity::Year => {+ DatePart::Year => {let (year, _, _) = civil_from_days(days());nanos_from_days(days_from_civil(year, 1, 1))}+ _ => unreachable!("unsupported date_trunc granularity"),}}@@ -758,11 +718,7 @@ fn _date_trunc_coarse_without_tz(/// epoch, for granularities greater than 1 second, in taking into/// account that some granularities are not uniform durations of time/// (e.g. months are not always the same lengths, leap seconds, etc)-fn date_trunc_coarse(- granularity: DateTruncGranularity,- value: i64,- tz: Option<Tz>,-) -> Result<i64> {+fn date_trunc_coarse(granularity: DatePart, value: i64, tz: Option<Tz>) -> Result<i64> {let input = value;let value = match tz {Some(tz) => {@@ -776,6 +732,7 @@ fn date_trunc_coarse(};value.ok_or_else(|| {+ let granularity = granularity.to_string().to_lowercase();exec_datafusion_err!("Timestamp {input} out of range after truncating to {granularity}")@@ -791,31 +748,31 @@ fn date_trunc_coarse(fn general_date_trunc_array_fine_granularity<T: ArrowTimestampType>(tu: TimeUnit,array: &PrimitiveArray<T>,- granularity: DateTruncGranularity,+ granularity: DatePart,tz_opt: Option<Arc<str>>,) -> Result<ArrayRef> {let unit = match (tu, granularity) {- (Second, DateTruncGranularity::Minute) => NonZeroI64::new(60),- (Second, DateTruncGranularity::Hour) => NonZeroI64::new(3600),- (Second, DateTruncGranularity::Day) => NonZeroI64::new(86400),-- (Millisecond, DateTruncGranularity::Second) => NonZeroI64::new(1_000),- (Millisecond, DateTruncGranularity::Minute) => NonZeroI64::new(60_000),- (Millisecond, DateTruncGranularity::Hour) => NonZeroI64::new(3_600_000),- (Millisecond, DateTruncGranularity::Day) => NonZeroI64::new(86_400_000),-- (Microsecond, DateTruncGranularity::Millisecond) => NonZeroI64::new(1_000),- (Microsecond, DateTruncGranularity::Second) => NonZeroI64::new(1_000_000),- (Microsecond, DateTruncGranularity::Minute) => NonZeroI64::new(60_000_000),- (Microsecond, DateTruncGranularity::Hour) => NonZeroI64::new(3_600_000_000),- (Microsecond, DateTruncGranularity::Day) => NonZeroI64::new(86_400_000_000),-- (Nanosecond, DateTruncGranularity::Microsecond) => NonZeroI64::new(1_000),- (Nanosecond, DateTruncGranularity::Millisecond) => NonZeroI64::new(1_000_000),- (Nanosecond, DateTruncGranularity::Second) => NonZeroI64::new(1_000_000_000),- (Nanosecond, DateTruncGranularity::Minute) => NonZeroI64::new(60_000_000_000),- (Nanosecond, DateTruncGranularity::Hour) => NonZeroI64::new(3_600_000_000_000),- (Nanosecond, DateTruncGranularity::Day) => NonZeroI64::new(86_400_000_000_000),+ (Second, DatePart::Minute) => NonZeroI64::new(60),+ (Second, DatePart::Hour) => NonZeroI64::new(3600),+ (Second, DatePart::Day) => NonZeroI64::new(86400),++ (Millisecond, DatePart::Second) => NonZeroI64::new(1_000),+ (Millisecond, DatePart::Minute) => NonZeroI64::new(60_000),+ (Millisecond, DatePart::Hour) => NonZeroI64::new(3_600_000),+ (Millisecond, DatePart::Day) => NonZeroI64::new(86_400_000),++ (Microsecond, DatePart::Millisecond) => NonZeroI64::new(1_000),+ (Microsecond, DatePart::Second) => NonZeroI64::new(1_000_000),+ (Microsecond, DatePart::Minute) => NonZeroI64::new(60_000_000),+ (Microsecond, DatePart::Hour) => NonZeroI64::new(3_600_000_000),+ (Microsecond, DatePart::Day) => NonZeroI64::new(86_400_000_000),++ (Nanosecond, DatePart::Microsecond) => NonZeroI64::new(1_000),+ (Nanosecond, DatePart::Millisecond) => NonZeroI64::new(1_000_000),+ (Nanosecond, DatePart::Second) => NonZeroI64::new(1_000_000_000),+ (Nanosecond, DatePart::Minute) => NonZeroI64::new(60_000_000_000),+ (Nanosecond, DatePart::Hour) => NonZeroI64::new(3_600_000_000_000),+ (Nanosecond, DatePart::Day) => NonZeroI64::new(86_400_000_000_000),_ => None,};@@ -841,7 +798,7 @@ fn general_date_trunc(tu: TimeUnit,value: i64,tz: Option<Tz>,- granularity: DateTruncGranularity,+ granularity: DatePart,) -> Result<i64, DataFusionError> {let scale = match tu {Second => 1_000_000_000,@@ -861,29 +818,25 @@ fn general_date_trunc(let result = match tu {Second => match granularity {- DateTruncGranularity::Minute => nano / 1_000_000_000 / 60 * 60,+ DatePart::Minute => nano / 1_000_000_000 / 60 * 60,_ => nano / 1_000_000_000,},Millisecond => match granularity {- DateTruncGranularity::Minute => nano / 1_000_000 / 1_000 / 60 * 1_000 * 60,- DateTruncGranularity::Second => nano / 1_000_000 / 1_000 * 1_000,+ DatePart::Minute => nano / 1_000_000 / 1_000 / 60 * 1_000 * 60,+ DatePart::Second => nano / 1_000_000 / 1_000 * 1_000,_ => nano / 1_000_000,},Microsecond => match granularity {- DateTruncGranularity::Minute => {- nano / 1_000 / 1_000_000 / 60 * 60 * 1_000_000- }- DateTruncGranularity::Second => nano / 1_000 / 1_000_000 * 1_000_000,- DateTruncGranularity::Millisecond => nano / 1_000 / 1_000 * 1_000,+ DatePart::Minute => nano / 1_000 / 1_000_000 / 60 * 60 * 1_000_000,+ DatePart::Second => nano / 1_000 / 1_000_000 * 1_000_000,+ DatePart::Millisecond => nano / 1_000 / 1_000 * 1_000,_ => nano / 1_000,},_ => match granularity {- DateTruncGranularity::Minute => {- nano / 1_000_000_000 / 60 * 1_000_000_000 * 60- }- DateTruncGranularity::Second => nano / 1_000_000_000 * 1_000_000_000,- DateTruncGranularity::Millisecond => nano / 1_000_000 * 1_000_000,- DateTruncGranularity::Microsecond => nano / 1_000 * 1_000,+ DatePart::Minute => nano / 1_000_000_000 / 60 * 1_000_000_000 * 60,+ DatePart::Second => nano / 1_000_000_000 * 1_000_000_000,+ DatePart::Millisecond => nano / 1_000_000 * 1_000_000,+ DatePart::Microsecond => nano / 1_000 * 1_000,_ => nano,},};@@ -903,12 +856,13 @@ mod tests {use std::sync::Arc;use crate::datetime::date_trunc::{- DateTruncFunc, DateTruncGranularity, date_trunc_coarse,+ DateTruncFunc, date_trunc_coarse, parse_granularity,};use arrow::array::cast::as_primitive_array;use arrow::array::types::TimestampNanosecondType;use arrow::array::{Array, TimestampNanosecondArray};+ use arrow::compute::DatePart;use arrow::compute::kernels::cast_utils::string_to_timestamp_nanos;use arrow::datatypes::{DataType, Field, TimeUnit};use datafusion_common::ScalarValue;@@ -1005,7 +959,7 @@ mod tests {for (original, granularity, expected) in &cases {let left = string_to_timestamp_nanos(original).unwrap();let right = string_to_timestamp_nanos(expected).unwrap();- let granularity_enum = DateTruncGranularity::from_str(granularity).unwrap();+ let granularity_enum = parse_granularity(granularity).unwrap();let result = date_trunc_coarse(granularity_enum, left, None).unwrap();assert_eq!(result, right, "{original} = {expected}");}@@ -1014,7 +968,7 @@ mod tests {#[test]fn date_trunc_out_of_range_lower_bound_returns_error() {let timestamp = string_to_timestamp_nanos("1677-09-22T00:00:00Z").unwrap();- let err = date_trunc_coarse(DateTruncGranularity::Year, timestamp, None)+ let err = date_trunc_coarse(DatePart::Year, timestamp, None).unwrap_err().to_string();diff --git a/datafusion/spark/src/function/datetime/date_part.rs b/datafusion/spark/src/function/datetime/date_part.rsindex 91bdb9a553..50ef1ece9d 100644--- a/datafusion/spark/src/function/datetime/date_part.rs+++ b/datafusion/spark/src/function/datetime/date_part.rs@@ -109,10 +109,10 @@ impl ScalarUDFImpl for SparkDatePart {}};- // Map Spark-specific date part aliases to datafusion ones+ // Map Spark-specific date part aliases to DataFusion ones.let part = match part.as_str() {"yearofweek" | "year_iso" => "isoyear",- "dayofweek" => "dow",+ "dayofweek" | "dow" => "dow1","dayofweek_iso" | "dow_iso" => "isodow",other => other,};@@ -124,15 +124,6 @@ impl ScalarUDFImpl for SparkDatePart {vec![part_expr, date_expr],));- match part {- // Spark's `dayofweek` is 1..=7 (Sun=1) but df's `dow` is 0..=6- // (Sun=0); shift by +1. df's `isodow` already returns the- // PG-correct 1..=7 (Mon=1), which matches Spark's- // `dayofweek_iso`/`dow_iso`, so no shift is needed there.- "dow" => Ok(ExprSimplifyResult::Simplified(- date_part_expr + Expr::Literal(ScalarValue::Int32(Some(1)), None),- )),- _ => Ok(ExprSimplifyResult::Simplified(date_part_expr)),- }+ Ok(ExprSimplifyResult::Simplified(date_part_expr))}}diff --git a/datafusion/spark/src/function/datetime/time_trunc.rs b/datafusion/spark/src/function/datetime/time_trunc.rsindex a66b8e9468..577c19733a 100644--- a/datafusion/spark/src/function/datetime/time_trunc.rs+++ b/datafusion/spark/src/function/datetime/time_trunc.rs@@ -90,24 +90,15 @@ impl ScalarUDFImpl for SparkTimeTrunc {) -> Result<ExprSimplifyResult> {let fmt_expr = &args[0];- let fmt = match fmt_expr.as_literal() {- Some(ScalarValue::Utf8(Some(v)))- | Some(ScalarValue::Utf8View(Some(v)))- | Some(ScalarValue::LargeUtf8(Some(v))) => v.to_lowercase(),+ match fmt_expr.as_literal() {+ Some(ScalarValue::Utf8(Some(_)))+ | Some(ScalarValue::Utf8View(Some(_)))+ | Some(ScalarValue::LargeUtf8(Some(_))) => {}_ => {return plan_err!("First argument of `TIME_TRUNC` must be non-null scalar Utf8");}- };-- if !matches!(- fmt.as_str(),- "hour" | "minute" | "second" | "millisecond" | "microsecond"- ) {- return plan_err!(- "The format argument of `TIME_TRUNC` must be one of: hour, minute, second, millisecond, microsecond"- );}Ok(ExprSimplifyResult::Simplified(Expr::ScalarFunction(diff --git a/datafusion/spark/src/function/datetime/trunc.rs b/datafusion/spark/src/function/datetime/trunc.rsindex 9d7da5969a..2c18e3909f 100644--- a/datafusion/spark/src/function/datetime/trunc.rs+++ b/datafusion/spark/src/function/datetime/trunc.rs@@ -15,8 +15,9 @@// specific language governing permissions and limitations// under the License.-use std::sync::Arc;+use std::{str::FromStr, sync::Arc};+use arrow::compute::DatePart;use arrow::datatypes::{DataType, Field, FieldRef, TimeUnit};use datafusion_common::types::{NativeType, logical_date, logical_string};use datafusion_common::utils::take_function_args;@@ -109,13 +110,22 @@ impl ScalarUDFImpl for SparkTrunc {let fmt = match fmt.as_str() {"yy" | "yyyy" => "year","mm" | "mon" => "month",- "year" | "month" | "day" | "week" | "quarter" => fmt.as_str(),- _ => {- return plan_err!(- "The format argument of `TRUNC` must be one of: year, yy, yyyy, month, mm, mon, day, week, quarter."- );- }+ other => other,};++ // Accept shared DatePart aliases for the date parts supported by TRUNC.+ if !matches!(+ DatePart::from_str(fmt),+ Ok(DatePart::Year+ | DatePart::Month+ | DatePart::Day+ | DatePart::Week+ | DatePart::Quarter)+ ) {+ return plan_err!(+ "The format argument of `TRUNC` must represent a year, month, day, week, or quarter."+ );+ }let return_type = dt_expr.get_type(info.schema())?;let fmt_expr = Expr::Literal(ScalarValue::new_utf8(fmt), None);diff --git a/datafusion/sql/src/unparser/expr.rs b/datafusion/sql/src/unparser/expr.rsindex cd3ac1f3a4..b33c366d00 100644--- a/datafusion/sql/src/unparser/expr.rs+++ b/datafusion/sql/src/unparser/expr.rs@@ -2931,11 +2931,17 @@ mod tests {"MONTH","EXTRACT(MONTH FROM x)",),+ (+ DateFieldExtractStyle::Extract,+ "MONS",+ "EXTRACT(MONTH FROM x)",+ ),(DateFieldExtractStyle::Strftime,"MONTH","strftime('%m', x)",),+ (DateFieldExtractStyle::Strftime, "YRS", "strftime('%Y', x)"),(DateFieldExtractStyle::DatePart,"DAY",diff --git a/datafusion/sql/src/unparser/utils.rs b/datafusion/sql/src/unparser/utils.rsindex 240032d26c..86f3e23115 100644--- a/datafusion/sql/src/unparser/utils.rs+++ b/datafusion/sql/src/unparser/utils.rs@@ -15,7 +15,7 @@// specific language governing permissions and limitations// under the License.-use std::{cmp::Ordering, sync::Arc, vec};+use std::{cmp::Ordering, str::FromStr, sync::Arc, vec};use super::{Unparser, dialect::CharacterLengthStyle, dialect::DateFieldExtractStyle,@@ -31,6 +31,7 @@ use datafusion_expr::{Window, expr, utils::grouping_set_to_exprlist,};+use arrow::compute::DatePart;use indexmap::IndexSet;use sqlparser::ast;use sqlparser::tokenizer::Span;@@ -455,13 +456,13 @@ pub(crate) fn date_part_to_sql((DateFieldExtractStyle::Extract, 2) => {let date_expr = unparser.expr_to_sql_with_nesting(&date_part_args[1])?;if let Expr::Literal(ScalarValue::Utf8(Some(field)), _) = &date_part_args[0] {- let field = match field.to_lowercase().as_str() {- "year" => ast::DateTimeField::Year,- "month" => ast::DateTimeField::Month,- "day" => ast::DateTimeField::Day,- "hour" => ast::DateTimeField::Hour,- "minute" => ast::DateTimeField::Minute,- "second" => ast::DateTimeField::Second,+ let field = match DatePart::from_str(field) {+ Ok(DatePart::Year) => ast::DateTimeField::Year,+ Ok(DatePart::Month) => ast::DateTimeField::Month,+ Ok(DatePart::Day) => ast::DateTimeField::Day,+ Ok(DatePart::Hour) => ast::DateTimeField::Hour,+ Ok(DatePart::Minute) => ast::DateTimeField::Minute,+ Ok(DatePart::Second) => ast::DateTimeField::Second,_ => return Ok(None),};@@ -476,13 +477,13 @@ pub(crate) fn date_part_to_sql(let column = unparser.expr_to_sql_with_nesting(&date_part_args[1])?;if let Expr::Literal(ScalarValue::Utf8(Some(field)), _) = &date_part_args[0] {- let field = match field.to_lowercase().as_str() {- "year" => "%Y",- "month" => "%m",- "day" => "%d",- "hour" => "%H",- "minute" => "%M",- "second" => "%S",+ let field = match DatePart::from_str(field) {+ Ok(DatePart::Year) => "%Y",+ Ok(DatePart::Month) => "%m",+ Ok(DatePart::Day) => "%d",+ Ok(DatePart::Hour) => "%H",+ Ok(DatePart::Minute) => "%M",+ Ok(DatePart::Second) => "%S",_ => return Ok(None),};diff --git a/datafusion/sqllogictest/test_files/datetime/date_part.slt b/datafusion/sqllogictest/test_files/datetime/date_part.sltindex 0a992b2d78..c13ee264ef 100644--- a/datafusion/sqllogictest/test_files/datetime/date_part.slt+++ b/datafusion/sqllogictest/test_files/datetime/date_part.slt@@ -288,6 +288,15 @@ SELECT date_part('QUARTER', CAST('2000-01-01' AS DATE))----1+query IIII+SELECT+ date_part('QUARTERS', CAST('2000-01-01' AS DATE)),+ date_part('ISOWEEK', CAST('2000-01-01' AS DATE)),+ date_part('DAYOFYEAR', CAST('2000-01-01' AS DATE)),+ date_part('DAYOFWEEK', CAST('2000-01-01' AS DATE))+----+1 52 1 6+query ISELECT EXTRACT(quarter FROM to_timestamp('2020-09-08T12:00:00+00:00'))----diff --git a/datafusion/sqllogictest/test_files/datetime/dates.slt b/datafusion/sqllogictest/test_files/datetime/dates.sltindex a6a5f480f7..6fab426c3c 100644--- a/datafusion/sqllogictest/test_files/datetime/dates.slt+++ b/datafusion/sqllogictest/test_files/datetime/dates.slt@@ -448,11 +448,11 @@ select to_date('2022-01-23', '%Y-%m-%d');2022-01-23# invalid date_trunc format-query error DataFusion error: Execution error: Unsupported date_trunc granularity: ''. Supported values are: microsecond, millisecond, second, minute, hour, day, week, month, quarter, year+query error DataFusion error: Execution error: Unsupported date_trunc granularity: ''. Supported granularities are: microsecond, millisecond, second, minute, hour, day, week, month, quarter, yearSELECT date_trunc('', to_date('2022-02-23', '%Y-%m-%d'))# invalid date_trunc format-query error DataFusion error: Execution error: Unsupported date_trunc granularity: 'invalid'. Supported values are: microsecond, millisecond, second, minute, hour, day, week, month, quarter, year+query error DataFusion error: Execution error: Unsupported date_trunc granularity: 'invalid'. Supported granularities are: microsecond, millisecond, second, minute, hour, day, week, month, quarter, yearSELECT date_trunc('invalid', to_date('2022-02-23', '%Y-%m-%d'))query PPPPdiff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.sltindex d73bc6eb06..e47a5f0b23 100644--- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt+++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt@@ -1407,6 +1407,16 @@ SELECT DATE_TRUNC('MONTH', TIMESTAMP '2022-08-03 14:38:50Z');----2022-08-01T00:00:00+# date_trunc uses the same field-name aliases as date_part+query PPPP+SELECT+ DATE_TRUNC('mon', TIMESTAMP '2022-08-03 14:38:50Z'),+ DATE_TRUNC('months', TIMESTAMP '2022-08-03 14:38:50Z'),+ DATE_TRUNC('qtr', TIMESTAMP '2022-08-03 14:38:50Z'),+ DATE_TRUNC('yrs', TIMESTAMP '2022-08-03 14:38:50Z');+----+2022-08-01T00:00:00 2022-08-01T00:00:00 2022-07-01T00:00:00 2022-01-01T00:00:00+query PSELECT DATE_TRUNC('month', NULL);----diff --git a/datafusion/sqllogictest/test_files/spark/datetime/date_part.slt b/datafusion/sqllogictest/test_files/spark/datetime/date_part.sltindex 48216bd551..106f60bc07 100644--- a/datafusion/sqllogictest/test_files/spark/datetime/date_part.slt+++ b/datafusion/sqllogictest/test_files/spark/datetime/date_part.slt@@ -58,6 +58,11 @@ SELECT date_part('QTR'::string, '2000-01-01'::date);----1+query I+SELECT date_part('QUARTERS'::string, '2000-01-01'::date);+----+1+# MONTHquery ISELECT date_part('MONTH'::string, '2000-01-01'::date);@@ -139,6 +144,11 @@ SELECT date_part('DOY'::string, '2000-01-01'::date);----1+query I+SELECT date_part('DAYOFYEAR'::string, '2000-01-01'::date);+----+1+# HOURquery ISELECT date_part('HOUR'::string, '2000-01-01 12:30:45'::timestamp);diff --git a/datafusion/sqllogictest/test_files/spark/datetime/date_trunc.slt b/datafusion/sqllogictest/test_files/spark/datetime/date_trunc.sltindex 7fc1583bb9..499ac75f76 100644--- a/datafusion/sqllogictest/test_files/spark/datetime/date_trunc.slt+++ b/datafusion/sqllogictest/test_files/spark/datetime/date_trunc.slt@@ -53,6 +53,11 @@ SELECT date_trunc('MON', '2015-03-05T09:32:05.123456'::timestamp);----2015-03-01T00:00:00+query P+SELECT date_trunc('MONTHS', '2015-03-05T09:32:05.123456'::timestamp);+----+2015-03-01T00:00:00+# WEEK - truncate to Monday of the week, time zeroedquery PSELECT date_trunc('WEEK', '2015-03-05T09:32:05.123456'::timestamp);@@ -129,7 +134,7 @@ SELECT date_trunc('YEAR', NULL::timestamp);NULL# incorrect format-query error DataFusion error: Execution error: Unsupported date_trunc granularity: 'test'. Supported values are: microsecond, millisecond, second, minute, hour, day, week, month, quarter, year+query error DataFusion error: Execution error: Unsupported date_trunc granularity: 'test'. Supported granularities are: microsecond, millisecond, second, minute, hour, day, week, month, quarter, yearSELECT date_trunc('test', '2015-03-05T09:32:05.123456');# Timezone handling - Spark-compatible behaviordiff --git a/datafusion/sqllogictest/test_files/spark/datetime/time_trunc.slt b/datafusion/sqllogictest/test_files/spark/datetime/time_trunc.sltindex 35ffa483bb..11f4704520 100644--- a/datafusion/sqllogictest/test_files/spark/datetime/time_trunc.slt+++ b/datafusion/sqllogictest/test_files/spark/datetime/time_trunc.slt@@ -27,6 +27,11 @@ SELECT time_trunc('MINUTE', '09:32:05.123456'::time);----09:32:00+query D+SELECT time_trunc('MINS', '09:32:05.123456'::time);+----+09:32:00+# SECOND - zero out fractionquery DSELECT time_trunc('SECOND', '09:32:05.123456'::time);@@ -69,5 +74,5 @@ SELECT time_trunc('HOUR', NULL::time);NULL# incorrect format-query error DataFusion error: Optimizer rule 'simplify_expressions' failed\ncaused by\nError during planning: The format argument of `TIME_TRUNC` must be one of: hour, minute, second, millisecond, microsecond+query error DataFusion error: Execution error: Unsupported date_trunc granularity: 'test'. Supported granularities are: microsecond, millisecond, second, minute, hour, day, week, month, quarter, yearSELECT time_trunc('test', '09:32:05.123456'::time);diff --git a/datafusion/sqllogictest/test_files/spark/datetime/trunc.slt b/datafusion/sqllogictest/test_files/spark/datetime/trunc.sltindex aa26d7bd0e..f79bfffc7d 100644--- a/datafusion/sqllogictest/test_files/spark/datetime/trunc.slt+++ b/datafusion/sqllogictest/test_files/spark/datetime/trunc.slt@@ -53,6 +53,11 @@ SELECT trunc('2009-02-12'::date, 'MON'::string);----2009-02-01+query D+SELECT trunc('2009-02-12'::date, 'MONTHS'::string);+----+2009-02-01+# WEEK - truncate to Monday of the weekquery DSELECT trunc('2009-02-12'::date, 'WEEK'::string);@@ -88,5 +93,5 @@ query error DataFusion error: Optimizer rule 'simplify_expressions' failed\ncausSELECT trunc('2009-02-12'::date, NULL::string);# incorrect format-query error DataFusion error: Optimizer rule 'simplify_expressions' failed\ncaused by\nError during planning: The format argument of `TRUNC` must be one of: year, yy, yyyy, month, mm, mon, day, week, quarter.+query error DataFusion error: Optimizer rule 'simplify_expressions' failed\ncaused by\nError during planning: The format argument of `TRUNC` must represent a year, month, day, week, or quarter.SELECT trunc('2009-02-12'::date, 'test'::string);