Apache DataFusion SQL Query Engine
fix: date_part sub-second parts and result nullability, reusing arrow kernels (#26686)
Matt Butrovich8ea0dbfc341e
diff --git a/datafusion/functions/benches/datetime_expressions/date_part.rs b/datafusion/functions/benches/datetime_expressions/date_part.rsindex 219f75f45e..073a5fdd93 100644--- a/datafusion/functions/benches/datetime_expressions/date_part.rs+++ b/datafusion/functions/benches/datetime_expressions/date_part.rs@@ -188,6 +188,36 @@ fn generate_duration_nanosecond_array_with_nulls()}+fn generate_timestamp_ns_array_with_nulls(rng: &mut StdRng) -> TimestampNanosecondArray {+ TimestampNanosecondArray::from(+ (0..BATCH_SIZE)+ .map(|_| {+ if rng.random_bool(NULL_DENSITY) {+ None+ } else {+ Some(rng.random_range(0..TS_BOUND * 1_000_000_000))+ }+ })+ .collect::<Vec<_>>(),+ )+}++fn generate_time64_nanosecond_array_with_nulls(+ rng: &mut StdRng,+) -> Time64NanosecondArray {+ Time64NanosecondArray::from(+ (0..BATCH_SIZE)+ .map(|_| {+ if rng.random_bool(NULL_DENSITY) {+ None+ } else {+ Some(rng.random_range(0..SEC_DAY * 1_000_000_000))+ }+ })+ .collect::<Vec<_>>(),+ )+}+fn bench_date_part(c: &mut Criterion,udf: &Arc<ScalarUDF>,@@ -244,6 +274,14 @@ fn criterion_benchmark(c: &mut Criterion) {let duration_ns_nulls =Arc::new(generate_duration_nanosecond_array_with_nulls(&mut rng)) as ArrayRef;+ let ts_ns_tz =+ Arc::new(generate_timestamp_ns_array(&mut rng).with_timezone("America/New_York"))+ as ArrayRef;+ let ts_ns_nulls =+ Arc::new(generate_timestamp_ns_array_with_nulls(&mut rng)) as ArrayRef;+ let time64_ns_nulls =+ Arc::new(generate_time64_nanosecond_array_with_nulls(&mut rng)) as ArrayRef;+let udf = date_part();for part in ["year", "month", "week", "day", "hour", "minute"] {@@ -365,6 +403,41 @@ fn criterion_benchmark(c: &mut Criterion) {);}+ // Durations past `i32::MAX` milliseconds or microseconds overflow `Int32`, so+ // only `nanosecond` runs on them+ for (part, return_type) in [+ ("millisecond", DataType::Int32),+ ("microsecond", DataType::Int32),+ ("nanosecond", DataType::Int64),+ ] {+ let mut arrays = vec![+ ("ns_tz", &ts_ns_tz),+ ("ns_nulls", &ts_ns_nulls),+ ("time32_ms", &time32_ms),+ ("time64_ns", &time64_ns),+ ("time64_ns_nulls", &time64_ns_nulls),+ ("interval_dt", &interval_dt),+ ("interval_mdn", &interval_mdn),+ ("duration_ns_small", &duration_ns_small),+ ];+ if part == "nanosecond" {+ arrays.extend([+ ("duration_ns", &duration_ns),+ ("duration_ns_nulls", &duration_ns_nulls),+ ]);+ }+ for (name, array) in arrays {+ bench_date_part(+ c,+ &udf,+ &format!("date_part_{part}_{name}_1000"),+ part,+ Arc::clone(array),+ return_type.clone(),+ );+ }+ }+for part in ["quarter", "isoyear", "doy", "dow", "isodow"] {bench_date_part(c,diff --git a/datafusion/functions/src/datetime/date_part.rs b/datafusion/functions/src/datetime/date_part.rsindex 6dcb948a0e..919f1b0ac8 100644--- a/datafusion/functions/src/datetime/date_part.rs+++ b/datafusion/functions/src/datetime/date_part.rs@@ -15,25 +15,33 @@// specific language governing permissions and limitations// under the License.-use std::iter::repeat_n;use std::str::FromStr;use std::sync::Arc;-use arrow::array::temporal_conversions::as_datetime_with_timezone;+use arrow::array::temporal_conversions::{+ MICROSECONDS, MILLISECONDS, NANOSECONDS, SECONDS_IN_DAY, as_datetime_with_timezone,+ timestamp_ms_to_datetime, timestamp_ns_to_datetime, timestamp_s_to_datetime,+ timestamp_us_to_datetime,+};use arrow::array::timezone::Tz;-use arrow::array::{Array, ArrayRef, Float64Array, Int32Array, Int64Array};-use arrow::compute::{DatePart, binary, date_part};+use arrow::array::{+ Array, ArrayRef, Float64Array, Int32Array, Int64Array, PrimitiveArray,+};+use arrow::compute::{DatePart, cast, date_part};use arrow::datatypes::DataType::{Date32, Date64, Duration, Interval, Time32, Time64, Timestamp,};use arrow::datatypes::TimeUnit::{Microsecond, Millisecond, Nanosecond, Second};use arrow::datatypes::{- ArrowTimestampType, DataType, Field, FieldRef, IntervalUnit as ArrowIntervalUnit,- TimeUnit, TimestampMicrosecondType, TimestampMillisecondType,- TimestampNanosecondType, TimestampSecondType,+ ArrowPrimitiveType, ArrowTimestampType, DataType, Field, FieldRef, Int32Type,+ Int64Type, IntervalUnit as ArrowIntervalUnit, Time32MillisecondType,+ Time32SecondType, Time64MicrosecondType, Time64NanosecondType,+ TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType,+ TimestampSecondType,};-use chrono::{Datelike, NaiveDate, Offset};+use chrono::{Datelike, NaiveDate, NaiveDateTime, Offset, TimeZone, Timelike, Utc};use datafusion_common::types::{NativeType, logical_date};+use num_traits::AsPrimitive;use crate::datetime::common::parse_tz;use datafusion_common::{@@ -41,14 +49,13 @@ use datafusion_common::{cast::{as_date32_array, as_date64_array, as_duration_microsecond_array,as_duration_millisecond_array, as_duration_nanosecond_array,- as_duration_second_array, as_int32_array, as_interval_dt_array,- as_interval_mdn_array, as_interval_ym_array, as_primitive_array,- as_time32_millisecond_array, as_time32_second_array, as_time64_microsecond_array,- as_time64_nanosecond_array, as_timestamp_microsecond_array,- as_timestamp_millisecond_array, as_timestamp_nanosecond_array,- as_timestamp_second_array,+ as_duration_second_array, as_interval_dt_array, as_interval_mdn_array,+ as_interval_ym_array, as_primitive_array, as_time32_millisecond_array,+ as_time32_second_array, as_time64_microsecond_array, as_time64_nanosecond_array,+ as_timestamp_microsecond_array, as_timestamp_millisecond_array,+ as_timestamp_nanosecond_array, as_timestamp_second_array,},- exec_err, internal_err, not_impl_err,+ exec_err, internal_err,types::logical_string,utils::take_function_args,};@@ -191,13 +198,16 @@ impl ScalarUDFImpl for DatePartFunc {.flatten().filter(|s| !s.is_empty()).map(|part| {+ // The integer parts are NULL for values arrow can't+ // convert or that overflow, even when the input is+ // not NULLif is_epoch(part) {Field::new(self.name(), DataType::Float64, nullable)} else if is_nanosecond(part) {- // See notes on [seconds_ns] for rationale- Field::new(self.name(), DataType::Int64, nullable)+ // See notes on [subsecond_part] for rationale+ Field::new(self.name(), DataType::Int64, true)} else {- Field::new(self.name(), DataType::Int32, nullable)+ Field::new(self.name(), DataType::Int32, true)}})})@@ -232,10 +242,11 @@ impl ScalarUDFImpl for DatePartFunc {let part_trim = part_normalization(&part);let arr = match DatePart::from_str(part_trim) {- Ok(DatePart::Second) => seconds_as_i32(array.as_ref(), Second)?,- Ok(DatePart::Millisecond) => seconds_as_i32(array.as_ref(), Millisecond)?,- Ok(DatePart::Microsecond) => seconds_as_i32(array.as_ref(), Microsecond)?,- Ok(DatePart::Nanosecond) => seconds_ns(array.as_ref())?,+ Ok(+ part @ (DatePart::Millisecond+ | DatePart::Microsecond+ | DatePart::Nanosecond),+ ) => subsecond_part(array.as_ref(), part)?,Ok(part) => date_part(array.as_ref(), part)?,Err(_) if is_epoch(part_trim) => epoch(array.as_ref())?,// `timezone`, `timezone_hour` and `timezone_minute` have no@@ -440,72 +451,6 @@ fn part_normalization(part: &str) -> &str {.unwrap_or(part)}-/// Invoke [`date_part`] on an `array` (e.g. Timestamp) and convert the-/// result to a total number of seconds, milliseconds, microseconds or-/// nanoseconds as an `Int32Array`-fn seconds_as_i32(array: &dyn Array, unit: TimeUnit) -> Result<ArrayRef> {- // Nanosecond is neither supported in Postgres nor DuckDB, to avoid dealing- // with overflow and precision issue we don't support nanosecond- if unit == Nanosecond {- return not_impl_err!("Date part {unit:?} not supported");- }-- // Fast path with seconds - no need to compute nanoseconds- if unit == Second {- return Ok(date_part(array, DatePart::Second)?);- }-- // Fast path for Date32 and Date64 - no seconds- if array.data_type() == &Date32 || array.data_type() == &Date64 {- return Ok(Arc::new(Int32Array::from_iter_values_with_nulls(- repeat_n(0, array.len()),- array.nulls().cloned(),- )));- }-- let conversion_factor = match unit {- Second => 1_000_000_000,- Millisecond => 1_000_000,- Microsecond => 1_000,- Nanosecond => 1,- };-- let second_factor = match unit {- Second => 1,- Millisecond => 1_000,- Microsecond => 1_000_000,- Nanosecond => 1_000_000_000,- };-- let secs = date_part(array, DatePart::Second)?;- // This assumes array is primitive and not a dictionary- let secs = as_int32_array(secs.as_ref())?;- let subsecs = date_part(array, DatePart::Nanosecond)?;- let subsecs = as_int32_array(subsecs.as_ref())?;-- // Special case where there are no nulls.- if subsecs.null_count() == 0 {- let r: Int32Array = binary(secs, subsecs, |secs, subsecs| {- secs * second_factor + (subsecs % 1_000_000_000) / conversion_factor- })?;- Ok(Arc::new(r))- } else {- // Nulls in secs are preserved, nulls in subsecs are treated as zero to account for the case- // where the number of nanoseconds overflows.- let r: Int32Array = secs- .iter()- .zip(subsecs)- .map(|(secs, subsecs)| {- secs.map(|secs| {- let subsecs = subsecs.unwrap_or(0);- secs * second_factor + (subsecs % 1_000_000_000) / conversion_factor- })- })- .collect();- Ok(Arc::new(r))- }-}-fn epoch(array: &dyn Array) -> Result<ArrayRef> {const SECONDS_IN_A_DAY: f64 = 86400_f64;// Note: Month-to-second conversion uses 30 days as an approximation.@@ -564,48 +509,189 @@ fn epoch(array: &dyn Array) -> Result<ArrayRef> {Ok(Arc::new(f))}-/// Invoke [`date_part`] on an `array` (e.g. Timestamp) and convert the-/// result to a total number of nanoseconds as an Int64 array.+/// Return the seconds of `array`, with their fraction, as a number of+/// milliseconds, microseconds or nanoseconds (`part`): the seconds of the+/// minute for timestamps, times and intervals, and the total length for+/// durations.///-/// This returns an Int64 rather than Int32 because there 1 billion-/// `nanosecond`s in each second, so representing up to 60 seconds as-/// nanoseconds can be values up to 60 billion, which does not fit in Int32.-fn seconds_ns(array: &dyn Array) -> Result<ArrayRef> {- // Fast path for Date32 and Date64 - no nanoseconds- if array.data_type() == &Date32 || array.data_type() == &Date64 {- return Ok(Arc::new(Int64Array::from_iter_values_with_nulls(- repeat_n(0, array.len()),- array.nulls().cloned(),- )));- }+/// `nanosecond` returns Int64, because 60 seconds are 60 billion nanoseconds+/// and a duration can be up to `i64::MAX` nanoseconds long. `millisecond` and+/// `microsecond` return Int32. A duration whose length overflows the return+/// type is NULL.+fn subsecond_part(array: &dyn Array, part: DatePart) -> Result<ArrayRef> {+ let len = array.len();+ let nulls = array.nulls().cloned();+ Ok(match (array.data_type(), part) {+ // Fast path for Date32 and Date64 - no seconds+ (Date32 | Date64, DatePart::Nanosecond) => {+ Arc::new(Int64Array::new(vec![0; len].into(), nulls))+ }+ (Date32 | Date64, _) => Arc::new(Int32Array::new(vec![0; len].into(), nulls)),+ // arrow's `date_part(Nanosecond)` computes these, but returns Int32,+ // which overflows past about 2.1 seconds+ (Interval(ArrowIntervalUnit::DayTime), DatePart::Nanosecond) => Arc::new(+ as_interval_dt_array(array)?+ .unary::<_, Int64Type>(|x| (x.milliseconds % 60_000) as i64 * 1_000_000),+ ),+ (Interval(ArrowIntervalUnit::MonthDayNano), DatePart::Nanosecond) => Arc::new(+ as_interval_mdn_array(array)?+ .unary::<_, Int64Type>(|x| x.nanoseconds % 60_000_000_000),+ ),+ // arrow's cast returns NULL on overflow, like its `date_part`, and+ // reinterprets `Duration(Nanosecond)` as Int64 without copying+ (Duration(_), DatePart::Nanosecond) => {+ cast(&cast(array, &Duration(Nanosecond))?, &DataType::Int64)?+ }+ // arrow's sub-second parts of intervals and durations already include+ // the whole seconds, and it rejects them for `Interval(YearMonth)`+ (Interval(_) | Duration(_), _) => date_part(array, part)?,+ (_, DatePart::Millisecond) => {+ seconds_with_fraction::<Int32Type, MILLISECONDS>(array, part)?+ }+ (_, DatePart::Microsecond) => {+ seconds_with_fraction::<Int32Type, MICROSECONDS>(array, part)?+ }+ (_, DatePart::Nanosecond) => {+ seconds_with_fraction::<Int64Type, NANOSECONDS>(array, part)?+ }+ (_, part) => return internal_err!("{part} is not a sub-second part"),+ })+}- let secs = date_part(array, DatePart::Second)?;- // This assumes array is primitive and not a dictionary- let secs = as_int32_array(secs.as_ref())?;- let subsecs = date_part(array, DatePart::Nanosecond)?;- let subsecs = as_int32_array(subsecs.as_ref())?;-- // Special case where there are no nulls.- if subsecs.null_count() == 0 {- let r: Int64Array = binary(secs, subsecs, |secs, subsecs| {- (secs as i64) * 1_000_000_000 + (subsecs as i64)- })?;- Ok(Arc::new(r))- } else {- // Nulls in secs are preserved, nulls in subsecs are treated as zero to account for the case- // where the number of nanoseconds overflows.- let r: Int64Array = secs- .iter()- .zip(subsecs)- .map(|(secs, subsecs)| {- secs.map(|secs| {- let subsecs = subsecs.unwrap_or(0);- (secs as i64) * 1_000_000_000 + (subsecs as i64)- })+/// Return the seconds of the minute of timestamps (in their timezone) and+/// times, with their fraction, in units of `1 / PER_SECOND` seconds.+fn seconds_with_fraction<O, const PER_SECOND: i64>(+ array: &dyn Array,+ part: DatePart,+) -> Result<ArrayRef>+where+ O: ArrowPrimitiveType,+ i64: AsPrimitive<O::Native>,+{+ Ok(match array.data_type() {+ Timestamp(unit, tz) => {+ let tz = parse_tz(tz.as_ref())?;+ match unit {+ Second => timestamp_seconds_with_fraction::<+ TimestampSecondType,+ O,+ PER_SECOND,+ >(array, tz, timestamp_s_to_datetime)?,+ Millisecond => timestamp_seconds_with_fraction::<+ TimestampMillisecondType,+ O,+ PER_SECOND,+ >(array, tz, timestamp_ms_to_datetime)?,+ Microsecond => timestamp_seconds_with_fraction::<+ TimestampMicrosecondType,+ O,+ PER_SECOND,+ >(array, tz, timestamp_us_to_datetime)?,+ Nanosecond => timestamp_seconds_with_fraction::<+ TimestampNanosecondType,+ O,+ PER_SECOND,+ >(array, tz, timestamp_ns_to_datetime)?,+ }+ }+ Time32(Second) => {+ time_seconds_with_fraction::<Time32SecondType, O, 1, PER_SECOND>(array)?+ }+ Time32(Millisecond) => time_seconds_with_fraction::<+ Time32MillisecondType,+ O,+ MILLISECONDS,+ PER_SECOND,+ >(array)?,+ Time64(Microsecond) => time_seconds_with_fraction::<+ Time64MicrosecondType,+ O,+ MICROSECONDS,+ PER_SECOND,+ >(array)?,+ Time64(Nanosecond) => {+ time_seconds_with_fraction::<Time64NanosecondType, O, NANOSECONDS, PER_SECOND>(+ array,+ )?+ }+ other => return internal_err!("{part} does not support {other}"),+ })+}++fn timestamp_seconds_with_fraction<T, O, const PER_SECOND: i64>(+ array: &dyn Array,+ tz: Option<Tz>,+ // arrow's converter for `T`, as its `date_part` uses. The generic+ // `as_datetime::<T>` measured about 1.5x slower.+ to_datetime: impl Fn(i64) -> Option<NaiveDateTime>,+) -> Result<ArrayRef>+where+ T: ArrowTimestampType,+ O: ArrowPrimitiveType,+ i64: AsPrimitive<O::Native>,+{+ let array = as_primitive_array::<T>(array)?;+ // One chrono conversion per row, where arrow's `date_part` needs one each+ // for the seconds and the fraction. `unary_opt` keeps the input nulls and additionally nulls out any value+ // that cannot be represented as a `DateTime` (out of chrono's range).+ let r: PrimitiveArray<O> = match tz {+ Some(tz) => array.unary_opt(|v| {+ to_datetime(v).map(|naive| {+ seconds_of_minute::<O, PER_SECOND>(+ &Utc.from_utc_datetime(&naive).with_timezone(&tz),+ )})- .collect();- Ok(Arc::new(r))- }+ }),+ None => array.unary_opt(|v| {+ to_datetime(v).map(|naive| seconds_of_minute::<O, PER_SECOND>(&naive))+ }),+ };+ Ok(Arc::new(r))+}++/// The seconds of the minute of `t`, with their fraction, in units of+/// `1 / PER_SECOND` seconds+fn seconds_of_minute<O: ArrowPrimitiveType, const PER_SECOND: i64>(+ t: &impl Timelike,+) -> O::Native+where+ i64: AsPrimitive<O::Native>,+{+ // At most 60 seconds, which fits in Int32 for milliseconds and+ // microseconds, so the narrowing `as_` never truncates+ (t.second() as i64 * PER_SECOND + t.nanosecond() as i64 / (NANOSECONDS / PER_SECOND))+ .as_()+}++/// Like [`timestamp_seconds_with_fraction`] for times, which count+/// `1 / UNITS_PER_SECOND` seconds since midnight.+fn time_seconds_with_fraction<T, O, const UNITS_PER_SECOND: i64, const PER_SECOND: i64>(+ array: &dyn Array,+) -> Result<ArrayRef>+where+ T: ArrowPrimitiveType,+ T::Native: AsPrimitive<i64>,+ O: ArrowPrimitiveType,+ i64: AsPrimitive<O::Native>,+{+ let array = as_primitive_array::<T>(array)?;+ // Values outside one day are NULL, as in arrow's `date_part`. `as_time`+ // wraps some to a valid time: https://github.com/apache/arrow-rs/issues/11893+ let r: PrimitiveArray<O> = array.unary_opt(|v| {+ let v: i64 = v.as_();+ (0..SECONDS_IN_DAY * UNITS_PER_SECOND)+ .contains(&v)+ .then(|| {+ let v = v % (60 * UNITS_PER_SECOND);+ if PER_SECOND >= UNITS_PER_SECOND {+ v * (PER_SECOND / UNITS_PER_SECOND)+ } else {+ v / (UNITS_PER_SECOND / PER_SECOND)+ }+ .as_()+ })+ });+ Ok(Arc::new(r))}#[cfg(test)]diff --git a/datafusion/spark/src/function/datetime/date_part.rs b/datafusion/spark/src/function/datetime/date_part.rsindex 50ef1ece9d..de463ef54e 100644--- a/datafusion/spark/src/function/datetime/date_part.rs+++ b/datafusion/spark/src/function/datetime/date_part.rs@@ -81,10 +81,10 @@ impl ScalarUDFImpl for SparkDatePart {internal_err!("Use return_field_from_args in this case instead.")}- fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {- let nullable = args.arg_fields.iter().any(|f| f.is_nullable());-- Ok(Arc::new(Field::new(self.name(), DataType::Int32, nullable)))+ // Nullable, like the integer parts of `date_part`, which this simplifies to,+ // because they can be NULL for a non-null input+ fn return_field_from_args(&self, _args: ReturnFieldArgs) -> Result<FieldRef> {+ Ok(Arc::new(Field::new(self.name(), DataType::Int32, true)))}fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {diff --git a/datafusion/sqllogictest/test_files/datetime/date_part.slt b/datafusion/sqllogictest/test_files/datetime/date_part.sltindex 96cbe78d49..7fcfa77825 100644--- a/datafusion/sqllogictest/test_files/datetime/date_part.slt+++ b/datafusion/sqllogictest/test_files/datetime/date_part.slt@@ -912,6 +912,58 @@ SELECT extract(epoch from arrow_cast('23:32:50.123456789'::time, 'Time64(Nanosec----84770.123456789+# sub-second parts of times, with a NULL row+query IIII+SELECT extract(second from t), extract(millisecond from t), extract(microsecond from t), extract(nanosecond from t)+FROM (SELECT arrow_cast(column1, 'Time32(Second)') AS t+ FROM (VALUES ('23:32:50.123456789'::time), (NULL), ('00:00:01.000000001'::time)))+----+50 50000 50000000 50000000000+NULL NULL NULL NULL+1 1000 1000000 1000000000++query IIII+SELECT extract(second from t), extract(millisecond from t), extract(microsecond from t), extract(nanosecond from t)+FROM (SELECT arrow_cast(column1, 'Time32(Millisecond)') AS t+ FROM (VALUES ('23:32:50.123456789'::time), (NULL), ('00:00:01.000000001'::time)))+----+50 50123 50123000 50123000000+NULL NULL NULL NULL+1 1000 1000000 1000000000++query IIII+SELECT extract(second from t), extract(millisecond from t), extract(microsecond from t), extract(nanosecond from t)+FROM (SELECT arrow_cast(column1, 'Time64(Microsecond)') AS t+ FROM (VALUES ('23:32:50.123456789'::time), (NULL), ('00:00:01.000000001'::time)))+----+50 50123 50123456 50123456000+NULL NULL NULL NULL+1 1000 1000000 1000000000++query IIII+SELECT extract(second from t), extract(millisecond from t), extract(microsecond from t), extract(nanosecond from t)+FROM (SELECT arrow_cast(column1, 'Time64(Nanosecond)') AS t+ FROM (VALUES ('23:32:50.123456789'::time), (NULL), ('00:00:01.000000001'::time)))+----+50 50123 50123456 50123456789+NULL NULL NULL NULL+1 1000 1000000 1000000001++# times outside one day are NULL, like the other parts. The last row is 2^32 + 5+# seconds, which must not wrap around to 5 seconds.+query IIII+SELECT extract(nanosecond from arrow_cast(arrow_cast(column1, 'Int32'), 'Time32(Second)')),+ extract(nanosecond from arrow_cast(arrow_cast(column2, 'Int32'), 'Time32(Millisecond)')),+ extract(nanosecond from arrow_cast(column3, 'Time64(Microsecond)')),+ extract(nanosecond from arrow_cast(column4, 'Time64(Nanosecond)'))+FROM (VALUES (-1, -1, -1, -1),+ (86400, 86400000, 86400000000, 86400000000000),+ (NULL, NULL, 4294967301000000, 4294967301000000000))+----+NULL NULL NULL NULL+NULL NULL NULL NULL+NULL NULL NULL NULL+# test_extract_epochquery R@@ -974,6 +1026,70 @@ SELECT extract(epoch from arrow_cast('1969-12-31', 'Date64'))-----86400+# sub-second parts of timestamps in each unit, before and after 1970, with a NULL row+query IIII+SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts)+FROM (SELECT arrow_cast(column1, 'Timestamp(Second, None)') AS ts+ FROM (VALUES ('2020-09-08T12:00:12'), (NULL), ('1969-12-31T23:59:58')))+----+12 12000 12000000 12000000000+NULL NULL NULL NULL+58 58000 58000000 58000000000++query IIII+SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts)+FROM (SELECT arrow_cast(column1, 'Timestamp(Millisecond, None)') AS ts+ FROM (VALUES ('2020-09-08T12:00:12.123'), (NULL), ('1969-12-31T23:59:58.123')))+----+12 12123 12123000 12123000000+NULL NULL NULL NULL+58 58123 58123000 58123000000++query IIII+SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts)+FROM (SELECT arrow_cast(column1, 'Timestamp(Microsecond, None)') AS ts+ FROM (VALUES ('2020-09-08T12:00:12.123456'), (NULL), ('1969-12-31T23:59:58.123456')))+----+12 12123 12123456 12123456000+NULL NULL NULL NULL+58 58123 58123456 58123456000++query IIII+SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts)+FROM (SELECT arrow_cast(column1, 'Timestamp(Nanosecond, None)') AS ts+ FROM (VALUES ('2020-09-08T12:00:12.123456789'), (NULL), ('1969-12-31T23:59:58.123456789')))+----+12 12123 12123456 12123456789+NULL NULL NULL NULL+58 58123 58123456 58123456789++# Africa/Monrovia was UTC-00:44:30 until 1972, so the seconds depend on the timezone+query IIII+SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts)+FROM (SELECT arrow_cast(column1, 'Timestamp(Millisecond, Some("Africa/Monrovia"))') AS ts+ FROM (VALUES (500), (-1500), (NULL)))+----+30 30500 30500000 30500000000+28 28500 28500000 28500000000+NULL NULL NULL NULL++query IIII+SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts)+FROM (SELECT arrow_cast(column1, 'Timestamp(Nanosecond, Some("Africa/Monrovia"))') AS ts+ FROM (VALUES (500000000), (-1500000000), (NULL)))+----+30 30500 30500000 30500000000+28 28500 28500000 28500000000+NULL NULL NULL NULL++# timestamps outside the range arrow represents are NULL, like the other parts+query IIII+SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts)+FROM (VALUES (arrow_cast(9223372036854775807, 'Timestamp(Second, None)')), (NULL)) AS t(ts)+----+NULL NULL NULL NULL+NULL NULL NULL NULL+# test_extract_intervalquery I@@ -1143,24 +1259,132 @@ create table t (id int, i interval) as values(4, interval '8 months'),(5, NULL);-query III+query IIIIselectid,extract(second from i),- extract(month from i)+ extract(month from i),+ extract(nanosecond from i)from torder by id;-----0 0 5-1 0 3-2 0 0-3 2 0-4 0 8-5 NULL NULL+0 0 5 10+1 0 3 0+2 0 0 2000000+3 2 0 2000000000+4 0 8 0+5 NULL NULL NULLstatement okdrop table t;+# nanosecond on an interval is the seconds field (within the minute) in nanoseconds,+# like microsecond and millisecond.+# Reproducer from https://github.com/apache/datafusion/issues/26678+query III+SELECT extract(nanosecond from interval '1.5 seconds'),+ extract(microsecond from interval '1.5 seconds'),+ extract(millisecond from interval '1.5 seconds')+----+1500000000 1500000 1500++# past i32::MAX nanoseconds+query I+SELECT extract(nanosecond from interval '62.5 seconds')+----+2500000000++query I+SELECT extract(nanosecond from interval '59.999999999 seconds')+----+59999999999++query I+SELECT extract(nanosecond from interval '1 day 2 hours 3 minutes')+----+0++query I+SELECT extract(nanosecond from interval '-1.5 seconds')+----+-1500000000++query I+SELECT extract(nanosecond from arrow_cast('5.1 seconds', 'Interval(DayTime)'))+----+5100000000++query I+SELECT extract(nanosecond from arrow_cast('62.5 seconds', 'Interval(DayTime)'))+----+2500000000++query I+SELECT extract(nanosecond from arrow_cast(NULL, 'Interval(DayTime)'))+----+NULL++# sub-second parts are the seconds field within the minute, with the sign of the+# milliseconds or nanoseconds field, whatever the days and months+query III+SELECT extract(millisecond from i), extract(microsecond from i), extract(nanosecond from i)+FROM (SELECT arrow_cast(column1, 'Interval(DayTime)') AS i FROM (VALUES+ ('-62.5 seconds'), ('-60 seconds'), ('1 day 2 hours 5.1 seconds'),+ ('-1 day 5.1 seconds'), ('-2 hours -5.1 seconds'), (NULL)))+----+-2500 -2500000 -2500000000+0 0 0+5100 5100000 5100000000+5100 5100000 5100000000+-5100 -5100000 -5100000000+NULL NULL NULL++query III+SELECT extract(millisecond from i), extract(microsecond from i), extract(nanosecond from i)+FROM (SELECT arrow_cast(column1, 'Interval(MonthDayNano)') AS i FROM (VALUES+ ('-62.5 seconds'), ('-60 seconds'), ('-59.999999999 seconds'),+ ('1 month -62.5 seconds'), ('-1 day -62.5 seconds'), (NULL)))+----+-2500 -2500000 -2500000000+0 0 0+-59999 -59999999 -59999999999+-2500 -2500000 -2500000000+-2500 -2500000 -2500000000+NULL NULL NULL++# millisecond and microsecond keep the fraction past i32::MAX nanoseconds.+# Reproducer from https://github.com/apache/datafusion/issues/14738#issuecomment-6372885638+query IIII+SELECT extract(millisecond from interval '62.5 seconds'),+ extract(microsecond from interval '62.5 seconds'),+ extract(microsecond from interval '5.5 seconds'),+ extract(microsecond from interval '59.999999 seconds')+----+2500 2500000 5500000 59999999++query II+SELECT extract(millisecond from interval '-62.5 seconds'),+ extract(microsecond from interval '-62.5 seconds')+----+-2500 -2500000++query II+SELECT extract(millisecond from arrow_cast('62.5 seconds', 'Interval(DayTime)')),+ extract(microsecond from arrow_cast('62.5 seconds', 'Interval(DayTime)'))+----+2500 2500000++# Reproducer from https://github.com/apache/datafusion/issues/14738+query III+SELECT date_part('microseconds', interval '1 hour'),+ date_part('milliseconds', interval '1 hour'),+ date_part('seconds', interval '1 hour')+----+0 0 0++query error DataFusion error: Arrow error: Compute error: Nanosecond does not support: Interval\(YearMonth\)+SELECT extract(nanosecond from arrow_cast('10 months', 'Interval(YearMonth)'))+# test_extract_durationquery I@@ -1289,6 +1513,146 @@ SELECT extract(day from arrow_cast(NULL, 'Duration(Second)'))----NULL+# nanosecond on a duration is its total length in nanoseconds.+# Reproducer from https://github.com/apache/datafusion/issues/26678+query IIII+SELECT extract(nanosecond from arrow_cast(1500000000, 'Duration(Nanosecond)')),+ extract(nanosecond from arrow_cast(1500000, 'Duration(Microsecond)')),+ extract(nanosecond from arrow_cast(1500, 'Duration(Millisecond)')),+ extract(nanosecond from arrow_cast(2, 'Duration(Second)'))+----+1500000000 1500000000 1500000000 2000000000++# around i32::MAX nanoseconds+query III+SELECT extract(nanosecond from arrow_cast(2147483647, 'Duration(Nanosecond)')),+ extract(nanosecond from arrow_cast(2147483648, 'Duration(Nanosecond)')),+ extract(nanosecond from arrow_cast(62500000000, 'Duration(Nanosecond)'))+----+2147483647 2147483648 62500000000++query II+SELECT extract(nanosecond from arrow_cast(-1500000000, 'Duration(Nanosecond)')),+ extract(nanosecond from arrow_cast(-62500, 'Duration(Millisecond)'))+----+-1500000000 -62500000000++query I+SELECT extract(nanosecond from arrow_cast(column1, 'Duration(Millisecond)'))+FROM (VALUES (1500), (NULL), (62500), (NULL), (-2))+----+1500000000+NULL+62500000000+NULL+-2000000++query II+SELECT extract(nanosecond from arrow_cast(9223372036854775807, 'Duration(Nanosecond)')),+ extract(nanosecond from arrow_cast(-9223372036854775808, 'Duration(Nanosecond)'))+----+9223372036854775807 -9223372036854775808++# values that overflow are NULL, like the other parts of a duration+query II+SELECT extract(nanosecond from arrow_cast(column1, 'Duration(Second)')),+ extract(nanosecond from arrow_cast(column2, 'Duration(Millisecond)'))+FROM (VALUES (9223372036, -9223372036854), (9223372037, -9223372036855), (NULL, NULL))+----+9223372036000000000 -9223372036854000000+NULL NULL+NULL NULL++# millisecond and microsecond on a duration are its total length in that unit.+# Reproducer from https://github.com/apache/datafusion/issues/14738#issuecomment-6372885638+query II+SELECT extract(millisecond from arrow_cast(2500000000, 'Duration(Nanosecond)')),+ extract(microsecond from arrow_cast(2500000000, 'Duration(Nanosecond)'))+----+2500 2500000++# truncates toward zero, like the other parts of a duration+query III+SELECT extract(millisecond from arrow_cast(-1500, 'Duration(Microsecond)')),+ extract(millisecond from arrow_cast(-1500000, 'Duration(Nanosecond)')),+ extract(second from arrow_cast(-1500, 'Duration(Millisecond)'))+----+-1 -1 -1++query I+SELECT extract(microsecond from arrow_cast(column1, 'Duration(Nanosecond)'))+FROM (VALUES (5500000000), (NULL), (-62500000000), (NULL))+----+5500000+NULL+-62500000+NULL++# around i32::MAX+query II+SELECT extract(millisecond from arrow_cast(2147483647, 'Duration(Millisecond)')),+ extract(microsecond from arrow_cast(2147, 'Duration(Second)'))+----+2147483647 2147000000++# values that overflow Int32 are NULL, like the other parts of a duration+query II+SELECT extract(millisecond from arrow_cast(column1, 'Duration(Millisecond)')),+ extract(microsecond from arrow_cast(column2, 'Duration(Second)'))+FROM (VALUES (2147483648, 2148), (-2147483649, -2148), (NULL, NULL))+----+NULL NULL+NULL NULL+NULL NULL++# Reproducer from https://github.com/apache/datafusion/issues/14738. Postgres returns 0, because+# there the difference is an interval. Here it is a Duration, whose microsecond is its total+# length, and 2.4e9 microseconds don't fit in Int32.+query I+SELECT date_part('microsecond', timestamp '1970-01-01T00:40:00' - timestamp '1970-01-01T00:00:00')+----+NULL++# Non-null inputs can give NULL results, so the result is nullable.+# Reproducers from https://github.com/apache/datafusion/issues/26685+query I+SELECT extract(second from ts) FROM (VALUES (arrow_cast(9223372036854775807, 'Timestamp(Second, None)'))) AS t(ts)+----+NULL++query I+SELECT extract(hour from d) FROM (VALUES (arrow_cast(9223372036854775807, 'Duration(Second)'))) AS t(d)+----+NULL++query I+SELECT extract(nanosecond from d) FROM (VALUES (arrow_cast(9223372037, 'Duration(Second)'))) AS t(d)+----+NULL++query I+SELECT extract(millisecond from d) FROM (VALUES (arrow_cast(2147483648, 'Duration(Millisecond)'))) AS t(d)+----+NULL++# epoch is NULL only for a NULL input, so it keeps the input's nullability+query TTT+DESCRIBE SELECT extract(epoch from ts) AS e, extract(second from ts) AS s,+ extract(millisecond from ts) AS ms, extract(nanosecond from ts) AS ns+FROM (VALUES (timestamp '2020-01-01')) AS t(ts)+----+e Float64 NO+s Int32 YES+ms Int32 YES+ns Int64 YES++query TTT+DESCRIBE SELECT extract(epoch from ts) AS e, extract(second from ts) AS s+FROM (VALUES (timestamp '2020-01-01'), (NULL)) AS t(ts)+----+e Float64 YES+s Int32 YES+# test_extract_date_part_funcquery Bdiff --git a/datafusion/sqllogictest/test_files/spark/datetime/date_part.slt b/datafusion/sqllogictest/test_files/spark/datetime/date_part.sltindex beece06d66..566179ba14 100644--- a/datafusion/sqllogictest/test_files/spark/datetime/date_part.slt+++ b/datafusion/sqllogictest/test_files/spark/datetime/date_part.slt@@ -284,3 +284,10 @@ FROM VALUES19992000NULL++# Nullable even for a non-null input, like the integer parts of the date_part+# this simplifies to+query TTT+DESCRIBE SELECT date_part('SECOND'::string, ts) AS s FROM (VALUES (timestamp '2020-01-01')) AS t(ts)+----+s Int32 YES