Skip to content

Apache DataFusion SQL Query Engine

fix: date_part sub-second parts and result nullability, reusing arrow kernels (#26686)

Matt Butrovich8ea0dbfc341e
diff --git a/datafusion/functions/benches/datetime_expressions/date_part.rs b/datafusion/functions/benches/datetime_expressions/date_part.rs
index 219f75f45e..073a5fdd93 100644
--- a/datafusion/functions/benches/datetime_expressions/date_part.rs
+++ b/datafusion/functions/benches/datetime_expressions/date_part.rs
@@ -188,6 +188,36 @@ fn generate_duration_nanosecond_array_with_nulls(
)
}
+fn generate_timestamp_ns_array_with_nulls(rng: &mut StdRng) -> TimestampNanosecondArray {
+ TimestampNanosecondArray::from(
+ (0..BATCH_SIZE)
+ .map(|_| {
+ if rng.random_bool(NULL_DENSITY) {
+ None
+ } else {
+ Some(rng.random_range(0..TS_BOUND * 1_000_000_000))
+ }
+ })
+ .collect::<Vec<_>>(),
+ )
+}
+
+fn generate_time64_nanosecond_array_with_nulls(
+ rng: &mut StdRng,
+) -> Time64NanosecondArray {
+ Time64NanosecondArray::from(
+ (0..BATCH_SIZE)
+ .map(|_| {
+ if rng.random_bool(NULL_DENSITY) {
+ None
+ } else {
+ Some(rng.random_range(0..SEC_DAY * 1_000_000_000))
+ }
+ })
+ .collect::<Vec<_>>(),
+ )
+}
+
fn bench_date_part(
c: &mut Criterion,
udf: &Arc<ScalarUDF>,
@@ -244,6 +274,14 @@ fn criterion_benchmark(c: &mut Criterion) {
let duration_ns_nulls =
Arc::new(generate_duration_nanosecond_array_with_nulls(&mut rng)) as ArrayRef;
+ let ts_ns_tz =
+ Arc::new(generate_timestamp_ns_array(&mut rng).with_timezone("America/New_York"))
+ as ArrayRef;
+ let ts_ns_nulls =
+ Arc::new(generate_timestamp_ns_array_with_nulls(&mut rng)) as ArrayRef;
+ let time64_ns_nulls =
+ Arc::new(generate_time64_nanosecond_array_with_nulls(&mut rng)) as ArrayRef;
+
let udf = date_part();
for part in ["year", "month", "week", "day", "hour", "minute"] {
@@ -365,6 +403,41 @@ fn criterion_benchmark(c: &mut Criterion) {
);
}
+ // Durations past `i32::MAX` milliseconds or microseconds overflow `Int32`, so
+ // only `nanosecond` runs on them
+ for (part, return_type) in [
+ ("millisecond", DataType::Int32),
+ ("microsecond", DataType::Int32),
+ ("nanosecond", DataType::Int64),
+ ] {
+ let mut arrays = vec![
+ ("ns_tz", &ts_ns_tz),
+ ("ns_nulls", &ts_ns_nulls),
+ ("time32_ms", &time32_ms),
+ ("time64_ns", &time64_ns),
+ ("time64_ns_nulls", &time64_ns_nulls),
+ ("interval_dt", &interval_dt),
+ ("interval_mdn", &interval_mdn),
+ ("duration_ns_small", &duration_ns_small),
+ ];
+ if part == "nanosecond" {
+ arrays.extend([
+ ("duration_ns", &duration_ns),
+ ("duration_ns_nulls", &duration_ns_nulls),
+ ]);
+ }
+ for (name, array) in arrays {
+ bench_date_part(
+ c,
+ &udf,
+ &format!("date_part_{part}_{name}_1000"),
+ part,
+ Arc::clone(array),
+ return_type.clone(),
+ );
+ }
+ }
+
for part in ["quarter", "isoyear", "doy", "dow", "isodow"] {
bench_date_part(
c,
diff --git a/datafusion/functions/src/datetime/date_part.rs b/datafusion/functions/src/datetime/date_part.rs
index 6dcb948a0e..919f1b0ac8 100644
--- a/datafusion/functions/src/datetime/date_part.rs
+++ b/datafusion/functions/src/datetime/date_part.rs
@@ -15,25 +15,33 @@
// specific language governing permissions and limitations
// under the License.
-use std::iter::repeat_n;
use std::str::FromStr;
use std::sync::Arc;
-use arrow::array::temporal_conversions::as_datetime_with_timezone;
+use arrow::array::temporal_conversions::{
+ MICROSECONDS, MILLISECONDS, NANOSECONDS, SECONDS_IN_DAY, as_datetime_with_timezone,
+ timestamp_ms_to_datetime, timestamp_ns_to_datetime, timestamp_s_to_datetime,
+ timestamp_us_to_datetime,
+};
use arrow::array::timezone::Tz;
-use arrow::array::{Array, ArrayRef, Float64Array, Int32Array, Int64Array};
-use arrow::compute::{DatePart, binary, date_part};
+use arrow::array::{
+ Array, ArrayRef, Float64Array, Int32Array, Int64Array, PrimitiveArray,
+};
+use arrow::compute::{DatePart, cast, date_part};
use arrow::datatypes::DataType::{
Date32, Date64, Duration, Interval, Time32, Time64, Timestamp,
};
use arrow::datatypes::TimeUnit::{Microsecond, Millisecond, Nanosecond, Second};
use arrow::datatypes::{
- ArrowTimestampType, DataType, Field, FieldRef, IntervalUnit as ArrowIntervalUnit,
- TimeUnit, TimestampMicrosecondType, TimestampMillisecondType,
- TimestampNanosecondType, TimestampSecondType,
+ ArrowPrimitiveType, ArrowTimestampType, DataType, Field, FieldRef, Int32Type,
+ Int64Type, IntervalUnit as ArrowIntervalUnit, Time32MillisecondType,
+ Time32SecondType, Time64MicrosecondType, Time64NanosecondType,
+ TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType,
+ TimestampSecondType,
};
-use chrono::{Datelike, NaiveDate, Offset};
+use chrono::{Datelike, NaiveDate, NaiveDateTime, Offset, TimeZone, Timelike, Utc};
use datafusion_common::types::{NativeType, logical_date};
+use num_traits::AsPrimitive;
use crate::datetime::common::parse_tz;
use datafusion_common::{
@@ -41,14 +49,13 @@ use datafusion_common::{
cast::{
as_date32_array, as_date64_array, as_duration_microsecond_array,
as_duration_millisecond_array, as_duration_nanosecond_array,
- as_duration_second_array, as_int32_array, as_interval_dt_array,
- as_interval_mdn_array, as_interval_ym_array, as_primitive_array,
- as_time32_millisecond_array, as_time32_second_array, as_time64_microsecond_array,
- as_time64_nanosecond_array, as_timestamp_microsecond_array,
- as_timestamp_millisecond_array, as_timestamp_nanosecond_array,
- as_timestamp_second_array,
+ as_duration_second_array, as_interval_dt_array, as_interval_mdn_array,
+ as_interval_ym_array, as_primitive_array, as_time32_millisecond_array,
+ as_time32_second_array, as_time64_microsecond_array, as_time64_nanosecond_array,
+ as_timestamp_microsecond_array, as_timestamp_millisecond_array,
+ as_timestamp_nanosecond_array, as_timestamp_second_array,
},
- exec_err, internal_err, not_impl_err,
+ exec_err, internal_err,
types::logical_string,
utils::take_function_args,
};
@@ -191,13 +198,16 @@ impl ScalarUDFImpl for DatePartFunc {
.flatten()
.filter(|s| !s.is_empty())
.map(|part| {
+ // The integer parts are NULL for values arrow can't
+ // convert or that overflow, even when the input is
+ // not NULL
if is_epoch(part) {
Field::new(self.name(), DataType::Float64, nullable)
} else if is_nanosecond(part) {
- // See notes on [seconds_ns] for rationale
- Field::new(self.name(), DataType::Int64, nullable)
+ // See notes on [subsecond_part] for rationale
+ Field::new(self.name(), DataType::Int64, true)
} else {
- Field::new(self.name(), DataType::Int32, nullable)
+ Field::new(self.name(), DataType::Int32, true)
}
})
})
@@ -232,10 +242,11 @@ impl ScalarUDFImpl for DatePartFunc {
let part_trim = part_normalization(&part);
let arr = match DatePart::from_str(part_trim) {
- Ok(DatePart::Second) => seconds_as_i32(array.as_ref(), Second)?,
- Ok(DatePart::Millisecond) => seconds_as_i32(array.as_ref(), Millisecond)?,
- Ok(DatePart::Microsecond) => seconds_as_i32(array.as_ref(), Microsecond)?,
- Ok(DatePart::Nanosecond) => seconds_ns(array.as_ref())?,
+ Ok(
+ part @ (DatePart::Millisecond
+ | DatePart::Microsecond
+ | DatePart::Nanosecond),
+ ) => subsecond_part(array.as_ref(), part)?,
Ok(part) => date_part(array.as_ref(), part)?,
Err(_) if is_epoch(part_trim) => epoch(array.as_ref())?,
// `timezone`, `timezone_hour` and `timezone_minute` have no
@@ -440,72 +451,6 @@ fn part_normalization(part: &str) -> &str {
.unwrap_or(part)
}
-/// Invoke [`date_part`] on an `array` (e.g. Timestamp) and convert the
-/// result to a total number of seconds, milliseconds, microseconds or
-/// nanoseconds as an `Int32Array`
-fn seconds_as_i32(array: &dyn Array, unit: TimeUnit) -> Result<ArrayRef> {
- // Nanosecond is neither supported in Postgres nor DuckDB, to avoid dealing
- // with overflow and precision issue we don't support nanosecond
- if unit == Nanosecond {
- return not_impl_err!("Date part {unit:?} not supported");
- }
-
- // Fast path with seconds - no need to compute nanoseconds
- if unit == Second {
- return Ok(date_part(array, DatePart::Second)?);
- }
-
- // Fast path for Date32 and Date64 - no seconds
- if array.data_type() == &Date32 || array.data_type() == &Date64 {
- return Ok(Arc::new(Int32Array::from_iter_values_with_nulls(
- repeat_n(0, array.len()),
- array.nulls().cloned(),
- )));
- }
-
- let conversion_factor = match unit {
- Second => 1_000_000_000,
- Millisecond => 1_000_000,
- Microsecond => 1_000,
- Nanosecond => 1,
- };
-
- let second_factor = match unit {
- Second => 1,
- Millisecond => 1_000,
- Microsecond => 1_000_000,
- Nanosecond => 1_000_000_000,
- };
-
- let secs = date_part(array, DatePart::Second)?;
- // This assumes array is primitive and not a dictionary
- let secs = as_int32_array(secs.as_ref())?;
- let subsecs = date_part(array, DatePart::Nanosecond)?;
- let subsecs = as_int32_array(subsecs.as_ref())?;
-
- // Special case where there are no nulls.
- if subsecs.null_count() == 0 {
- let r: Int32Array = binary(secs, subsecs, |secs, subsecs| {
- secs * second_factor + (subsecs % 1_000_000_000) / conversion_factor
- })?;
- Ok(Arc::new(r))
- } else {
- // Nulls in secs are preserved, nulls in subsecs are treated as zero to account for the case
- // where the number of nanoseconds overflows.
- let r: Int32Array = secs
- .iter()
- .zip(subsecs)
- .map(|(secs, subsecs)| {
- secs.map(|secs| {
- let subsecs = subsecs.unwrap_or(0);
- secs * second_factor + (subsecs % 1_000_000_000) / conversion_factor
- })
- })
- .collect();
- Ok(Arc::new(r))
- }
-}
-
fn epoch(array: &dyn Array) -> Result<ArrayRef> {
const SECONDS_IN_A_DAY: f64 = 86400_f64;
// Note: Month-to-second conversion uses 30 days as an approximation.
@@ -564,48 +509,189 @@ fn epoch(array: &dyn Array) -> Result<ArrayRef> {
Ok(Arc::new(f))
}
-/// Invoke [`date_part`] on an `array` (e.g. Timestamp) and convert the
-/// result to a total number of nanoseconds as an Int64 array.
+/// Return the seconds of `array`, with their fraction, as a number of
+/// milliseconds, microseconds or nanoseconds (`part`): the seconds of the
+/// minute for timestamps, times and intervals, and the total length for
+/// durations.
///
-/// This returns an Int64 rather than Int32 because there 1 billion
-/// `nanosecond`s in each second, so representing up to 60 seconds as
-/// nanoseconds can be values up to 60 billion, which does not fit in Int32.
-fn seconds_ns(array: &dyn Array) -> Result<ArrayRef> {
- // Fast path for Date32 and Date64 - no nanoseconds
- if array.data_type() == &Date32 || array.data_type() == &Date64 {
- return Ok(Arc::new(Int64Array::from_iter_values_with_nulls(
- repeat_n(0, array.len()),
- array.nulls().cloned(),
- )));
- }
+/// `nanosecond` returns Int64, because 60 seconds are 60 billion nanoseconds
+/// and a duration can be up to `i64::MAX` nanoseconds long. `millisecond` and
+/// `microsecond` return Int32. A duration whose length overflows the return
+/// type is NULL.
+fn subsecond_part(array: &dyn Array, part: DatePart) -> Result<ArrayRef> {
+ let len = array.len();
+ let nulls = array.nulls().cloned();
+ Ok(match (array.data_type(), part) {
+ // Fast path for Date32 and Date64 - no seconds
+ (Date32 | Date64, DatePart::Nanosecond) => {
+ Arc::new(Int64Array::new(vec![0; len].into(), nulls))
+ }
+ (Date32 | Date64, _) => Arc::new(Int32Array::new(vec![0; len].into(), nulls)),
+ // arrow's `date_part(Nanosecond)` computes these, but returns Int32,
+ // which overflows past about 2.1 seconds
+ (Interval(ArrowIntervalUnit::DayTime), DatePart::Nanosecond) => Arc::new(
+ as_interval_dt_array(array)?
+ .unary::<_, Int64Type>(|x| (x.milliseconds % 60_000) as i64 * 1_000_000),
+ ),
+ (Interval(ArrowIntervalUnit::MonthDayNano), DatePart::Nanosecond) => Arc::new(
+ as_interval_mdn_array(array)?
+ .unary::<_, Int64Type>(|x| x.nanoseconds % 60_000_000_000),
+ ),
+ // arrow's cast returns NULL on overflow, like its `date_part`, and
+ // reinterprets `Duration(Nanosecond)` as Int64 without copying
+ (Duration(_), DatePart::Nanosecond) => {
+ cast(&cast(array, &Duration(Nanosecond))?, &DataType::Int64)?
+ }
+ // arrow's sub-second parts of intervals and durations already include
+ // the whole seconds, and it rejects them for `Interval(YearMonth)`
+ (Interval(_) | Duration(_), _) => date_part(array, part)?,
+ (_, DatePart::Millisecond) => {
+ seconds_with_fraction::<Int32Type, MILLISECONDS>(array, part)?
+ }
+ (_, DatePart::Microsecond) => {
+ seconds_with_fraction::<Int32Type, MICROSECONDS>(array, part)?
+ }
+ (_, DatePart::Nanosecond) => {
+ seconds_with_fraction::<Int64Type, NANOSECONDS>(array, part)?
+ }
+ (_, part) => return internal_err!("{part} is not a sub-second part"),
+ })
+}
- let secs = date_part(array, DatePart::Second)?;
- // This assumes array is primitive and not a dictionary
- let secs = as_int32_array(secs.as_ref())?;
- let subsecs = date_part(array, DatePart::Nanosecond)?;
- let subsecs = as_int32_array(subsecs.as_ref())?;
-
- // Special case where there are no nulls.
- if subsecs.null_count() == 0 {
- let r: Int64Array = binary(secs, subsecs, |secs, subsecs| {
- (secs as i64) * 1_000_000_000 + (subsecs as i64)
- })?;
- Ok(Arc::new(r))
- } else {
- // Nulls in secs are preserved, nulls in subsecs are treated as zero to account for the case
- // where the number of nanoseconds overflows.
- let r: Int64Array = secs
- .iter()
- .zip(subsecs)
- .map(|(secs, subsecs)| {
- secs.map(|secs| {
- let subsecs = subsecs.unwrap_or(0);
- (secs as i64) * 1_000_000_000 + (subsecs as i64)
- })
+/// Return the seconds of the minute of timestamps (in their timezone) and
+/// times, with their fraction, in units of `1 / PER_SECOND` seconds.
+fn seconds_with_fraction<O, const PER_SECOND: i64>(
+ array: &dyn Array,
+ part: DatePart,
+) -> Result<ArrayRef>
+where
+ O: ArrowPrimitiveType,
+ i64: AsPrimitive<O::Native>,
+{
+ Ok(match array.data_type() {
+ Timestamp(unit, tz) => {
+ let tz = parse_tz(tz.as_ref())?;
+ match unit {
+ Second => timestamp_seconds_with_fraction::<
+ TimestampSecondType,
+ O,
+ PER_SECOND,
+ >(array, tz, timestamp_s_to_datetime)?,
+ Millisecond => timestamp_seconds_with_fraction::<
+ TimestampMillisecondType,
+ O,
+ PER_SECOND,
+ >(array, tz, timestamp_ms_to_datetime)?,
+ Microsecond => timestamp_seconds_with_fraction::<
+ TimestampMicrosecondType,
+ O,
+ PER_SECOND,
+ >(array, tz, timestamp_us_to_datetime)?,
+ Nanosecond => timestamp_seconds_with_fraction::<
+ TimestampNanosecondType,
+ O,
+ PER_SECOND,
+ >(array, tz, timestamp_ns_to_datetime)?,
+ }
+ }
+ Time32(Second) => {
+ time_seconds_with_fraction::<Time32SecondType, O, 1, PER_SECOND>(array)?
+ }
+ Time32(Millisecond) => time_seconds_with_fraction::<
+ Time32MillisecondType,
+ O,
+ MILLISECONDS,
+ PER_SECOND,
+ >(array)?,
+ Time64(Microsecond) => time_seconds_with_fraction::<
+ Time64MicrosecondType,
+ O,
+ MICROSECONDS,
+ PER_SECOND,
+ >(array)?,
+ Time64(Nanosecond) => {
+ time_seconds_with_fraction::<Time64NanosecondType, O, NANOSECONDS, PER_SECOND>(
+ array,
+ )?
+ }
+ other => return internal_err!("{part} does not support {other}"),
+ })
+}
+
+fn timestamp_seconds_with_fraction<T, O, const PER_SECOND: i64>(
+ array: &dyn Array,
+ tz: Option<Tz>,
+ // arrow's converter for `T`, as its `date_part` uses. The generic
+ // `as_datetime::<T>` measured about 1.5x slower.
+ to_datetime: impl Fn(i64) -> Option<NaiveDateTime>,
+) -> Result<ArrayRef>
+where
+ T: ArrowTimestampType,
+ O: ArrowPrimitiveType,
+ i64: AsPrimitive<O::Native>,
+{
+ let array = as_primitive_array::<T>(array)?;
+ // One chrono conversion per row, where arrow's `date_part` needs one each
+ // for the seconds and the fraction. `unary_opt` keeps the input nulls and additionally nulls out any value
+ // that cannot be represented as a `DateTime` (out of chrono's range).
+ let r: PrimitiveArray<O> = match tz {
+ Some(tz) => array.unary_opt(|v| {
+ to_datetime(v).map(|naive| {
+ seconds_of_minute::<O, PER_SECOND>(
+ &Utc.from_utc_datetime(&naive).with_timezone(&tz),
+ )
})
- .collect();
- Ok(Arc::new(r))
- }
+ }),
+ None => array.unary_opt(|v| {
+ to_datetime(v).map(|naive| seconds_of_minute::<O, PER_SECOND>(&naive))
+ }),
+ };
+ Ok(Arc::new(r))
+}
+
+/// The seconds of the minute of `t`, with their fraction, in units of
+/// `1 / PER_SECOND` seconds
+fn seconds_of_minute<O: ArrowPrimitiveType, const PER_SECOND: i64>(
+ t: &impl Timelike,
+) -> O::Native
+where
+ i64: AsPrimitive<O::Native>,
+{
+ // At most 60 seconds, which fits in Int32 for milliseconds and
+ // microseconds, so the narrowing `as_` never truncates
+ (t.second() as i64 * PER_SECOND + t.nanosecond() as i64 / (NANOSECONDS / PER_SECOND))
+ .as_()
+}
+
+/// Like [`timestamp_seconds_with_fraction`] for times, which count
+/// `1 / UNITS_PER_SECOND` seconds since midnight.
+fn time_seconds_with_fraction<T, O, const UNITS_PER_SECOND: i64, const PER_SECOND: i64>(
+ array: &dyn Array,
+) -> Result<ArrayRef>
+where
+ T: ArrowPrimitiveType,
+ T::Native: AsPrimitive<i64>,
+ O: ArrowPrimitiveType,
+ i64: AsPrimitive<O::Native>,
+{
+ let array = as_primitive_array::<T>(array)?;
+ // Values outside one day are NULL, as in arrow's `date_part`. `as_time`
+ // wraps some to a valid time: https://github.com/apache/arrow-rs/issues/11893
+ let r: PrimitiveArray<O> = array.unary_opt(|v| {
+ let v: i64 = v.as_();
+ (0..SECONDS_IN_DAY * UNITS_PER_SECOND)
+ .contains(&v)
+ .then(|| {
+ let v = v % (60 * UNITS_PER_SECOND);
+ if PER_SECOND >= UNITS_PER_SECOND {
+ v * (PER_SECOND / UNITS_PER_SECOND)
+ } else {
+ v / (UNITS_PER_SECOND / PER_SECOND)
+ }
+ .as_()
+ })
+ });
+ Ok(Arc::new(r))
}
#[cfg(test)]
diff --git a/datafusion/spark/src/function/datetime/date_part.rs b/datafusion/spark/src/function/datetime/date_part.rs
index 50ef1ece9d..de463ef54e 100644
--- a/datafusion/spark/src/function/datetime/date_part.rs
+++ b/datafusion/spark/src/function/datetime/date_part.rs
@@ -81,10 +81,10 @@ impl ScalarUDFImpl for SparkDatePart {
internal_err!("Use return_field_from_args in this case instead.")
}
- fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {
- let nullable = args.arg_fields.iter().any(|f| f.is_nullable());
-
- Ok(Arc::new(Field::new(self.name(), DataType::Int32, nullable)))
+ // Nullable, like the integer parts of `date_part`, which this simplifies to,
+ // because they can be NULL for a non-null input
+ fn return_field_from_args(&self, _args: ReturnFieldArgs) -> Result<FieldRef> {
+ Ok(Arc::new(Field::new(self.name(), DataType::Int32, true)))
}
fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {
diff --git a/datafusion/sqllogictest/test_files/datetime/date_part.slt b/datafusion/sqllogictest/test_files/datetime/date_part.slt
index 96cbe78d49..7fcfa77825 100644
--- a/datafusion/sqllogictest/test_files/datetime/date_part.slt
+++ b/datafusion/sqllogictest/test_files/datetime/date_part.slt
@@ -912,6 +912,58 @@ SELECT extract(epoch from arrow_cast('23:32:50.123456789'::time, 'Time64(Nanosec
----
84770.123456789
+# sub-second parts of times, with a NULL row
+query IIII
+SELECT extract(second from t), extract(millisecond from t), extract(microsecond from t), extract(nanosecond from t)
+FROM (SELECT arrow_cast(column1, 'Time32(Second)') AS t
+ FROM (VALUES ('23:32:50.123456789'::time), (NULL), ('00:00:01.000000001'::time)))
+----
+50 50000 50000000 50000000000
+NULL NULL NULL NULL
+1 1000 1000000 1000000000
+
+query IIII
+SELECT extract(second from t), extract(millisecond from t), extract(microsecond from t), extract(nanosecond from t)
+FROM (SELECT arrow_cast(column1, 'Time32(Millisecond)') AS t
+ FROM (VALUES ('23:32:50.123456789'::time), (NULL), ('00:00:01.000000001'::time)))
+----
+50 50123 50123000 50123000000
+NULL NULL NULL NULL
+1 1000 1000000 1000000000
+
+query IIII
+SELECT extract(second from t), extract(millisecond from t), extract(microsecond from t), extract(nanosecond from t)
+FROM (SELECT arrow_cast(column1, 'Time64(Microsecond)') AS t
+ FROM (VALUES ('23:32:50.123456789'::time), (NULL), ('00:00:01.000000001'::time)))
+----
+50 50123 50123456 50123456000
+NULL NULL NULL NULL
+1 1000 1000000 1000000000
+
+query IIII
+SELECT extract(second from t), extract(millisecond from t), extract(microsecond from t), extract(nanosecond from t)
+FROM (SELECT arrow_cast(column1, 'Time64(Nanosecond)') AS t
+ FROM (VALUES ('23:32:50.123456789'::time), (NULL), ('00:00:01.000000001'::time)))
+----
+50 50123 50123456 50123456789
+NULL NULL NULL NULL
+1 1000 1000000 1000000001
+
+# times outside one day are NULL, like the other parts. The last row is 2^32 + 5
+# seconds, which must not wrap around to 5 seconds.
+query IIII
+SELECT extract(nanosecond from arrow_cast(arrow_cast(column1, 'Int32'), 'Time32(Second)')),
+ extract(nanosecond from arrow_cast(arrow_cast(column2, 'Int32'), 'Time32(Millisecond)')),
+ extract(nanosecond from arrow_cast(column3, 'Time64(Microsecond)')),
+ extract(nanosecond from arrow_cast(column4, 'Time64(Nanosecond)'))
+FROM (VALUES (-1, -1, -1, -1),
+ (86400, 86400000, 86400000000, 86400000000000),
+ (NULL, NULL, 4294967301000000, 4294967301000000000))
+----
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+
# test_extract_epoch
query R
@@ -974,6 +1026,70 @@ SELECT extract(epoch from arrow_cast('1969-12-31', 'Date64'))
----
-86400
+# sub-second parts of timestamps in each unit, before and after 1970, with a NULL row
+query IIII
+SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts)
+FROM (SELECT arrow_cast(column1, 'Timestamp(Second, None)') AS ts
+ FROM (VALUES ('2020-09-08T12:00:12'), (NULL), ('1969-12-31T23:59:58')))
+----
+12 12000 12000000 12000000000
+NULL NULL NULL NULL
+58 58000 58000000 58000000000
+
+query IIII
+SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts)
+FROM (SELECT arrow_cast(column1, 'Timestamp(Millisecond, None)') AS ts
+ FROM (VALUES ('2020-09-08T12:00:12.123'), (NULL), ('1969-12-31T23:59:58.123')))
+----
+12 12123 12123000 12123000000
+NULL NULL NULL NULL
+58 58123 58123000 58123000000
+
+query IIII
+SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts)
+FROM (SELECT arrow_cast(column1, 'Timestamp(Microsecond, None)') AS ts
+ FROM (VALUES ('2020-09-08T12:00:12.123456'), (NULL), ('1969-12-31T23:59:58.123456')))
+----
+12 12123 12123456 12123456000
+NULL NULL NULL NULL
+58 58123 58123456 58123456000
+
+query IIII
+SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts)
+FROM (SELECT arrow_cast(column1, 'Timestamp(Nanosecond, None)') AS ts
+ FROM (VALUES ('2020-09-08T12:00:12.123456789'), (NULL), ('1969-12-31T23:59:58.123456789')))
+----
+12 12123 12123456 12123456789
+NULL NULL NULL NULL
+58 58123 58123456 58123456789
+
+# Africa/Monrovia was UTC-00:44:30 until 1972, so the seconds depend on the timezone
+query IIII
+SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts)
+FROM (SELECT arrow_cast(column1, 'Timestamp(Millisecond, Some("Africa/Monrovia"))') AS ts
+ FROM (VALUES (500), (-1500), (NULL)))
+----
+30 30500 30500000 30500000000
+28 28500 28500000 28500000000
+NULL NULL NULL NULL
+
+query IIII
+SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts)
+FROM (SELECT arrow_cast(column1, 'Timestamp(Nanosecond, Some("Africa/Monrovia"))') AS ts
+ FROM (VALUES (500000000), (-1500000000), (NULL)))
+----
+30 30500 30500000 30500000000
+28 28500 28500000 28500000000
+NULL NULL NULL NULL
+
+# timestamps outside the range arrow represents are NULL, like the other parts
+query IIII
+SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts)
+FROM (VALUES (arrow_cast(9223372036854775807, 'Timestamp(Second, None)')), (NULL)) AS t(ts)
+----
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+
# test_extract_interval
query I
@@ -1143,24 +1259,132 @@ create table t (id int, i interval) as values
(4, interval '8 months'),
(5, NULL);
-query III
+query IIII
select
id,
extract(second from i),
- extract(month from i)
+ extract(month from i),
+ extract(nanosecond from i)
from t
order by id;
----
-0 0 5
-1 0 3
-2 0 0
-3 2 0
-4 0 8
-5 NULL NULL
+0 0 5 10
+1 0 3 0
+2 0 0 2000000
+3 2 0 2000000000
+4 0 8 0
+5 NULL NULL NULL
statement ok
drop table t;
+# nanosecond on an interval is the seconds field (within the minute) in nanoseconds,
+# like microsecond and millisecond.
+# Reproducer from https://github.com/apache/datafusion/issues/26678
+query III
+SELECT extract(nanosecond from interval '1.5 seconds'),
+ extract(microsecond from interval '1.5 seconds'),
+ extract(millisecond from interval '1.5 seconds')
+----
+1500000000 1500000 1500
+
+# past i32::MAX nanoseconds
+query I
+SELECT extract(nanosecond from interval '62.5 seconds')
+----
+2500000000
+
+query I
+SELECT extract(nanosecond from interval '59.999999999 seconds')
+----
+59999999999
+
+query I
+SELECT extract(nanosecond from interval '1 day 2 hours 3 minutes')
+----
+0
+
+query I
+SELECT extract(nanosecond from interval '-1.5 seconds')
+----
+-1500000000
+
+query I
+SELECT extract(nanosecond from arrow_cast('5.1 seconds', 'Interval(DayTime)'))
+----
+5100000000
+
+query I
+SELECT extract(nanosecond from arrow_cast('62.5 seconds', 'Interval(DayTime)'))
+----
+2500000000
+
+query I
+SELECT extract(nanosecond from arrow_cast(NULL, 'Interval(DayTime)'))
+----
+NULL
+
+# sub-second parts are the seconds field within the minute, with the sign of the
+# milliseconds or nanoseconds field, whatever the days and months
+query III
+SELECT extract(millisecond from i), extract(microsecond from i), extract(nanosecond from i)
+FROM (SELECT arrow_cast(column1, 'Interval(DayTime)') AS i FROM (VALUES
+ ('-62.5 seconds'), ('-60 seconds'), ('1 day 2 hours 5.1 seconds'),
+ ('-1 day 5.1 seconds'), ('-2 hours -5.1 seconds'), (NULL)))
+----
+-2500 -2500000 -2500000000
+0 0 0
+5100 5100000 5100000000
+5100 5100000 5100000000
+-5100 -5100000 -5100000000
+NULL NULL NULL
+
+query III
+SELECT extract(millisecond from i), extract(microsecond from i), extract(nanosecond from i)
+FROM (SELECT arrow_cast(column1, 'Interval(MonthDayNano)') AS i FROM (VALUES
+ ('-62.5 seconds'), ('-60 seconds'), ('-59.999999999 seconds'),
+ ('1 month -62.5 seconds'), ('-1 day -62.5 seconds'), (NULL)))
+----
+-2500 -2500000 -2500000000
+0 0 0
+-59999 -59999999 -59999999999
+-2500 -2500000 -2500000000
+-2500 -2500000 -2500000000
+NULL NULL NULL
+
+# millisecond and microsecond keep the fraction past i32::MAX nanoseconds.
+# Reproducer from https://github.com/apache/datafusion/issues/14738#issuecomment-6372885638
+query IIII
+SELECT extract(millisecond from interval '62.5 seconds'),
+ extract(microsecond from interval '62.5 seconds'),
+ extract(microsecond from interval '5.5 seconds'),
+ extract(microsecond from interval '59.999999 seconds')
+----
+2500 2500000 5500000 59999999
+
+query II
+SELECT extract(millisecond from interval '-62.5 seconds'),
+ extract(microsecond from interval '-62.5 seconds')
+----
+-2500 -2500000
+
+query II
+SELECT extract(millisecond from arrow_cast('62.5 seconds', 'Interval(DayTime)')),
+ extract(microsecond from arrow_cast('62.5 seconds', 'Interval(DayTime)'))
+----
+2500 2500000
+
+# Reproducer from https://github.com/apache/datafusion/issues/14738
+query III
+SELECT date_part('microseconds', interval '1 hour'),
+ date_part('milliseconds', interval '1 hour'),
+ date_part('seconds', interval '1 hour')
+----
+0 0 0
+
+query error DataFusion error: Arrow error: Compute error: Nanosecond does not support: Interval\(YearMonth\)
+SELECT extract(nanosecond from arrow_cast('10 months', 'Interval(YearMonth)'))
+
# test_extract_duration
query I
@@ -1289,6 +1513,146 @@ SELECT extract(day from arrow_cast(NULL, 'Duration(Second)'))
----
NULL
+# nanosecond on a duration is its total length in nanoseconds.
+# Reproducer from https://github.com/apache/datafusion/issues/26678
+query IIII
+SELECT extract(nanosecond from arrow_cast(1500000000, 'Duration(Nanosecond)')),
+ extract(nanosecond from arrow_cast(1500000, 'Duration(Microsecond)')),
+ extract(nanosecond from arrow_cast(1500, 'Duration(Millisecond)')),
+ extract(nanosecond from arrow_cast(2, 'Duration(Second)'))
+----
+1500000000 1500000000 1500000000 2000000000
+
+# around i32::MAX nanoseconds
+query III
+SELECT extract(nanosecond from arrow_cast(2147483647, 'Duration(Nanosecond)')),
+ extract(nanosecond from arrow_cast(2147483648, 'Duration(Nanosecond)')),
+ extract(nanosecond from arrow_cast(62500000000, 'Duration(Nanosecond)'))
+----
+2147483647 2147483648 62500000000
+
+query II
+SELECT extract(nanosecond from arrow_cast(-1500000000, 'Duration(Nanosecond)')),
+ extract(nanosecond from arrow_cast(-62500, 'Duration(Millisecond)'))
+----
+-1500000000 -62500000000
+
+query I
+SELECT extract(nanosecond from arrow_cast(column1, 'Duration(Millisecond)'))
+FROM (VALUES (1500), (NULL), (62500), (NULL), (-2))
+----
+1500000000
+NULL
+62500000000
+NULL
+-2000000
+
+query II
+SELECT extract(nanosecond from arrow_cast(9223372036854775807, 'Duration(Nanosecond)')),
+ extract(nanosecond from arrow_cast(-9223372036854775808, 'Duration(Nanosecond)'))
+----
+9223372036854775807 -9223372036854775808
+
+# values that overflow are NULL, like the other parts of a duration
+query II
+SELECT extract(nanosecond from arrow_cast(column1, 'Duration(Second)')),
+ extract(nanosecond from arrow_cast(column2, 'Duration(Millisecond)'))
+FROM (VALUES (9223372036, -9223372036854), (9223372037, -9223372036855), (NULL, NULL))
+----
+9223372036000000000 -9223372036854000000
+NULL NULL
+NULL NULL
+
+# millisecond and microsecond on a duration are its total length in that unit.
+# Reproducer from https://github.com/apache/datafusion/issues/14738#issuecomment-6372885638
+query II
+SELECT extract(millisecond from arrow_cast(2500000000, 'Duration(Nanosecond)')),
+ extract(microsecond from arrow_cast(2500000000, 'Duration(Nanosecond)'))
+----
+2500 2500000
+
+# truncates toward zero, like the other parts of a duration
+query III
+SELECT extract(millisecond from arrow_cast(-1500, 'Duration(Microsecond)')),
+ extract(millisecond from arrow_cast(-1500000, 'Duration(Nanosecond)')),
+ extract(second from arrow_cast(-1500, 'Duration(Millisecond)'))
+----
+-1 -1 -1
+
+query I
+SELECT extract(microsecond from arrow_cast(column1, 'Duration(Nanosecond)'))
+FROM (VALUES (5500000000), (NULL), (-62500000000), (NULL))
+----
+5500000
+NULL
+-62500000
+NULL
+
+# around i32::MAX
+query II
+SELECT extract(millisecond from arrow_cast(2147483647, 'Duration(Millisecond)')),
+ extract(microsecond from arrow_cast(2147, 'Duration(Second)'))
+----
+2147483647 2147000000
+
+# values that overflow Int32 are NULL, like the other parts of a duration
+query II
+SELECT extract(millisecond from arrow_cast(column1, 'Duration(Millisecond)')),
+ extract(microsecond from arrow_cast(column2, 'Duration(Second)'))
+FROM (VALUES (2147483648, 2148), (-2147483649, -2148), (NULL, NULL))
+----
+NULL NULL
+NULL NULL
+NULL NULL
+
+# Reproducer from https://github.com/apache/datafusion/issues/14738. Postgres returns 0, because
+# there the difference is an interval. Here it is a Duration, whose microsecond is its total
+# length, and 2.4e9 microseconds don't fit in Int32.
+query I
+SELECT date_part('microsecond', timestamp '1970-01-01T00:40:00' - timestamp '1970-01-01T00:00:00')
+----
+NULL
+
+# Non-null inputs can give NULL results, so the result is nullable.
+# Reproducers from https://github.com/apache/datafusion/issues/26685
+query I
+SELECT extract(second from ts) FROM (VALUES (arrow_cast(9223372036854775807, 'Timestamp(Second, None)'))) AS t(ts)
+----
+NULL
+
+query I
+SELECT extract(hour from d) FROM (VALUES (arrow_cast(9223372036854775807, 'Duration(Second)'))) AS t(d)
+----
+NULL
+
+query I
+SELECT extract(nanosecond from d) FROM (VALUES (arrow_cast(9223372037, 'Duration(Second)'))) AS t(d)
+----
+NULL
+
+query I
+SELECT extract(millisecond from d) FROM (VALUES (arrow_cast(2147483648, 'Duration(Millisecond)'))) AS t(d)
+----
+NULL
+
+# epoch is NULL only for a NULL input, so it keeps the input's nullability
+query TTT
+DESCRIBE SELECT extract(epoch from ts) AS e, extract(second from ts) AS s,
+ extract(millisecond from ts) AS ms, extract(nanosecond from ts) AS ns
+FROM (VALUES (timestamp '2020-01-01')) AS t(ts)
+----
+e Float64 NO
+s Int32 YES
+ms Int32 YES
+ns Int64 YES
+
+query TTT
+DESCRIBE SELECT extract(epoch from ts) AS e, extract(second from ts) AS s
+FROM (VALUES (timestamp '2020-01-01'), (NULL)) AS t(ts)
+----
+e Float64 YES
+s Int32 YES
+
# test_extract_date_part_func
query B
diff --git a/datafusion/sqllogictest/test_files/spark/datetime/date_part.slt b/datafusion/sqllogictest/test_files/spark/datetime/date_part.slt
index beece06d66..566179ba14 100644
--- a/datafusion/sqllogictest/test_files/spark/datetime/date_part.slt
+++ b/datafusion/sqllogictest/test_files/spark/datetime/date_part.slt
@@ -284,3 +284,10 @@ FROM VALUES
1999
2000
NULL
+
+# Nullable even for a non-null input, like the integer parts of the date_part
+# this simplifies to
+query TTT
+DESCRIBE SELECT date_part('SECOND'::string, ts) AS s FROM (VALUES (timestamp '2020-01-01')) AS t(ts)
+----
+s Int32 YES