Skip to content

Apache DataFusion SQL Query Engine

perf: optimisation for date_part with seconds (#23444)

theirixd0304b3ff5fc
diff --git a/datafusion/functions/src/datetime/date_part.rs b/datafusion/functions/src/datetime/date_part.rs
index 80f6bc0f66..e3f67db905 100644
--- a/datafusion/functions/src/datetime/date_part.rs
+++ b/datafusion/functions/src/datetime/date_part.rs
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+use std::iter::repeat_n;
use std::str::FromStr;
use std::sync::Arc;
@@ -390,7 +391,7 @@ fn part_normalization(part: &str) -> &str {
/// Invoke [`date_part`] on an `array` (e.g. Timestamp) and convert the
/// result to a total number of seconds, milliseconds, microseconds or
-/// nanoseconds
+/// nanoseconds as an `Int32Array`
fn seconds_as_i32(array: &dyn Array, unit: TimeUnit) -> Result<ArrayRef> {
// Nanosecond is neither supported in Postgres nor DuckDB, to avoid dealing
// with overflow and precision issue we don't support nanosecond
@@ -398,6 +399,19 @@ fn seconds_as_i32(array: &dyn Array, unit: TimeUnit) -> Result<ArrayRef> {
return not_impl_err!("Date part {unit:?} not supported");
}
+ // Fast path with seconds - no need to compute nanoseconds
+ if unit == Second {
+ return Ok(date_part(array, DatePart::Second)?);
+ }
+
+ // Fast path for Date32 and Date64 - no seconds
+ if array.data_type() == &Date32 || array.data_type() == &Date64 {
+ return Ok(Arc::new(Int32Array::from_iter_values_with_nulls(
+ repeat_n(0, array.len()),
+ array.nulls().cloned(),
+ )));
+ }
+
let conversion_factor = match unit {
Second => 1_000_000_000,
Millisecond => 1_000_000,
@@ -539,6 +553,14 @@ fn epoch(array: &dyn Array) -> Result<ArrayRef> {
/// `nanosecond`s in each second, so representing up to 60 seconds as
/// nanoseconds can be values up to 60 billion, which does not fit in Int32.
fn seconds_ns(array: &dyn Array) -> Result<ArrayRef> {
+ // Fast path for Date32 and Date64 - no nanoseconds
+ if array.data_type() == &Date32 || array.data_type() == &Date64 {
+ return Ok(Arc::new(Int64Array::from_iter_values_with_nulls(
+ repeat_n(0, array.len()),
+ array.nulls().cloned(),
+ )));
+ }
+
let secs = date_part(array, DatePart::Second)?;
// This assumes array is primitive and not a dictionary
let secs = as_int32_array(secs.as_ref())?;
diff --git a/datafusion/sqllogictest/test_files/datetime/date_part.slt b/datafusion/sqllogictest/test_files/datetime/date_part.slt
index 891319f9e2..0a992b2d78 100644
--- a/datafusion/sqllogictest/test_files/datetime/date_part.slt
+++ b/datafusion/sqllogictest/test_files/datetime/date_part.slt
@@ -838,6 +838,40 @@ SELECT extract(millisecond from arrow_cast('23:32:50.123456789'::time, 'Time64(N
----
50123
+# date32 and date64
+
+statement ok
+CREATE TABLE source_dt AS
+with t as (values
+ ('1970-01-01'),
+ ('2020-06-02'),
+ ('2026-02-28'),
+ (NULL)
+)
+SELECT
+ arrow_cast(column1, 'Date32') as date32,
+ arrow_cast(column1, 'Date64') as date64,
+FROM t;
+
+query IIIIIIIIII
+SELECT date_part('year', date32), date_part('month', date32), date_part('week', date32), date_part('day', date32), date_part('hour', date32), date_part('minute', date32), date_part('second', date32), date_part('millisecond', date32), date_part('microsecond', date32), date_part('nanosecond', date32) FROM source_dt;
+----
+1970 1 1 1 0 0 0 0 0 0
+2020 6 23 2 0 0 0 0 0 0
+2026 2 9 28 0 0 0 0 0 0
+NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
+
+query IIIIIIIIII
+SELECT date_part('year', date64), date_part('month', date64), date_part('week', date64), date_part('day', date64), date_part('hour', date64), date_part('minute', date64), date_part('second', date64), date_part('millisecond', date64), date_part('microsecond', date64), date_part('nanosecond', date64) FROM source_dt;
+----
+1970 1 1 1 0 0 0 0 0 0
+2020 6 23 2 0 0 0 0 0 0
+2026 2 9 28 0 0 0 0 0 0
+NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
+
+statement ok
+drop table source_dt;
+
# just some floating point stuff happening in the result here
query I
SELECT date_part('microsecond', arrow_cast('23:32:50.123456789'::time, 'Time64(Nanosecond)'))