Skip to content

Apache DataFusion SQL Query Engine

fix: Fix scalar broadcast for to_timestamp() (#20224)

Neil Conway73bce15f292e
diff --git a/datafusion/functions/src/datetime/to_timestamp.rs b/datafusion/functions/src/datetime/to_timestamp.rs
index 738c4acee7..6d40133bd2 100644
--- a/datafusion/functions/src/datetime/to_timestamp.rs
+++ b/datafusion/functions/src/datetime/to_timestamp.rs
@@ -430,27 +430,56 @@ impl ScalarUDFImpl for ToTimestampFunc {
.cast_to(&Timestamp(Second, None), None)?
.cast_to(&Timestamp(Nanosecond, tz), None),
Null | Timestamp(_, _) => args[0].cast_to(&Timestamp(Nanosecond, tz), None),
- Float16 => {
- let arr = args[0].to_array(1)?;
- let f16_arr = downcast_arg!(&arr, Float16Array);
- let result: TimestampNanosecondArray =
- f16_arr.unary(|x| (x.to_f64() * 1_000_000_000.0) as i64);
- Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(tz))))
- }
- Float32 => {
- let arr = args[0].to_array(1)?;
- let f32_arr = downcast_arg!(&arr, Float32Array);
- let result: TimestampNanosecondArray =
- f32_arr.unary(|x| (x as f64 * 1_000_000_000.0) as i64);
- Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(tz))))
- }
- Float64 => {
- let arr = args[0].to_array(1)?;
- let f64_arr = downcast_arg!(&arr, Float64Array);
- let result: TimestampNanosecondArray =
- f64_arr.unary(|x| (x * 1_000_000_000.0) as i64);
- Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(tz))))
- }
+ Float16 => match &args[0] {
+ ColumnarValue::Scalar(ScalarValue::Float16(value)) => {
+ let timestamp_nanos =
+ value.map(|v| (v.to_f64() * 1_000_000_000.0) as i64);
+ Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(
+ timestamp_nanos,
+ tz,
+ )))
+ }
+ ColumnarValue::Array(arr) => {
+ let f16_arr = downcast_arg!(arr, Float16Array);
+ let result: TimestampNanosecondArray =
+ f16_arr.unary(|x| (x.to_f64() * 1_000_000_000.0) as i64);
+ Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(tz))))
+ }
+ _ => exec_err!("Invalid Float16 value for to_timestamp"),
+ },
+ Float32 => match &args[0] {
+ ColumnarValue::Scalar(ScalarValue::Float32(value)) => {
+ let timestamp_nanos =
+ value.map(|v| (v as f64 * 1_000_000_000.0) as i64);
+ Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(
+ timestamp_nanos,
+ tz,
+ )))
+ }
+ ColumnarValue::Array(arr) => {
+ let f32_arr = downcast_arg!(arr, Float32Array);
+ let result: TimestampNanosecondArray =
+ f32_arr.unary(|x| (x as f64 * 1_000_000_000.0) as i64);
+ Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(tz))))
+ }
+ _ => exec_err!("Invalid Float32 value for to_timestamp"),
+ },
+ Float64 => match &args[0] {
+ ColumnarValue::Scalar(ScalarValue::Float64(value)) => {
+ let timestamp_nanos = value.map(|v| (v * 1_000_000_000.0) as i64);
+ Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(
+ timestamp_nanos,
+ tz,
+ )))
+ }
+ ColumnarValue::Array(arr) => {
+ let f64_arr = downcast_arg!(arr, Float64Array);
+ let result: TimestampNanosecondArray =
+ f64_arr.unary(|x| (x * 1_000_000_000.0) as i64);
+ Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(tz))))
+ }
+ _ => exec_err!("Invalid Float64 value for to_timestamp"),
+ },
Decimal32(_, _) | Decimal64(_, _) | Decimal256(_, _) => {
let arg = args[0].cast_to(&Decimal128(38, 9), None)?;
decimal128_to_timestamp_nanos(&arg, tz)
diff --git a/datafusion/sqllogictest/test_files/datetime/arith_date_time.slt b/datafusion/sqllogictest/test_files/datetime/arith_date_time.slt
index bc796a51ff..8e85c8f905 100644
--- a/datafusion/sqllogictest/test_files/datetime/arith_date_time.slt
+++ b/datafusion/sqllogictest/test_files/datetime/arith_date_time.slt
@@ -113,4 +113,3 @@ SELECT '2001-09-28'::date / '03:00'::time
query error Invalid timestamp arithmetic operation
SELECT '2001-09-28'::date % '03:00'::time
-
diff --git a/datafusion/sqllogictest/test_files/datetime/arith_timestamp_duration.slt b/datafusion/sqllogictest/test_files/datetime/arith_timestamp_duration.slt
index 10381346f8..aeeebe73db 100644
--- a/datafusion/sqllogictest/test_files/datetime/arith_timestamp_duration.slt
+++ b/datafusion/sqllogictest/test_files/datetime/arith_timestamp_duration.slt
@@ -144,4 +144,4 @@ query error Invalid timestamp arithmetic operation
SELECT '2001-09-28T01:00:00'::timestamp % arrow_cast(12345, 'Duration(Second)');
query error Invalid timestamp arithmetic operation
-SELECT '2001-09-28T01:00:00'::timestamp / arrow_cast(12345, 'Duration(Second)');
\ No newline at end of file
+SELECT '2001-09-28T01:00:00'::timestamp / arrow_cast(12345, 'Duration(Second)');
diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.slt
index fa25994ed7..8ed32940e8 100644
--- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt
+++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt
@@ -5328,3 +5328,33 @@ drop table ts_data_secs
statement ok
drop table ts_data_micros_kolkata
+
+##########
+## Test to_timestamp with scalar float inputs
+##########
+
+statement ok
+create table test_to_timestamp_scalar(id int, name varchar) as values
+ (1, 'foo'),
+ (2, 'bar');
+
+query P
+SELECT to_timestamp(123.5, name) FROM test_to_timestamp_scalar ORDER BY id;
+----
+1970-01-01T00:02:03.500
+1970-01-01T00:02:03.500
+
+query P
+SELECT to_timestamp(456.789::float, name) FROM test_to_timestamp_scalar ORDER BY id;
+----
+1970-01-01T00:07:36.789001464
+1970-01-01T00:07:36.789001464
+
+query P
+SELECT to_timestamp(arrow_cast(100.5, 'Float16'), name) FROM test_to_timestamp_scalar ORDER BY id;
+----
+1970-01-01T00:01:40.500
+1970-01-01T00:01:40.500
+
+statement ok
+drop table test_to_timestamp_scalar
diff --git a/datafusion/sqllogictest/test_files/limit.slt b/datafusion/sqllogictest/test_files/limit.slt
index 429181a2d3..ec8363f51a 100644
--- a/datafusion/sqllogictest/test_files/limit.slt
+++ b/datafusion/sqllogictest/test_files/limit.slt
@@ -871,4 +871,4 @@ DROP TABLE test_limit_with_partitions;
# Tear down src_table table:
statement ok
-DROP TABLE src_table;
\ No newline at end of file
+DROP TABLE src_table;
diff --git a/datafusion/sqllogictest/test_files/limit_single_row_batches.slt b/datafusion/sqllogictest/test_files/limit_single_row_batches.slt
index fbdb0140e0..9f626816e2 100644
--- a/datafusion/sqllogictest/test_files/limit_single_row_batches.slt
+++ b/datafusion/sqllogictest/test_files/limit_single_row_batches.slt
@@ -19,4 +19,4 @@ SELECT COUNT(*) FROM (SELECT i FROM filter_limit WHERE i <> 0 LIMIT 1);
1
statement ok
-DROP TABLE filter_limit;
\ No newline at end of file
+DROP TABLE filter_limit;
diff --git a/datafusion/sqllogictest/test_files/spark/collection/size.slt b/datafusion/sqllogictest/test_files/spark/collection/size.slt
index dabcfd069b..106760eebf 100644
--- a/datafusion/sqllogictest/test_files/spark/collection/size.slt
+++ b/datafusion/sqllogictest/test_files/spark/collection/size.slt
@@ -129,4 +129,3 @@ SELECT size(column1) FROM VALUES (map(['a'], [1])), (map(['a','b'], [1,2])), (NU
1
2
-1
-
diff --git a/datafusion/sqllogictest/test_files/spark/datetime/time_trunc.slt b/datafusion/sqllogictest/test_files/spark/datetime/time_trunc.slt
index f00c40f0a9..35ffa483bb 100644
--- a/datafusion/sqllogictest/test_files/spark/datetime/time_trunc.slt
+++ b/datafusion/sqllogictest/test_files/spark/datetime/time_trunc.slt
@@ -71,4 +71,3 @@ NULL
# incorrect format
query error DataFusion error: Optimizer rule 'simplify_expressions' failed\ncaused by\nError during planning: The format argument of `TIME_TRUNC` must be one of: hour, minute, second, millisecond, microsecond
SELECT time_trunc('test', '09:32:05.123456'::time);
-
diff --git a/datafusion/sqllogictest/test_files/spark/datetime/trunc.slt b/datafusion/sqllogictest/test_files/spark/datetime/trunc.slt
index f6bf6b5751..aa26d7bd0e 100644
--- a/datafusion/sqllogictest/test_files/spark/datetime/trunc.slt
+++ b/datafusion/sqllogictest/test_files/spark/datetime/trunc.slt
@@ -90,4 +90,3 @@ SELECT trunc('2009-02-12'::date, NULL::string);
# incorrect format
query error DataFusion error: Optimizer rule 'simplify_expressions' failed\ncaused by\nError during planning: The format argument of `TRUNC` must be one of: year, yy, yyyy, month, mm, mon, day, week, quarter.
SELECT trunc('2009-02-12'::date, 'test'::string);
-
diff --git a/datafusion/sqllogictest/test_files/struct.slt b/datafusion/sqllogictest/test_files/struct.slt
index e20815a58c..53a1bb4ec6 100644
--- a/datafusion/sqllogictest/test_files/struct.slt
+++ b/datafusion/sqllogictest/test_files/struct.slt
@@ -1666,4 +1666,4 @@ order by id;
3 2 150
statement ok
-drop table t_agg_window;
\ No newline at end of file
+drop table t_agg_window;
diff --git a/datafusion/sqllogictest/test_files/truncate.slt b/datafusion/sqllogictest/test_files/truncate.slt
index 5a5d47760d..ad3ccbb1a7 100644
--- a/datafusion/sqllogictest/test_files/truncate.slt
+++ b/datafusion/sqllogictest/test_files/truncate.slt
@@ -82,4 +82,4 @@ logical_plan
physical_plan_error
01)TRUNCATE operation on table 't1'
02)caused by
-03)This feature is not implemented: TRUNCATE not supported for Base table
\ No newline at end of file
+03)This feature is not implemented: TRUNCATE not supported for Base table