Apache DataFusion SQL Query Engine
fix: Fix scalar broadcast for to_timestamp() (#20224)
Neil Conway73bce15f292e
diff --git a/datafusion/functions/src/datetime/to_timestamp.rs b/datafusion/functions/src/datetime/to_timestamp.rsindex 738c4acee7..6d40133bd2 100644--- a/datafusion/functions/src/datetime/to_timestamp.rs+++ b/datafusion/functions/src/datetime/to_timestamp.rs@@ -430,27 +430,56 @@ impl ScalarUDFImpl for ToTimestampFunc {.cast_to(&Timestamp(Second, None), None)?.cast_to(&Timestamp(Nanosecond, tz), None),Null | Timestamp(_, _) => args[0].cast_to(&Timestamp(Nanosecond, tz), None),- Float16 => {- let arr = args[0].to_array(1)?;- let f16_arr = downcast_arg!(&arr, Float16Array);- let result: TimestampNanosecondArray =- f16_arr.unary(|x| (x.to_f64() * 1_000_000_000.0) as i64);- Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(tz))))- }- Float32 => {- let arr = args[0].to_array(1)?;- let f32_arr = downcast_arg!(&arr, Float32Array);- let result: TimestampNanosecondArray =- f32_arr.unary(|x| (x as f64 * 1_000_000_000.0) as i64);- Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(tz))))- }- Float64 => {- let arr = args[0].to_array(1)?;- let f64_arr = downcast_arg!(&arr, Float64Array);- let result: TimestampNanosecondArray =- f64_arr.unary(|x| (x * 1_000_000_000.0) as i64);- Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(tz))))- }+ Float16 => match &args[0] {+ ColumnarValue::Scalar(ScalarValue::Float16(value)) => {+ let timestamp_nanos =+ value.map(|v| (v.to_f64() * 1_000_000_000.0) as i64);+ Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(+ timestamp_nanos,+ tz,+ )))+ }+ ColumnarValue::Array(arr) => {+ let f16_arr = downcast_arg!(arr, Float16Array);+ let result: TimestampNanosecondArray =+ f16_arr.unary(|x| (x.to_f64() * 1_000_000_000.0) as i64);+ Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(tz))))+ }+ _ => exec_err!("Invalid Float16 value for to_timestamp"),+ },+ Float32 => match &args[0] {+ ColumnarValue::Scalar(ScalarValue::Float32(value)) => {+ let timestamp_nanos =+ value.map(|v| (v as f64 * 1_000_000_000.0) as i64);+ Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(+ timestamp_nanos,+ tz,+ )))+ }+ ColumnarValue::Array(arr) => {+ let f32_arr = downcast_arg!(arr, Float32Array);+ let result: TimestampNanosecondArray =+ f32_arr.unary(|x| (x as f64 * 1_000_000_000.0) as i64);+ Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(tz))))+ }+ _ => exec_err!("Invalid Float32 value for to_timestamp"),+ },+ Float64 => match &args[0] {+ ColumnarValue::Scalar(ScalarValue::Float64(value)) => {+ let timestamp_nanos = value.map(|v| (v * 1_000_000_000.0) as i64);+ Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(+ timestamp_nanos,+ tz,+ )))+ }+ ColumnarValue::Array(arr) => {+ let f64_arr = downcast_arg!(arr, Float64Array);+ let result: TimestampNanosecondArray =+ f64_arr.unary(|x| (x * 1_000_000_000.0) as i64);+ Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(tz))))+ }+ _ => exec_err!("Invalid Float64 value for to_timestamp"),+ },Decimal32(_, _) | Decimal64(_, _) | Decimal256(_, _) => {let arg = args[0].cast_to(&Decimal128(38, 9), None)?;decimal128_to_timestamp_nanos(&arg, tz)diff --git a/datafusion/sqllogictest/test_files/datetime/arith_date_time.slt b/datafusion/sqllogictest/test_files/datetime/arith_date_time.sltindex bc796a51ff..8e85c8f905 100644--- a/datafusion/sqllogictest/test_files/datetime/arith_date_time.slt+++ b/datafusion/sqllogictest/test_files/datetime/arith_date_time.slt@@ -113,4 +113,3 @@ SELECT '2001-09-28'::date / '03:00'::timequery error Invalid timestamp arithmetic operationSELECT '2001-09-28'::date % '03:00'::time-diff --git a/datafusion/sqllogictest/test_files/datetime/arith_timestamp_duration.slt b/datafusion/sqllogictest/test_files/datetime/arith_timestamp_duration.sltindex 10381346f8..aeeebe73db 100644--- a/datafusion/sqllogictest/test_files/datetime/arith_timestamp_duration.slt+++ b/datafusion/sqllogictest/test_files/datetime/arith_timestamp_duration.slt@@ -144,4 +144,4 @@ query error Invalid timestamp arithmetic operationSELECT '2001-09-28T01:00:00'::timestamp % arrow_cast(12345, 'Duration(Second)');query error Invalid timestamp arithmetic operation-SELECT '2001-09-28T01:00:00'::timestamp / arrow_cast(12345, 'Duration(Second)');\ No newline at end of file+SELECT '2001-09-28T01:00:00'::timestamp / arrow_cast(12345, 'Duration(Second)');diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.sltindex fa25994ed7..8ed32940e8 100644--- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt+++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt@@ -5328,3 +5328,33 @@ drop table ts_data_secsstatement okdrop table ts_data_micros_kolkata++##########+## Test to_timestamp with scalar float inputs+##########++statement ok+create table test_to_timestamp_scalar(id int, name varchar) as values+ (1, 'foo'),+ (2, 'bar');++query P+SELECT to_timestamp(123.5, name) FROM test_to_timestamp_scalar ORDER BY id;+----+1970-01-01T00:02:03.500+1970-01-01T00:02:03.500++query P+SELECT to_timestamp(456.789::float, name) FROM test_to_timestamp_scalar ORDER BY id;+----+1970-01-01T00:07:36.789001464+1970-01-01T00:07:36.789001464++query P+SELECT to_timestamp(arrow_cast(100.5, 'Float16'), name) FROM test_to_timestamp_scalar ORDER BY id;+----+1970-01-01T00:01:40.500+1970-01-01T00:01:40.500++statement ok+drop table test_to_timestamp_scalardiff --git a/datafusion/sqllogictest/test_files/limit.slt b/datafusion/sqllogictest/test_files/limit.sltindex 429181a2d3..ec8363f51a 100644--- a/datafusion/sqllogictest/test_files/limit.slt+++ b/datafusion/sqllogictest/test_files/limit.slt@@ -871,4 +871,4 @@ DROP TABLE test_limit_with_partitions;# Tear down src_table table:statement ok-DROP TABLE src_table;\ No newline at end of file+DROP TABLE src_table;diff --git a/datafusion/sqllogictest/test_files/limit_single_row_batches.slt b/datafusion/sqllogictest/test_files/limit_single_row_batches.sltindex fbdb0140e0..9f626816e2 100644--- a/datafusion/sqllogictest/test_files/limit_single_row_batches.slt+++ b/datafusion/sqllogictest/test_files/limit_single_row_batches.slt@@ -19,4 +19,4 @@ SELECT COUNT(*) FROM (SELECT i FROM filter_limit WHERE i <> 0 LIMIT 1);1statement ok-DROP TABLE filter_limit;\ No newline at end of file+DROP TABLE filter_limit;diff --git a/datafusion/sqllogictest/test_files/spark/collection/size.slt b/datafusion/sqllogictest/test_files/spark/collection/size.sltindex dabcfd069b..106760eebf 100644--- a/datafusion/sqllogictest/test_files/spark/collection/size.slt+++ b/datafusion/sqllogictest/test_files/spark/collection/size.slt@@ -129,4 +129,3 @@ SELECT size(column1) FROM VALUES (map(['a'], [1])), (map(['a','b'], [1,2])), (NU12-1-diff --git a/datafusion/sqllogictest/test_files/spark/datetime/time_trunc.slt b/datafusion/sqllogictest/test_files/spark/datetime/time_trunc.sltindex f00c40f0a9..35ffa483bb 100644--- a/datafusion/sqllogictest/test_files/spark/datetime/time_trunc.slt+++ b/datafusion/sqllogictest/test_files/spark/datetime/time_trunc.slt@@ -71,4 +71,3 @@ NULL# incorrect formatquery error DataFusion error: Optimizer rule 'simplify_expressions' failed\ncaused by\nError during planning: The format argument of `TIME_TRUNC` must be one of: hour, minute, second, millisecond, microsecondSELECT time_trunc('test', '09:32:05.123456'::time);-diff --git a/datafusion/sqllogictest/test_files/spark/datetime/trunc.slt b/datafusion/sqllogictest/test_files/spark/datetime/trunc.sltindex f6bf6b5751..aa26d7bd0e 100644--- a/datafusion/sqllogictest/test_files/spark/datetime/trunc.slt+++ b/datafusion/sqllogictest/test_files/spark/datetime/trunc.slt@@ -90,4 +90,3 @@ SELECT trunc('2009-02-12'::date, NULL::string);# incorrect formatquery error DataFusion error: Optimizer rule 'simplify_expressions' failed\ncaused by\nError during planning: The format argument of `TRUNC` must be one of: year, yy, yyyy, month, mm, mon, day, week, quarter.SELECT trunc('2009-02-12'::date, 'test'::string);-diff --git a/datafusion/sqllogictest/test_files/struct.slt b/datafusion/sqllogictest/test_files/struct.sltindex e20815a58c..53a1bb4ec6 100644--- a/datafusion/sqllogictest/test_files/struct.slt+++ b/datafusion/sqllogictest/test_files/struct.slt@@ -1666,4 +1666,4 @@ order by id;3 2 150statement ok-drop table t_agg_window;\ No newline at end of file+drop table t_agg_window;diff --git a/datafusion/sqllogictest/test_files/truncate.slt b/datafusion/sqllogictest/test_files/truncate.sltindex 5a5d47760d..ad3ccbb1a7 100644--- a/datafusion/sqllogictest/test_files/truncate.slt+++ b/datafusion/sqllogictest/test_files/truncate.slt@@ -82,4 +82,4 @@ logical_planphysical_plan_error01)TRUNCATE operation on table 't1'02)caused by-03)This feature is not implemented: TRUNCATE not supported for Base table\ No newline at end of file+03)This feature is not implemented: TRUNCATE not supported for Base table