Apache DataFusion SQL Query Engine
fix: support float16 for `abs()` (#18304)
Jeffrey Vo410d29238d39
diff --git a/Cargo.lock b/Cargo.lockindex aaa75ecf32..c6e2855576 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2265,6 +2265,7 @@ dependencies = ["itertools 0.14.0","log","md-5",+ "num-traits","rand 0.9.2","regex","sha2",diff --git a/Cargo.toml b/Cargo.tomlindex 1cfb23bb18..e48afb19ff 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -159,6 +159,7 @@ indexmap = "2.12.0"insta = { version = "1.43.2", features = ["glob", "filters"] }itertools = "0.14"log = "^0.4"+num-traits = { version = "0.2" }object_store = { version = "0.12.4", default-features = false }parking_lot = "0.12"parquet = { version = "57.0.0", default-features = false, features = [diff --git a/datafusion/datasource-avro/Cargo.toml b/datafusion/datasource-avro/Cargo.tomlindex e013e8a3d0..6bab899e7f 100644--- a/datafusion/datasource-avro/Cargo.toml+++ b/datafusion/datasource-avro/Cargo.toml@@ -41,7 +41,7 @@ datafusion-physical-expr-common = { workspace = true }datafusion-physical-plan = { workspace = true }datafusion-session = { workspace = true }futures = { workspace = true }-num-traits = { version = "0.2" }+num-traits = { workspace = true }object_store = { workspace = true }[dev-dependencies]diff --git a/datafusion/functions/Cargo.toml b/datafusion/functions/Cargo.tomlindex 90331fbcca..1dbeee7159 100644--- a/datafusion/functions/Cargo.toml+++ b/datafusion/functions/Cargo.toml@@ -78,6 +78,7 @@ hex = { version = "0.4", optional = true }itertools = { workspace = true }log = { workspace = true }md-5 = { version = "^0.10.0", optional = true }+num-traits = { workspace = true }rand = { workspace = true }regex = { workspace = true, optional = true }sha2 = { version = "^0.10.9", optional = true }diff --git a/datafusion/functions/src/math/abs.rs b/datafusion/functions/src/math/abs.rsindex 040f13c014..b3dc2b2eb6 100644--- a/datafusion/functions/src/math/abs.rs+++ b/datafusion/functions/src/math/abs.rs@@ -22,7 +22,8 @@ use std::sync::Arc;use arrow::array::{ArrayRef, Decimal128Array, Decimal256Array, Decimal32Array, Decimal64Array,- Float32Array, Float64Array, Int16Array, Int32Array, Int64Array, Int8Array,+ Float16Array, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array,+ Int8Array,};use arrow::datatypes::DataType;use arrow::error::ArrowError;@@ -34,6 +35,7 @@ use datafusion_expr::{Volatility,};use datafusion_macros::user_doc;+use num_traits::sign::Signed;type MathArrayFunction = fn(&ArrayRef) -> Result<ArrayRef>;@@ -81,6 +83,7 @@ macro_rules! make_decimal_abs_function {/// Return different implementations based on input datatype to reduce branches during executionfn create_abs_function(input_data_type: &DataType) -> Result<MathArrayFunction> {match input_data_type {+ DataType::Float16 => Ok(make_abs_function!(Float16Array)),DataType::Float32 => Ok(make_abs_function!(Float32Array)),DataType::Float64 => Ok(make_abs_function!(Float64Array)),@@ -143,6 +146,7 @@ impl ScalarUDFImpl for AbsFunc {fn as_any(&self) -> &dyn Any {self}+fn name(&self) -> &str {"abs"}@@ -152,35 +156,7 @@ impl ScalarUDFImpl for AbsFunc {}fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {- match arg_types[0] {- DataType::Float32 => Ok(DataType::Float32),- DataType::Float64 => Ok(DataType::Float64),- DataType::Int8 => Ok(DataType::Int8),- DataType::Int16 => Ok(DataType::Int16),- DataType::Int32 => Ok(DataType::Int32),- DataType::Int64 => Ok(DataType::Int64),- DataType::Null => Ok(DataType::Null),- DataType::UInt8 => Ok(DataType::UInt8),- DataType::UInt16 => Ok(DataType::UInt16),- DataType::UInt32 => Ok(DataType::UInt32),- DataType::UInt64 => Ok(DataType::UInt64),- DataType::Decimal32(precision, scale) => {- Ok(DataType::Decimal32(precision, scale))- }- DataType::Decimal64(precision, scale) => {- Ok(DataType::Decimal64(precision, scale))- }- DataType::Decimal128(precision, scale) => {- Ok(DataType::Decimal128(precision, scale))- }- DataType::Decimal256(precision, scale) => {- Ok(DataType::Decimal256(precision, scale))- }- _ => not_impl_err!(- "Unsupported data type {} for function abs",- arg_types[0].to_string()- ),- }+ Ok(arg_types[0].clone())}fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {diff --git a/datafusion/sqllogictest/test_files/math.slt b/datafusion/sqllogictest/test_files/math.sltindex e206aa16b8..1cb68b85b2 100644--- a/datafusion/sqllogictest/test_files/math.slt+++ b/datafusion/sqllogictest/test_files/math.slt@@ -139,16 +139,16 @@ select abs(arrow_cast('-1.2', 'Utf8'));statement okCREATE TABLE test_nullable_integer(- c1 TINYINT,- c2 SMALLINT,- c3 INT,- c4 BIGINT,- c5 TINYINT UNSIGNED,- c6 SMALLINT UNSIGNED,- c7 INT UNSIGNED,- c8 BIGINT UNSIGNED,+ c1 TINYINT,+ c2 SMALLINT,+ c3 INT,+ c4 BIGINT,+ c5 TINYINT UNSIGNED,+ c6 SMALLINT UNSIGNED,+ c7 INT UNSIGNED,+ c8 BIGINT UNSIGNED,dataset TEXT- )+ )AS VALUES(NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, 'nulls'),(0, 0, 0, 0, 0, 0, 0, 0, 'zeros'),@@ -237,7 +237,7 @@ SELECT c8%0 FROM test_nullable_integer# abs: return typequery TTTTTTTT rowsort-select+selectarrow_typeof(abs(c1)), arrow_typeof(abs(c2)), arrow_typeof(abs(c3)), arrow_typeof(abs(c4)),arrow_typeof(abs(c5)), arrow_typeof(abs(c6)), arrow_typeof(abs(c7)), arrow_typeof(abs(c8))from test_nullable_integer limit 1@@ -285,13 +285,13 @@ drop table test_nullable_integerstatement okCREATE TABLE test_non_nullable_integer(- c1 TINYINT NOT NULL,- c2 SMALLINT NOT NULL,- c3 INT NOT NULL,- c4 BIGINT NOT NULL,- c5 TINYINT UNSIGNED NOT NULL,- c6 SMALLINT UNSIGNED NOT NULL,- c7 INT UNSIGNED NOT NULL,+ c1 TINYINT NOT NULL,+ c2 SMALLINT NOT NULL,+ c3 INT NOT NULL,+ c4 BIGINT NOT NULL,+ c5 TINYINT UNSIGNED NOT NULL,+ c6 SMALLINT UNSIGNED NOT NULL,+ c7 INT UNSIGNED NOT NULL,c8 BIGINT UNSIGNED NOT NULL);@@ -363,7 +363,7 @@ CREATE TABLE test_nullable_float(c2 double) AS VALUES(-1.0, -1.0),- (1.0, 1.0),+ (1.0, 1.0),(NULL, NULL),(0., 0.),('NaN'::double, 'NaN'::double);@@ -412,7 +412,7 @@ Float32 Float64# abs: floatsquery RR rowsort-SELECT abs(c1), abs(c2) from test_nullable_float+SELECT abs(c1), abs(c2) from test_nullable_float----0 01 1@@ -420,6 +420,17 @@ SELECT abs(c1), abs(c2) from test_nullable_floatNULL NULLNaN NaN+# f16+query TR rowsort+SELECT arrow_typeof(abs(arrow_cast(c1, 'Float16'))), abs(arrow_cast(c1, 'Float16'))+FROM test_nullable_float+----+Float16 0+Float16 1+Float16 1+Float16 NULL+Float16 NaN+statement okdrop table test_nullable_float@@ -428,7 +439,7 @@ statement okCREATE TABLE test_non_nullable_float(c1 float NOT NULL,c2 double NOT NULL- );+ );query IINSERT INTO test_non_nullable_float VALUES@@ -478,27 +489,27 @@ drop table test_non_nullable_floatstatement okCREATE TABLE test_nullable_decimal(c1 DECIMAL(10, 2), /* Decimal128 */- c2 DECIMAL(38, 10), /* Decimal128 with max precision */+ c2 DECIMAL(38, 10), /* Decimal128 with max precision */c3 DECIMAL(40, 2), /* Decimal256 */- c4 DECIMAL(76, 10) /* Decimal256 with max precision */- ) AS VALUES- (0, 0, 0, 0),+ c4 DECIMAL(76, 10) /* Decimal256 with max precision */+ ) AS VALUES+ (0, 0, 0, 0),(NULL, NULL, NULL, NULL);query IINSERT into test_nullable_decimal values(- -99999999.99,- '-9999999999999999999999999999.9999999999',- '-99999999999999999999999999999999999999.99',+ -99999999.99,+ '-9999999999999999999999999999.9999999999',+ '-99999999999999999999999999999999999999.99','-999999999999999999999999999999999999999999999999999999999999999999.9999999999'- ),+ ),(- 99999999.99,- '9999999999999999999999999999.9999999999',- '99999999999999999999999999999999999999.99',+ 99999999.99,+ '9999999999999999999999999999.9999999999',+ '99999999999999999999999999999999999999.99','999999999999999999999999999999999999999999999999999999999999999999.9999999999'- )+ )----2@@ -533,9 +544,9 @@ SELECT c1%0 FROM test_nullable_decimal WHERE c1 IS NOT NULL;# abs: return typequery TTTT-SELECT- arrow_typeof(abs(c1)),- arrow_typeof(abs(c2)),+SELECT+ arrow_typeof(abs(c1)),+ arrow_typeof(abs(c2)),arrow_typeof(abs(c3)),arrow_typeof(abs(c4))FROM test_nullable_decimal limit 1@@ -552,11 +563,11 @@ SELECT abs(c1), abs(c2), abs(c3), abs(c4) FROM test_nullable_decimalNULL NULL NULL NULLstatement ok-drop table test_nullable_decimal+drop table test_nullable_decimalstatement ok-CREATE TABLE test_non_nullable_decimal(c1 DECIMAL(9,2) NOT NULL);+CREATE TABLE test_non_nullable_decimal(c1 DECIMAL(9,2) NOT NULL);query IINSERT INTO test_non_nullable_decimal VALUES(1)@@ -569,13 +580,13 @@ SELECT c1*0 FROM test_non_nullable_decimal0query error DataFusion error: Arrow error: Divide by zero error-SELECT c1/0 FROM test_non_nullable_decimal+SELECT c1/0 FROM test_non_nullable_decimalquery error DataFusion error: Arrow error: Divide by zero error-SELECT c1%0 FROM test_non_nullable_decimal+SELECT c1%0 FROM test_non_nullable_decimalstatement ok-drop table test_non_nullable_decimal+drop table test_non_nullable_decimalstatement okCREATE TABLE signed_integers(@@ -615,7 +626,7 @@ NULL NULL NULL# scalar maxes and/or negative 1query III-select+selectgcd(9223372036854775807, -9223372036854775808), -- i64::MAX, i64::MINgcd(9223372036854775807, -1), -- i64::MAX, -1gcd(-9223372036854775808, -1); -- i64::MIN, -1