Apache DataFusion SQL Query Engine
Various refactors to string functions (#19402)
Jeffrey Vo1ce4b51a4827
diff --git a/datafusion/functions/src/string/ends_with.rs b/datafusion/functions/src/string/ends_with.rsindex a1fa124548..f65349a837 100644--- a/datafusion/functions/src/string/ends_with.rs+++ b/datafusion/functions/src/string/ends_with.rs@@ -23,6 +23,7 @@ use arrow::compute::kernels::comparison::ends_with as arrow_ends_with;use arrow::datatypes::DataType;use datafusion_common::types::logical_string;+use datafusion_common::utils::take_function_args;use datafusion_common::{Result, ScalarValue, exec_err};use datafusion_expr::binary::{binary_to_string_coercion, string_coercion};use datafusion_expr::{@@ -95,12 +96,7 @@ impl ScalarUDFImpl for EndsWithFunc {}fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {- let [str_arg, suffix_arg] = args.args.as_slice() else {- return exec_err!(- "ends_with was called with {} arguments, expected 2",- args.args.len()- );- };+ let [str_arg, suffix_arg] = take_function_args(self.name(), &args.args)?;// Determine the common type for coercionlet coercion_type = string_coercion(diff --git a/datafusion/functions/src/string/ltrim.rs b/datafusion/functions/src/string/ltrim.rsindex 18a61869a8..40f525408f 100644--- a/datafusion/functions/src/string/ltrim.rs+++ b/datafusion/functions/src/string/ltrim.rs@@ -21,7 +21,7 @@ use std::any::Any;use std::sync::Arc;use crate::string::common::*;-use crate::utils::{make_scalar_function, utf8_to_str_type};+use crate::utils::make_scalar_function;use datafusion_common::types::logical_string;use datafusion_common::{Result, exec_err};use datafusion_expr::function::Hint;@@ -115,11 +115,7 @@ impl ScalarUDFImpl for LtrimFunc {}fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {- if arg_types[0] == DataType::Utf8View {- Ok(DataType::Utf8View)- } else {- utf8_to_str_type(&arg_types[0], "ltrim")- }+ Ok(arg_types[0].clone())}fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {diff --git a/datafusion/functions/src/string/rtrim.rs b/datafusion/functions/src/string/rtrim.rsindex f0bafc980e..77a08bf533 100644--- a/datafusion/functions/src/string/rtrim.rs+++ b/datafusion/functions/src/string/rtrim.rs@@ -21,7 +21,7 @@ use std::any::Any;use std::sync::Arc;use crate::string::common::*;-use crate::utils::{make_scalar_function, utf8_to_str_type};+use crate::utils::make_scalar_function;use datafusion_common::types::logical_string;use datafusion_common::{Result, exec_err};use datafusion_expr::function::Hint;@@ -115,11 +115,7 @@ impl ScalarUDFImpl for RtrimFunc {}fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {- if arg_types[0] == DataType::Utf8View {- Ok(DataType::Utf8View)- } else {- utf8_to_str_type(&arg_types[0], "rtrim")- }+ Ok(arg_types[0].clone())}fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {diff --git a/datafusion/functions/src/string/split_part.rs b/datafusion/functions/src/string/split_part.rsindex 8ac505bf36..d29d33a154 100644--- a/datafusion/functions/src/string/split_part.rs+++ b/datafusion/functions/src/string/split_part.rs@@ -24,8 +24,11 @@ use arrow::array::{AsArray, GenericStringBuilder};use arrow::datatypes::DataType;use datafusion_common::ScalarValue;use datafusion_common::cast::as_int64_array;+use datafusion_common::types::{NativeType, logical_int64, logical_string};use datafusion_common::{DataFusionError, Result, exec_err};-use datafusion_expr::{ColumnarValue, Documentation, TypeSignature, Volatility};+use datafusion_expr::{+ Coercion, ColumnarValue, Documentation, TypeSignatureClass, Volatility,+};use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl, Signature};use datafusion_macros::user_doc;use std::any::Any;@@ -60,19 +63,16 @@ impl Default for SplitPartFunc {impl SplitPartFunc {pub fn new() -> Self {- use DataType::*;Self {- signature: Signature::one_of(+ signature: Signature::coercible(vec![- TypeSignature::Exact(vec![Utf8View, Utf8View, Int64]),- TypeSignature::Exact(vec![Utf8View, Utf8, Int64]),- TypeSignature::Exact(vec![Utf8View, LargeUtf8, Int64]),- TypeSignature::Exact(vec![Utf8, Utf8View, Int64]),- TypeSignature::Exact(vec![Utf8, Utf8, Int64]),- TypeSignature::Exact(vec![LargeUtf8, Utf8View, Int64]),- TypeSignature::Exact(vec![LargeUtf8, Utf8, Int64]),- TypeSignature::Exact(vec![Utf8, LargeUtf8, Int64]),- TypeSignature::Exact(vec![LargeUtf8, LargeUtf8, Int64]),+ Coercion::new_exact(TypeSignatureClass::Native(logical_string())),+ Coercion::new_exact(TypeSignatureClass::Native(logical_string())),+ Coercion::new_implicit(+ TypeSignatureClass::Native(logical_int64()),+ vec![TypeSignatureClass::Integer],+ NativeType::Int64,+ ),],Volatility::Immutable,),diff --git a/datafusion/functions/src/string/starts_with.rs b/datafusion/functions/src/string/starts_with.rsindex 259612c429..c38a5bffcb 100644--- a/datafusion/functions/src/string/starts_with.rs+++ b/datafusion/functions/src/string/starts_with.rs@@ -21,6 +21,7 @@ use std::sync::Arc;use arrow::array::{ArrayRef, Scalar};use arrow::compute::kernels::comparison::starts_with as arrow_starts_with;use arrow::datatypes::DataType;+use datafusion_common::utils::take_function_args;use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyInfo};use datafusion_expr::type_coercion::binary::{binary_to_string_coercion, string_coercion,@@ -92,12 +93,7 @@ impl ScalarUDFImpl for StartsWithFunc {}fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {- let [str_arg, prefix_arg] = args.args.as_slice() else {- return exec_err!(- "starts_with was called with {} arguments, expected 2",- args.args.len()- );- };+ let [str_arg, prefix_arg] = take_function_args(self.name(), &args.args)?;// Determine the common type for coercionlet coercion_type = string_coercion(diff --git a/datafusion/functions/src/string/to_hex.rs b/datafusion/functions/src/string/to_hex.rsindex dd4f417426..891cbe2549 100644--- a/datafusion/functions/src/string/to_hex.rs+++ b/datafusion/functions/src/string/to_hex.rs@@ -21,20 +21,16 @@ use std::sync::Arc;use crate::utils::make_scalar_function;use arrow::array::{Array, ArrayRef, StringArray};use arrow::buffer::{Buffer, OffsetBuffer};-use arrow::datatypes::DataType::{- Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64, Utf8,-};use arrow::datatypes::{ArrowNativeType, ArrowPrimitiveType, DataType, Int8Type, Int16Type, Int32Type,Int64Type, UInt8Type, UInt16Type, UInt32Type, UInt64Type,};-use datafusion_common::Result;use datafusion_common::cast::as_primitive_array;-use datafusion_common::{exec_err, plan_err};--use datafusion_expr::{ColumnarValue, Documentation};-use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};-use datafusion_expr_common::signature::TypeSignature::Exact;+use datafusion_common::{Result, ScalarValue, exec_err};+use datafusion_expr::{+ Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,+ TypeSignatureClass, Volatility,+};use datafusion_macros::user_doc;/// Hex lookup table for fast conversion@@ -201,17 +197,8 @@ impl Default for ToHexFunc {impl ToHexFunc {pub fn new() -> Self {Self {- signature: Signature::one_of(- vec![- Exact(vec![Int8]),- Exact(vec![Int16]),- Exact(vec![Int32]),- Exact(vec![Int64]),- Exact(vec![UInt8]),- Exact(vec![UInt16]),- Exact(vec![UInt32]),- Exact(vec![UInt64]),- ],+ signature: Signature::coercible(+ vec![Coercion::new_exact(TypeSignatureClass::Integer)],Volatility::Immutable,),}@@ -231,25 +218,37 @@ impl ScalarUDFImpl for ToHexFunc {&self.signature}- fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {- Ok(match arg_types[0] {- Int8 | Int16 | Int32 | Int64 | UInt8 | UInt16 | UInt32 | UInt64 => Utf8,- _ => {- return plan_err!("The to_hex function can only accept integers.");- }- })+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {+ Ok(DataType::Utf8)}fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {match args.args[0].data_type() {- Int64 => make_scalar_function(to_hex::<Int64Type>, vec![])(&args.args),- UInt64 => make_scalar_function(to_hex::<UInt64Type>, vec![])(&args.args),- Int32 => make_scalar_function(to_hex::<Int32Type>, vec![])(&args.args),- UInt32 => make_scalar_function(to_hex::<UInt32Type>, vec![])(&args.args),- Int16 => make_scalar_function(to_hex::<Int16Type>, vec![])(&args.args),- UInt16 => make_scalar_function(to_hex::<UInt16Type>, vec![])(&args.args),- Int8 => make_scalar_function(to_hex::<Int8Type>, vec![])(&args.args),- UInt8 => make_scalar_function(to_hex::<UInt8Type>, vec![])(&args.args),+ DataType::Null => Ok(ColumnarValue::Scalar(ScalarValue::Utf8(None))),+ DataType::Int64 => {+ make_scalar_function(to_hex::<Int64Type>, vec![])(&args.args)+ }+ DataType::UInt64 => {+ make_scalar_function(to_hex::<UInt64Type>, vec![])(&args.args)+ }+ DataType::Int32 => {+ make_scalar_function(to_hex::<Int32Type>, vec![])(&args.args)+ }+ DataType::UInt32 => {+ make_scalar_function(to_hex::<UInt32Type>, vec![])(&args.args)+ }+ DataType::Int16 => {+ make_scalar_function(to_hex::<Int16Type>, vec![])(&args.args)+ }+ DataType::UInt16 => {+ make_scalar_function(to_hex::<UInt16Type>, vec![])(&args.args)+ }+ DataType::Int8 => {+ make_scalar_function(to_hex::<Int8Type>, vec![])(&args.args)+ }+ DataType::UInt8 => {+ make_scalar_function(to_hex::<UInt8Type>, vec![])(&args.args)+ }other => exec_err!("Unsupported data type {other:?} for function to_hex"),}}diff --git a/datafusion/functions/src/string/uuid.rs b/datafusion/functions/src/string/uuid.rsindex 3171eb98fa..3a99412f5e 100644--- a/datafusion/functions/src/string/uuid.rs+++ b/datafusion/functions/src/string/uuid.rs@@ -56,7 +56,7 @@ impl Default for UuidFunc {impl UuidFunc {pub fn new() -> Self {Self {- signature: Signature::exact(vec![], Volatility::Volatile),+ signature: Signature::nullary(Volatility::Volatile),}}}diff --git a/datafusion/sqllogictest/test_files/encoding.slt b/datafusion/sqllogictest/test_files/encoding.sltindex f715f8f46a..ef91eade01 100644--- a/datafusion/sqllogictest/test_files/encoding.slt+++ b/datafusion/sqllogictest/test_files/encoding.slt@@ -73,7 +73,7 @@ select decode('', null) from test;query error DataFusion error: This feature is not implemented: Encoding must be a scalar; array specified encoding is not yet supportedselect decode('', hex_field) from test;-query error DataFusion error: Error during planning: Failed to coerce arguments to satisfy a call to 'to_hex' function+query error DataFusion error: Error during planning: Internal error: Expect TypeSignatureClass::Integer but received NativeType::String, DataType: Utf8Viewselect to_hex(hex_field) from test;query error DataFusion error: Execution error: Failed to decode value using base64