Apache DataFusion SQL Query Engine
Update reverse UDF to emit utf8view when input is utf8view (#20604)
Bruce Ritchieb51edffc44f2
diff --git a/datafusion/functions/src/unicode/reverse.rs b/datafusion/functions/src/unicode/reverse.rsindex 71a1e6842f..7b501c0a65 100644--- a/datafusion/functions/src/unicode/reverse.rs+++ b/datafusion/functions/src/unicode/reverse.rs@@ -18,10 +18,11 @@use std::any::Any;use std::sync::Arc;-use crate::utils::{make_scalar_function, utf8_to_str_type};+use crate::utils::make_scalar_function;use DataType::{LargeUtf8, Utf8, Utf8View};use arrow::array::{- Array, ArrayRef, AsArray, GenericStringBuilder, OffsetSizeTrait, StringArrayType,+ Array, ArrayRef, AsArray, LargeStringBuilder, StringArrayType, StringBuilder,+ StringLikeArrayBuilder, StringViewBuilder,};use arrow::datatypes::DataType;use datafusion_common::{Result, exec_err};@@ -82,7 +83,7 @@ impl ScalarUDFImpl for ReverseFunc {}fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {- utf8_to_str_type(&arg_types[0], "reverse")+ Ok(arg_types[0].clone())}fn invoke_with_args(@@ -91,8 +92,7 @@ impl ScalarUDFImpl for ReverseFunc {) -> Result<ColumnarValue> {let args = &args.args;match args[0].data_type() {- Utf8 | Utf8View => make_scalar_function(reverse::<i32>, vec![])(args),- LargeUtf8 => make_scalar_function(reverse::<i64>, vec![])(args),+ Utf8 | Utf8View | LargeUtf8 => make_scalar_function(reverse, vec![])(args),other => {exec_err!("Unsupported data type {other:?} for function reverse")}@@ -106,21 +106,39 @@ impl ScalarUDFImpl for ReverseFunc {/// Reverses the order of the characters in the string `reverse('abcde') = 'edcba'`./// The implementation uses UTF-8 code points as characters-fn reverse<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {- if args[0].data_type() == &Utf8View {- reverse_impl::<T, _>(&args[0].as_string_view())- } else {- reverse_impl::<T, _>(&args[0].as_string::<T>())+fn reverse(args: &[ArrayRef]) -> Result<ArrayRef> {+ let len = args[0].len();++ match args[0].data_type() {+ Utf8 => reverse_impl(+ &args[0].as_string::<i32>(),+ StringBuilder::with_capacity(len, 1024),+ ),+ Utf8View => reverse_impl(+ &args[0].as_string_view(),+ StringViewBuilder::with_capacity(len),+ ),+ LargeUtf8 => reverse_impl(+ &args[0].as_string::<i64>(),+ LargeStringBuilder::with_capacity(len, 1024),+ ),+ _ => unreachable!(+ "Reverse can only be applied to Utf8View, Utf8 and LargeUtf8 types"+ ),}}-fn reverse_impl<'a, T: OffsetSizeTrait, V: StringArrayType<'a>>(- string_array: &V,-) -> Result<ArrayRef> {- let mut builder = GenericStringBuilder::<T>::with_capacity(string_array.len(), 1024);-+fn reverse_impl<'a, StringArrType, StringBuilderType>(+ string_array: &StringArrType,+ mut array_builder: StringBuilderType,+) -> Result<ArrayRef>+where+ StringArrType: StringArrayType<'a>,+ StringBuilderType: StringLikeArrayBuilder,+{let mut string_buf = String::new();let mut byte_buf = Vec::<u8>::new();+for string in string_array.iter() {if let Some(s) = string {if s.is_ascii() {@@ -129,25 +147,25 @@ fn reverse_impl<'a, T: OffsetSizeTrait, V: StringArrayType<'a>>(byte_buf.reverse();// SAFETY: Since the original string was ASCII, reversing the bytes still results in valid UTF-8.let reversed = unsafe { std::str::from_utf8_unchecked(&byte_buf) };- builder.append_value(reversed);+ array_builder.append_value(reversed);byte_buf.clear();} else {string_buf.extend(s.chars().rev());- builder.append_value(&string_buf);+ array_builder.append_value(&string_buf);string_buf.clear();}} else {- builder.append_null();+ array_builder.append_null();}}- Ok(Arc::new(builder.finish()) as ArrayRef)+ Ok(Arc::new(array_builder.finish()) as ArrayRef)}#[cfg(test)]mod tests {- use arrow::array::{Array, LargeStringArray, StringArray};- use arrow::datatypes::DataType::{LargeUtf8, Utf8};+ use arrow::array::{Array, LargeStringArray, StringArray, StringViewArray};+ use arrow::datatypes::DataType::{LargeUtf8, Utf8, Utf8View};use datafusion_common::{Result, ScalarValue};use datafusion_expr::{ColumnarValue, ScalarUDFImpl};@@ -180,8 +198,8 @@ mod tests {vec![ColumnarValue::Scalar(ScalarValue::Utf8View($INPUT))],$EXPECTED,&str,- Utf8,- StringArray+ Utf8View,+ StringViewArray);};}diff --git a/datafusion/sqllogictest/test_files/string/string_literal.slt b/datafusion/sqllogictest/test_files/string/string_literal.sltindex 6cf0221887..2272de74cf 100644--- a/datafusion/sqllogictest/test_files/string/string_literal.slt+++ b/datafusion/sqllogictest/test_files/string/string_literal.slt@@ -389,6 +389,21 @@ SELECT reverse(arrow_cast('abcde', 'Utf8View'))----edcba+query T+SELECT arrow_typeof(reverse('abcde'))+----+Utf8++query T+SELECT arrow_typeof(reverse(arrow_cast('abcde', 'LargeUtf8')))+----+LargeUtf8++query T+SELECT arrow_typeof(reverse(arrow_cast('abcde', 'Utf8View')))+----+Utf8View+query TSELECT reverse(arrow_cast('abcde', 'Dictionary(Int32, Utf8)'))----