Apache DataFusion SQL Query Engine
perf: preserve dictionary encoding for `btrim`, `ltrim`, and `rtrim` (#24100)
linfeng6b6cc0939ef5
diff --git a/datafusion/functions/benches/dictionary_encoding.rs b/datafusion/functions/benches/dictionary_encoding.rsindex 05541fc10e..4ba04a4940 100644--- a/datafusion/functions/benches/dictionary_encoding.rs+++ b/datafusion/functions/benches/dictionary_encoding.rs@@ -24,7 +24,7 @@ use arrow::datatypes::{Field, Int32Type};use criterion::{Criterion, criterion_group, criterion_main};use datafusion_common::config::ConfigOptions;use datafusion_expr::type_coercion::functions::fields_with_udf;-use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDF};+use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};const NUM_ROWS: usize = 8_192;const DICTIONARY_CARDINALITIES: [usize; 4] = [10, 100, 1_000, 8_192];@@ -42,16 +42,19 @@ fn create_string_dictionary(cardinality: usize) -> ArrayRef {}fn benchmark_dictionary_string_udfs(c: &mut Criterion) {- let udfs: [(&str, Arc<ScalarUDF>); 6] = [+ let udfs = [("ascii", datafusion_functions::string::ascii()),("bit_length", datafusion_functions::string::bit_length()),+ ("btrim", datafusion_functions::string::btrim()),("character_length",datafusion_functions::unicode::character_length(),),("initcap", datafusion_functions::unicode::initcap()),+ ("ltrim", datafusion_functions::string::ltrim()),("octet_length", datafusion_functions::string::octet_length()),("reverse", datafusion_functions::unicode::reverse()),+ ("rtrim", datafusion_functions::string::rtrim()),];let config_options = Arc::new(ConfigOptions::default());diff --git a/datafusion/functions/src/string/btrim.rs b/datafusion/functions/src/string/btrim.rsindex 279f444d9f..82e1f7d2c7 100644--- a/datafusion/functions/src/string/btrim.rs+++ b/datafusion/functions/src/string/btrim.rs@@ -16,30 +16,41 @@// under the License.use crate::string::common::*;-use crate::utils::{make_scalar_function, utf8_to_str_type};-use arrow::array::{ArrayRef, OffsetSizeTrait};+use crate::utils::make_scalar_function;+use arrow::array::{ArrayRef, AsArray};use arrow::datatypes::DataType;use datafusion_common::types::logical_string;use datafusion_common::{Result, exec_err};use datafusion_expr::function::Hint;use datafusion_expr::{- Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,- TypeSignature, TypeSignatureClass, Volatility,+ Coercion, ColumnarValue, Documentation, EncodingPreservation, ScalarFunctionArgs,+ ScalarUDFImpl, Signature, TypeSignature, TypeSignatureClass, Volatility,};use datafusion_macros::user_doc;use std::sync::Arc;/// Returns the longest string with leading and trailing characters removed. If the characters are not specified, spaces are removed./// btrim('xyxtrimyyx', 'xyz') = 'trim'-fn btrim<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {- let use_string_view = args[0].data_type() == &DataType::Utf8View;+fn btrim(args: &[ArrayRef]) -> Result<ArrayRef> {let args = if args.len() > 1 {let arg1 = arrow::compute::kernels::cast::cast(&args[1], args[0].data_type())?;vec![Arc::clone(&args[0]), arg1]} else {args.to_owned()};- general_trim::<T, TrimBoth>(&args, use_string_view)+ match args[0].data_type() {+ DataType::Utf8 => general_trim::<i32, TrimBoth>(&args, false),+ DataType::LargeUtf8 => general_trim::<i64, TrimBoth>(&args, false),+ DataType::Utf8View => general_trim::<i32, TrimBoth>(&args, true),+ DataType::Dictionary(_, _) => {+ let dictionary = args[0].as_any_dictionary();+ let trimmed = btrim(&[Arc::clone(dictionary.values())])?;+ Ok(dictionary.with_values(trimmed))+ }+ other => exec_err!(+ "Unsupported data type {other:?} for function btrim, expected Utf8, LargeUtf8 or Utf8View."+ ),+ }}#[user_doc(@@ -85,9 +96,12 @@ impl BTrimFunc {Coercion::new_exact(TypeSignatureClass::Native(logical_string())),Coercion::new_exact(TypeSignatureClass::Native(logical_string())),]),- TypeSignature::Coercible(vec![Coercion::new_exact(- TypeSignatureClass::Native(logical_string()),- )]),+ TypeSignature::Coercible(vec![+ Coercion::new_exact(TypeSignatureClass::Native(logical_string()))+ .with_encoding_preservation(+ EncodingPreservation::dictionary(),+ ),+ ]),],Volatility::Immutable,),@@ -106,28 +120,11 @@ impl ScalarUDFImpl for BTrimFunc {}fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {- if arg_types[0] == DataType::Utf8View {- Ok(DataType::Utf8View)- } else {- utf8_to_str_type(&arg_types[0], "btrim")- }+ Ok(arg_types[0].clone())}fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {- match args.args[0].data_type() {- DataType::Utf8 | DataType::Utf8View => make_scalar_function(- btrim::<i32>,- vec![Hint::Pad, Hint::AcceptsSingular],- )(&args.args),- DataType::LargeUtf8 => make_scalar_function(- btrim::<i64>,- vec![Hint::Pad, Hint::AcceptsSingular],- )(&args.args),- other => exec_err!(- "Unsupported data type {other:?} for function btrim,\- expected Utf8, LargeUtf8 or Utf8View."- ),- }+ make_scalar_function(btrim, vec![Hint::Pad, Hint::AcceptsSingular])(&args.args)}fn aliases(&self) -> &[String] {diff --git a/datafusion/functions/src/string/ltrim.rs b/datafusion/functions/src/string/ltrim.rsindex e49ffeb054..04e33253ed 100644--- a/datafusion/functions/src/string/ltrim.rs+++ b/datafusion/functions/src/string/ltrim.rs@@ -15,7 +15,7 @@// specific language governing permissions and limitations// under the License.-use arrow::array::{ArrayRef, OffsetSizeTrait};+use arrow::array::{ArrayRef, AsArray};use arrow::datatypes::DataType;use std::sync::Arc;@@ -25,22 +25,33 @@ use datafusion_common::types::logical_string;use datafusion_common::{Result, exec_err};use datafusion_expr::function::Hint;use datafusion_expr::{- Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,- TypeSignature, TypeSignatureClass, Volatility,+ Coercion, ColumnarValue, Documentation, EncodingPreservation, ScalarFunctionArgs,+ ScalarUDFImpl, Signature, TypeSignature, TypeSignatureClass, Volatility,};use datafusion_macros::user_doc;/// Returns the longest string with leading characters removed. If the characters are not specified, spaces are removed./// ltrim('zzzytest', 'xyz') = 'test'-fn ltrim<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {- let use_string_view = args[0].data_type() == &DataType::Utf8View;+fn ltrim(args: &[ArrayRef]) -> Result<ArrayRef> {let args = if args.len() > 1 {let arg1 = arrow::compute::kernels::cast::cast(&args[1], args[0].data_type())?;vec![Arc::clone(&args[0]), arg1]} else {args.to_owned()};- general_trim::<T, TrimLeft>(&args, use_string_view)+ match args[0].data_type() {+ DataType::Utf8 => general_trim::<i32, TrimLeft>(&args, false),+ DataType::LargeUtf8 => general_trim::<i64, TrimLeft>(&args, false),+ DataType::Utf8View => general_trim::<i32, TrimLeft>(&args, true),+ DataType::Dictionary(_, _) => {+ let dictionary = args[0].as_any_dictionary();+ let trimmed = ltrim(&[Arc::clone(dictionary.values())])?;+ Ok(dictionary.with_values(trimmed))+ }+ other => exec_err!(+ "Unsupported data type {other:?} for function ltrim, expected Utf8, LargeUtf8 or Utf8View."+ ),+ }}#[user_doc(@@ -90,9 +101,12 @@ impl LtrimFunc {Coercion::new_exact(TypeSignatureClass::Native(logical_string())),Coercion::new_exact(TypeSignatureClass::Native(logical_string())),]),- TypeSignature::Coercible(vec![Coercion::new_exact(- TypeSignatureClass::Native(logical_string()),- )]),+ TypeSignature::Coercible(vec![+ Coercion::new_exact(TypeSignatureClass::Native(logical_string()))+ .with_encoding_preservation(+ EncodingPreservation::dictionary(),+ ),+ ]),],Volatility::Immutable,),@@ -114,20 +128,7 @@ impl ScalarUDFImpl for LtrimFunc {}fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {- match args.args[0].data_type() {- DataType::Utf8 | DataType::Utf8View => make_scalar_function(- ltrim::<i32>,- vec![Hint::Pad, Hint::AcceptsSingular],- )(&args.args),- DataType::LargeUtf8 => make_scalar_function(- ltrim::<i64>,- vec![Hint::Pad, Hint::AcceptsSingular],- )(&args.args),- other => exec_err!(- "Unsupported data type {other:?} for function ltrim,\- expected Utf8, LargeUtf8 or Utf8View."- ),- }+ make_scalar_function(ltrim, vec![Hint::Pad, Hint::AcceptsSingular])(&args.args)}fn documentation(&self) -> Option<&Documentation> {diff --git a/datafusion/functions/src/string/rtrim.rs b/datafusion/functions/src/string/rtrim.rsindex 05ad9e8559..d1126cb241 100644--- a/datafusion/functions/src/string/rtrim.rs+++ b/datafusion/functions/src/string/rtrim.rs@@ -15,7 +15,7 @@// specific language governing permissions and limitations// under the License.-use arrow::array::{ArrayRef, OffsetSizeTrait};+use arrow::array::{ArrayRef, AsArray};use arrow::datatypes::DataType;use std::sync::Arc;@@ -25,22 +25,33 @@ use datafusion_common::types::logical_string;use datafusion_common::{Result, exec_err};use datafusion_expr::function::Hint;use datafusion_expr::{- Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,- TypeSignature, TypeSignatureClass, Volatility,+ Coercion, ColumnarValue, Documentation, EncodingPreservation, ScalarFunctionArgs,+ ScalarUDFImpl, Signature, TypeSignature, TypeSignatureClass, Volatility,};use datafusion_macros::user_doc;/// Returns the longest string with trailing characters removed. If the characters are not specified, spaces are removed./// rtrim('testxxzx', 'xyz') = 'test'-fn rtrim<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {- let use_string_view = args[0].data_type() == &DataType::Utf8View;+fn rtrim(args: &[ArrayRef]) -> Result<ArrayRef> {let args = if args.len() > 1 {let arg1 = arrow::compute::kernels::cast::cast(&args[1], args[0].data_type())?;vec![Arc::clone(&args[0]), arg1]} else {args.to_owned()};- general_trim::<T, TrimRight>(&args, use_string_view)+ match args[0].data_type() {+ DataType::Utf8 => general_trim::<i32, TrimRight>(&args, false),+ DataType::LargeUtf8 => general_trim::<i64, TrimRight>(&args, false),+ DataType::Utf8View => general_trim::<i32, TrimRight>(&args, true),+ DataType::Dictionary(_, _) => {+ let dictionary = args[0].as_any_dictionary();+ let trimmed = rtrim(&[Arc::clone(dictionary.values())])?;+ Ok(dictionary.with_values(trimmed))+ }+ other => exec_err!(+ "Unsupported data type {other:?} for function rtrim, expected Utf8, LargeUtf8 or Utf8View."+ ),+ }}#[user_doc(@@ -90,9 +101,12 @@ impl RtrimFunc {Coercion::new_exact(TypeSignatureClass::Native(logical_string())),Coercion::new_exact(TypeSignatureClass::Native(logical_string())),]),- TypeSignature::Coercible(vec![Coercion::new_exact(- TypeSignatureClass::Native(logical_string()),- )]),+ TypeSignature::Coercible(vec![+ Coercion::new_exact(TypeSignatureClass::Native(logical_string()))+ .with_encoding_preservation(+ EncodingPreservation::dictionary(),+ ),+ ]),],Volatility::Immutable,),@@ -114,20 +128,7 @@ impl ScalarUDFImpl for RtrimFunc {}fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {- match args.args[0].data_type() {- DataType::Utf8 | DataType::Utf8View => make_scalar_function(- rtrim::<i32>,- vec![Hint::Pad, Hint::AcceptsSingular],- )(&args.args),- DataType::LargeUtf8 => make_scalar_function(- rtrim::<i64>,- vec![Hint::Pad, Hint::AcceptsSingular],- )(&args.args),- other => exec_err!(- "Unsupported data type {other:?} for function rtrim,\- expected Utf8, LargeUtf8 or Utf8View."- ),- }+ make_scalar_function(rtrim, vec![Hint::Pad, Hint::AcceptsSingular])(&args.args)}fn documentation(&self) -> Option<&Documentation> {diff --git a/datafusion/sqllogictest/test_files/functions.slt b/datafusion/sqllogictest/test_files/functions.sltindex 78bdeb3e15..008be05852 100644--- a/datafusion/sqllogictest/test_files/functions.slt+++ b/datafusion/sqllogictest/test_files/functions.slt@@ -492,10 +492,11 @@ SELECT btrim(' foo ')----foo-query T-SELECT btrim(arrow_cast(' foo ', 'Dictionary(Int32, Utf8)'))+query TT+SELECT btrim(arrow_cast(' foo ', 'Dictionary(Int32, Utf8)')),+ arrow_typeof(btrim(arrow_cast(' foo ', 'Dictionary(Int32, Utf8)')))-----foo+foo Dictionary(Int32, Utf8)query TSELECT initcap('foo')@@ -689,10 +690,11 @@ SELECT ltrim(' foo')----foo-query T-SELECT ltrim(arrow_cast(' foo', 'Dictionary(Int32, Utf8)'))+query TT+SELECT ltrim(arrow_cast(' foo', 'Dictionary(Int32, Utf8)')),+ arrow_typeof(ltrim(arrow_cast(' foo', 'Dictionary(Int32, Utf8)')))-----foo+foo Dictionary(Int32, Utf8)query TSELECT md5('foo')@@ -710,20 +712,75 @@ SELECT rtrim(' foo ')----foo-query T-SELECT rtrim(arrow_cast(' foo ', 'Dictionary(Int32, Utf8)'))+query TT+SELECT rtrim(arrow_cast(' foo ', 'Dictionary(Int32, Utf8)')),+ arrow_typeof(rtrim(arrow_cast(' foo ', 'Dictionary(Int32, Utf8)')))----- foo+ foo Dictionary(Int32, Utf8)query TSELECT trim(' foo ')----foo-query T-SELECT trim(arrow_cast(' foo ', 'Dictionary(Int32, Utf8)'))+query TT+SELECT trim(arrow_cast(' foo ', 'Dictionary(Int32, Utf8)')),+ arrow_typeof(trim(arrow_cast(' foo ', 'Dictionary(Int32, Utf8)')))-----foo+foo Dictionary(Int32, Utf8)++query TTTT?T+SELECT btrim(arrow_cast(' foo ', 'Dictionary(Int32, LargeUtf8)')),+ arrow_typeof(btrim(arrow_cast(' foo ', 'Dictionary(Int32, LargeUtf8)'))),+ ltrim(arrow_cast(+ arrow_cast(' bar', 'Dictionary(Int32, Utf8)'),+ 'Dictionary(Int32, Utf8View)'+ )),+ arrow_typeof(ltrim(arrow_cast(+ arrow_cast(' bar', 'Dictionary(Int32, Utf8)'),+ 'Dictionary(Int32, Utf8View)'+ ))),+ rtrim(arrow_cast(+ arrow_cast('baz ', 'Dictionary(UInt32, Utf8)'),+ 'Dictionary(Int32, Dictionary(UInt32, Utf8))'+ )),+ arrow_typeof(rtrim(arrow_cast(+ arrow_cast('baz ', 'Dictionary(UInt32, Utf8)'),+ 'Dictionary(Int32, Dictionary(UInt32, Utf8))'+ )))+----+foo Dictionary(Int32, LargeUtf8) bar Dictionary(Int32, Utf8View) baz Dictionary(Int32, Dictionary(UInt32, Utf8))++statement ok+CREATE TABLE trim_dictionary_test AS+SELECT arrow_cast(column1, 'Dictionary(Int32, Utf8)') AS both,+ arrow_cast(column2, 'Dictionary(Int32, Utf8)') AS leading,+ arrow_cast(column3, 'Dictionary(Int32, Utf8)') AS trailing+FROM (VALUES+(' foo ', ' bar', 'baz '),+(NULL, NULL, NULL));++query TTTTTT+SELECT btrim(both), arrow_typeof(btrim(both)),+ ltrim(leading), arrow_typeof(ltrim(leading)),+ rtrim(trailing), arrow_typeof(rtrim(trailing))+FROM trim_dictionary_test+----+foo Dictionary(Int32, Utf8) bar Dictionary(Int32, Utf8) baz Dictionary(Int32, Utf8)+NULL Dictionary(Int32, Utf8) NULL Dictionary(Int32, Utf8) NULL Dictionary(Int32, Utf8)++statement ok+DROP TABLE trim_dictionary_test++query TTTTTT+SELECT btrim(arrow_cast('__foo__', 'Dictionary(Int32, Utf8)'), '_'),+ arrow_typeof(btrim(arrow_cast('__foo__', 'Dictionary(Int32, Utf8)'), '_')),+ ltrim(arrow_cast('__bar', 'Dictionary(Int32, Utf8)'), '_'),+ arrow_typeof(ltrim(arrow_cast('__bar', 'Dictionary(Int32, Utf8)'), '_')),+ rtrim(arrow_cast('baz__', 'Dictionary(Int32, Utf8)'), '_'),+ arrow_typeof(rtrim(arrow_cast('baz__', 'Dictionary(Int32, Utf8)'), '_'))+----+foo Utf8 bar Utf8 baz Utf8# Verify that trim, ltrim, and rtrim only strip spaces by default,# not other whitespace characters (tabs, newlines, etc.)