Skip to content

Apache DataFusion SQL Query Engine

perf: preserve dictionary encoding for `btrim`, `ltrim`, and `rtrim` (#24100)

linfeng6b6cc0939ef5
diff --git a/datafusion/functions/benches/dictionary_encoding.rs b/datafusion/functions/benches/dictionary_encoding.rs
index 05541fc10e..4ba04a4940 100644
--- a/datafusion/functions/benches/dictionary_encoding.rs
+++ b/datafusion/functions/benches/dictionary_encoding.rs
@@ -24,7 +24,7 @@ use arrow::datatypes::{Field, Int32Type};
use criterion::{Criterion, criterion_group, criterion_main};
use datafusion_common::config::ConfigOptions;
use datafusion_expr::type_coercion::functions::fields_with_udf;
-use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDF};
+use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
const NUM_ROWS: usize = 8_192;
const DICTIONARY_CARDINALITIES: [usize; 4] = [10, 100, 1_000, 8_192];
@@ -42,16 +42,19 @@ fn create_string_dictionary(cardinality: usize) -> ArrayRef {
}
fn benchmark_dictionary_string_udfs(c: &mut Criterion) {
- let udfs: [(&str, Arc<ScalarUDF>); 6] = [
+ let udfs = [
("ascii", datafusion_functions::string::ascii()),
("bit_length", datafusion_functions::string::bit_length()),
+ ("btrim", datafusion_functions::string::btrim()),
(
"character_length",
datafusion_functions::unicode::character_length(),
),
("initcap", datafusion_functions::unicode::initcap()),
+ ("ltrim", datafusion_functions::string::ltrim()),
("octet_length", datafusion_functions::string::octet_length()),
("reverse", datafusion_functions::unicode::reverse()),
+ ("rtrim", datafusion_functions::string::rtrim()),
];
let config_options = Arc::new(ConfigOptions::default());
diff --git a/datafusion/functions/src/string/btrim.rs b/datafusion/functions/src/string/btrim.rs
index 279f444d9f..82e1f7d2c7 100644
--- a/datafusion/functions/src/string/btrim.rs
+++ b/datafusion/functions/src/string/btrim.rs
@@ -16,30 +16,41 @@
// under the License.
use crate::string::common::*;
-use crate::utils::{make_scalar_function, utf8_to_str_type};
-use arrow::array::{ArrayRef, OffsetSizeTrait};
+use crate::utils::make_scalar_function;
+use arrow::array::{ArrayRef, AsArray};
use arrow::datatypes::DataType;
use datafusion_common::types::logical_string;
use datafusion_common::{Result, exec_err};
use datafusion_expr::function::Hint;
use datafusion_expr::{
- Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
- TypeSignature, TypeSignatureClass, Volatility,
+ Coercion, ColumnarValue, Documentation, EncodingPreservation, ScalarFunctionArgs,
+ ScalarUDFImpl, Signature, TypeSignature, TypeSignatureClass, Volatility,
};
use datafusion_macros::user_doc;
use std::sync::Arc;
/// Returns the longest string with leading and trailing characters removed. If the characters are not specified, spaces are removed.
/// btrim('xyxtrimyyx', 'xyz') = 'trim'
-fn btrim<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
- let use_string_view = args[0].data_type() == &DataType::Utf8View;
+fn btrim(args: &[ArrayRef]) -> Result<ArrayRef> {
let args = if args.len() > 1 {
let arg1 = arrow::compute::kernels::cast::cast(&args[1], args[0].data_type())?;
vec![Arc::clone(&args[0]), arg1]
} else {
args.to_owned()
};
- general_trim::<T, TrimBoth>(&args, use_string_view)
+ match args[0].data_type() {
+ DataType::Utf8 => general_trim::<i32, TrimBoth>(&args, false),
+ DataType::LargeUtf8 => general_trim::<i64, TrimBoth>(&args, false),
+ DataType::Utf8View => general_trim::<i32, TrimBoth>(&args, true),
+ DataType::Dictionary(_, _) => {
+ let dictionary = args[0].as_any_dictionary();
+ let trimmed = btrim(&[Arc::clone(dictionary.values())])?;
+ Ok(dictionary.with_values(trimmed))
+ }
+ other => exec_err!(
+ "Unsupported data type {other:?} for function btrim, expected Utf8, LargeUtf8 or Utf8View."
+ ),
+ }
}
#[user_doc(
@@ -85,9 +96,12 @@ impl BTrimFunc {
Coercion::new_exact(TypeSignatureClass::Native(logical_string())),
Coercion::new_exact(TypeSignatureClass::Native(logical_string())),
]),
- TypeSignature::Coercible(vec![Coercion::new_exact(
- TypeSignatureClass::Native(logical_string()),
- )]),
+ TypeSignature::Coercible(vec![
+ Coercion::new_exact(TypeSignatureClass::Native(logical_string()))
+ .with_encoding_preservation(
+ EncodingPreservation::dictionary(),
+ ),
+ ]),
],
Volatility::Immutable,
),
@@ -106,28 +120,11 @@ impl ScalarUDFImpl for BTrimFunc {
}
fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
- if arg_types[0] == DataType::Utf8View {
- Ok(DataType::Utf8View)
- } else {
- utf8_to_str_type(&arg_types[0], "btrim")
- }
+ Ok(arg_types[0].clone())
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
- match args.args[0].data_type() {
- DataType::Utf8 | DataType::Utf8View => make_scalar_function(
- btrim::<i32>,
- vec![Hint::Pad, Hint::AcceptsSingular],
- )(&args.args),
- DataType::LargeUtf8 => make_scalar_function(
- btrim::<i64>,
- vec![Hint::Pad, Hint::AcceptsSingular],
- )(&args.args),
- other => exec_err!(
- "Unsupported data type {other:?} for function btrim,\
- expected Utf8, LargeUtf8 or Utf8View."
- ),
- }
+ make_scalar_function(btrim, vec![Hint::Pad, Hint::AcceptsSingular])(&args.args)
}
fn aliases(&self) -> &[String] {
diff --git a/datafusion/functions/src/string/ltrim.rs b/datafusion/functions/src/string/ltrim.rs
index e49ffeb054..04e33253ed 100644
--- a/datafusion/functions/src/string/ltrim.rs
+++ b/datafusion/functions/src/string/ltrim.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use arrow::array::{ArrayRef, OffsetSizeTrait};
+use arrow::array::{ArrayRef, AsArray};
use arrow::datatypes::DataType;
use std::sync::Arc;
@@ -25,22 +25,33 @@ use datafusion_common::types::logical_string;
use datafusion_common::{Result, exec_err};
use datafusion_expr::function::Hint;
use datafusion_expr::{
- Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
- TypeSignature, TypeSignatureClass, Volatility,
+ Coercion, ColumnarValue, Documentation, EncodingPreservation, ScalarFunctionArgs,
+ ScalarUDFImpl, Signature, TypeSignature, TypeSignatureClass, Volatility,
};
use datafusion_macros::user_doc;
/// Returns the longest string with leading characters removed. If the characters are not specified, spaces are removed.
/// ltrim('zzzytest', 'xyz') = 'test'
-fn ltrim<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
- let use_string_view = args[0].data_type() == &DataType::Utf8View;
+fn ltrim(args: &[ArrayRef]) -> Result<ArrayRef> {
let args = if args.len() > 1 {
let arg1 = arrow::compute::kernels::cast::cast(&args[1], args[0].data_type())?;
vec![Arc::clone(&args[0]), arg1]
} else {
args.to_owned()
};
- general_trim::<T, TrimLeft>(&args, use_string_view)
+ match args[0].data_type() {
+ DataType::Utf8 => general_trim::<i32, TrimLeft>(&args, false),
+ DataType::LargeUtf8 => general_trim::<i64, TrimLeft>(&args, false),
+ DataType::Utf8View => general_trim::<i32, TrimLeft>(&args, true),
+ DataType::Dictionary(_, _) => {
+ let dictionary = args[0].as_any_dictionary();
+ let trimmed = ltrim(&[Arc::clone(dictionary.values())])?;
+ Ok(dictionary.with_values(trimmed))
+ }
+ other => exec_err!(
+ "Unsupported data type {other:?} for function ltrim, expected Utf8, LargeUtf8 or Utf8View."
+ ),
+ }
}
#[user_doc(
@@ -90,9 +101,12 @@ impl LtrimFunc {
Coercion::new_exact(TypeSignatureClass::Native(logical_string())),
Coercion::new_exact(TypeSignatureClass::Native(logical_string())),
]),
- TypeSignature::Coercible(vec![Coercion::new_exact(
- TypeSignatureClass::Native(logical_string()),
- )]),
+ TypeSignature::Coercible(vec![
+ Coercion::new_exact(TypeSignatureClass::Native(logical_string()))
+ .with_encoding_preservation(
+ EncodingPreservation::dictionary(),
+ ),
+ ]),
],
Volatility::Immutable,
),
@@ -114,20 +128,7 @@ impl ScalarUDFImpl for LtrimFunc {
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
- match args.args[0].data_type() {
- DataType::Utf8 | DataType::Utf8View => make_scalar_function(
- ltrim::<i32>,
- vec![Hint::Pad, Hint::AcceptsSingular],
- )(&args.args),
- DataType::LargeUtf8 => make_scalar_function(
- ltrim::<i64>,
- vec![Hint::Pad, Hint::AcceptsSingular],
- )(&args.args),
- other => exec_err!(
- "Unsupported data type {other:?} for function ltrim,\
- expected Utf8, LargeUtf8 or Utf8View."
- ),
- }
+ make_scalar_function(ltrim, vec![Hint::Pad, Hint::AcceptsSingular])(&args.args)
}
fn documentation(&self) -> Option<&Documentation> {
diff --git a/datafusion/functions/src/string/rtrim.rs b/datafusion/functions/src/string/rtrim.rs
index 05ad9e8559..d1126cb241 100644
--- a/datafusion/functions/src/string/rtrim.rs
+++ b/datafusion/functions/src/string/rtrim.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use arrow::array::{ArrayRef, OffsetSizeTrait};
+use arrow::array::{ArrayRef, AsArray};
use arrow::datatypes::DataType;
use std::sync::Arc;
@@ -25,22 +25,33 @@ use datafusion_common::types::logical_string;
use datafusion_common::{Result, exec_err};
use datafusion_expr::function::Hint;
use datafusion_expr::{
- Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
- TypeSignature, TypeSignatureClass, Volatility,
+ Coercion, ColumnarValue, Documentation, EncodingPreservation, ScalarFunctionArgs,
+ ScalarUDFImpl, Signature, TypeSignature, TypeSignatureClass, Volatility,
};
use datafusion_macros::user_doc;
/// Returns the longest string with trailing characters removed. If the characters are not specified, spaces are removed.
/// rtrim('testxxzx', 'xyz') = 'test'
-fn rtrim<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
- let use_string_view = args[0].data_type() == &DataType::Utf8View;
+fn rtrim(args: &[ArrayRef]) -> Result<ArrayRef> {
let args = if args.len() > 1 {
let arg1 = arrow::compute::kernels::cast::cast(&args[1], args[0].data_type())?;
vec![Arc::clone(&args[0]), arg1]
} else {
args.to_owned()
};
- general_trim::<T, TrimRight>(&args, use_string_view)
+ match args[0].data_type() {
+ DataType::Utf8 => general_trim::<i32, TrimRight>(&args, false),
+ DataType::LargeUtf8 => general_trim::<i64, TrimRight>(&args, false),
+ DataType::Utf8View => general_trim::<i32, TrimRight>(&args, true),
+ DataType::Dictionary(_, _) => {
+ let dictionary = args[0].as_any_dictionary();
+ let trimmed = rtrim(&[Arc::clone(dictionary.values())])?;
+ Ok(dictionary.with_values(trimmed))
+ }
+ other => exec_err!(
+ "Unsupported data type {other:?} for function rtrim, expected Utf8, LargeUtf8 or Utf8View."
+ ),
+ }
}
#[user_doc(
@@ -90,9 +101,12 @@ impl RtrimFunc {
Coercion::new_exact(TypeSignatureClass::Native(logical_string())),
Coercion::new_exact(TypeSignatureClass::Native(logical_string())),
]),
- TypeSignature::Coercible(vec![Coercion::new_exact(
- TypeSignatureClass::Native(logical_string()),
- )]),
+ TypeSignature::Coercible(vec![
+ Coercion::new_exact(TypeSignatureClass::Native(logical_string()))
+ .with_encoding_preservation(
+ EncodingPreservation::dictionary(),
+ ),
+ ]),
],
Volatility::Immutable,
),
@@ -114,20 +128,7 @@ impl ScalarUDFImpl for RtrimFunc {
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
- match args.args[0].data_type() {
- DataType::Utf8 | DataType::Utf8View => make_scalar_function(
- rtrim::<i32>,
- vec![Hint::Pad, Hint::AcceptsSingular],
- )(&args.args),
- DataType::LargeUtf8 => make_scalar_function(
- rtrim::<i64>,
- vec![Hint::Pad, Hint::AcceptsSingular],
- )(&args.args),
- other => exec_err!(
- "Unsupported data type {other:?} for function rtrim,\
- expected Utf8, LargeUtf8 or Utf8View."
- ),
- }
+ make_scalar_function(rtrim, vec![Hint::Pad, Hint::AcceptsSingular])(&args.args)
}
fn documentation(&self) -> Option<&Documentation> {
diff --git a/datafusion/sqllogictest/test_files/functions.slt b/datafusion/sqllogictest/test_files/functions.slt
index 78bdeb3e15..008be05852 100644
--- a/datafusion/sqllogictest/test_files/functions.slt
+++ b/datafusion/sqllogictest/test_files/functions.slt
@@ -492,10 +492,11 @@ SELECT btrim(' foo ')
----
foo
-query T
-SELECT btrim(arrow_cast(' foo ', 'Dictionary(Int32, Utf8)'))
+query TT
+SELECT btrim(arrow_cast(' foo ', 'Dictionary(Int32, Utf8)')),
+ arrow_typeof(btrim(arrow_cast(' foo ', 'Dictionary(Int32, Utf8)')))
----
-foo
+foo Dictionary(Int32, Utf8)
query T
SELECT initcap('foo')
@@ -689,10 +690,11 @@ SELECT ltrim(' foo')
----
foo
-query T
-SELECT ltrim(arrow_cast(' foo', 'Dictionary(Int32, Utf8)'))
+query TT
+SELECT ltrim(arrow_cast(' foo', 'Dictionary(Int32, Utf8)')),
+ arrow_typeof(ltrim(arrow_cast(' foo', 'Dictionary(Int32, Utf8)')))
----
-foo
+foo Dictionary(Int32, Utf8)
query T
SELECT md5('foo')
@@ -710,20 +712,75 @@ SELECT rtrim(' foo ')
----
foo
-query T
-SELECT rtrim(arrow_cast(' foo ', 'Dictionary(Int32, Utf8)'))
+query TT
+SELECT rtrim(arrow_cast(' foo ', 'Dictionary(Int32, Utf8)')),
+ arrow_typeof(rtrim(arrow_cast(' foo ', 'Dictionary(Int32, Utf8)')))
----
- foo
+ foo Dictionary(Int32, Utf8)
query T
SELECT trim(' foo ')
----
foo
-query T
-SELECT trim(arrow_cast(' foo ', 'Dictionary(Int32, Utf8)'))
+query TT
+SELECT trim(arrow_cast(' foo ', 'Dictionary(Int32, Utf8)')),
+ arrow_typeof(trim(arrow_cast(' foo ', 'Dictionary(Int32, Utf8)')))
----
-foo
+foo Dictionary(Int32, Utf8)
+
+query TTTT?T
+SELECT btrim(arrow_cast(' foo ', 'Dictionary(Int32, LargeUtf8)')),
+ arrow_typeof(btrim(arrow_cast(' foo ', 'Dictionary(Int32, LargeUtf8)'))),
+ ltrim(arrow_cast(
+ arrow_cast(' bar', 'Dictionary(Int32, Utf8)'),
+ 'Dictionary(Int32, Utf8View)'
+ )),
+ arrow_typeof(ltrim(arrow_cast(
+ arrow_cast(' bar', 'Dictionary(Int32, Utf8)'),
+ 'Dictionary(Int32, Utf8View)'
+ ))),
+ rtrim(arrow_cast(
+ arrow_cast('baz ', 'Dictionary(UInt32, Utf8)'),
+ 'Dictionary(Int32, Dictionary(UInt32, Utf8))'
+ )),
+ arrow_typeof(rtrim(arrow_cast(
+ arrow_cast('baz ', 'Dictionary(UInt32, Utf8)'),
+ 'Dictionary(Int32, Dictionary(UInt32, Utf8))'
+ )))
+----
+foo Dictionary(Int32, LargeUtf8) bar Dictionary(Int32, Utf8View) baz Dictionary(Int32, Dictionary(UInt32, Utf8))
+
+statement ok
+CREATE TABLE trim_dictionary_test AS
+SELECT arrow_cast(column1, 'Dictionary(Int32, Utf8)') AS both,
+ arrow_cast(column2, 'Dictionary(Int32, Utf8)') AS leading,
+ arrow_cast(column3, 'Dictionary(Int32, Utf8)') AS trailing
+FROM (VALUES
+(' foo ', ' bar', 'baz '),
+(NULL, NULL, NULL));
+
+query TTTTTT
+SELECT btrim(both), arrow_typeof(btrim(both)),
+ ltrim(leading), arrow_typeof(ltrim(leading)),
+ rtrim(trailing), arrow_typeof(rtrim(trailing))
+FROM trim_dictionary_test
+----
+foo Dictionary(Int32, Utf8) bar Dictionary(Int32, Utf8) baz Dictionary(Int32, Utf8)
+NULL Dictionary(Int32, Utf8) NULL Dictionary(Int32, Utf8) NULL Dictionary(Int32, Utf8)
+
+statement ok
+DROP TABLE trim_dictionary_test
+
+query TTTTTT
+SELECT btrim(arrow_cast('__foo__', 'Dictionary(Int32, Utf8)'), '_'),
+ arrow_typeof(btrim(arrow_cast('__foo__', 'Dictionary(Int32, Utf8)'), '_')),
+ ltrim(arrow_cast('__bar', 'Dictionary(Int32, Utf8)'), '_'),
+ arrow_typeof(ltrim(arrow_cast('__bar', 'Dictionary(Int32, Utf8)'), '_')),
+ rtrim(arrow_cast('baz__', 'Dictionary(Int32, Utf8)'), '_'),
+ arrow_typeof(rtrim(arrow_cast('baz__', 'Dictionary(Int32, Utf8)'), '_'))
+----
+foo Utf8 bar Utf8 baz Utf8
# Verify that trim, ltrim, and rtrim only strip spaces by default,
# not other whitespace characters (tabs, newlines, etc.)