Skip to content

Apache DataFusion SQL Query Engine

perf: Optimize scalar path for chr function (#20073)

Kumar Ujjawalf779e966809f
diff --git a/datafusion/functions/benches/chr.rs b/datafusion/functions/benches/chr.rs
index 9a6342ca40..fcb72771ac 100644
--- a/datafusion/functions/benches/chr.rs
+++ b/datafusion/functions/benches/chr.rs
@@ -19,6 +19,7 @@ extern crate criterion;
use arrow::{array::PrimitiveArray, datatypes::Int64Type};
use criterion::{Criterion, criterion_group, criterion_main};
+use datafusion_common::ScalarValue;
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
use datafusion_functions::string::chr;
use rand::{Rng, SeedableRng};
@@ -35,11 +36,32 @@ pub fn seedable_rng() -> StdRng {
}
fn criterion_benchmark(c: &mut Criterion) {
- let cot_fn = chr();
+ let chr_fn = chr();
+ let config_options = Arc::new(ConfigOptions::default());
+
+ // Scalar benchmarks
+ c.bench_function("chr/scalar", |b| {
+ let args = vec![ColumnarValue::Scalar(ScalarValue::Int64(Some(65)))];
+ let arg_fields = vec![Field::new("arg_0", DataType::Int64, true).into()];
+ b.iter(|| {
+ black_box(
+ chr_fn
+ .invoke_with_args(ScalarFunctionArgs {
+ args: args.clone(),
+ arg_fields: arg_fields.clone(),
+ number_rows: 1,
+ return_field: Field::new("f", DataType::Utf8, true).into(),
+ config_options: Arc::clone(&config_options),
+ })
+ .unwrap(),
+ )
+ })
+ });
+
let size = 1024;
let input: PrimitiveArray<Int64Type> = {
let null_density = 0.2;
- let mut rng = StdRng::seed_from_u64(42);
+ let mut rng = seedable_rng();
(0..size)
.map(|_| {
if rng.random::<f32>() < null_density {
@@ -57,12 +79,11 @@ fn criterion_benchmark(c: &mut Criterion) {
.enumerate()
.map(|(idx, arg)| Field::new(format!("arg_{idx}"), arg.data_type(), true).into())
.collect::<Vec<_>>();
- let config_options = Arc::new(ConfigOptions::default());
- c.bench_function("chr", |b| {
+ c.bench_function("chr/array", |b| {
b.iter(|| {
black_box(
- cot_fn
+ chr_fn
.invoke_with_args(ScalarFunctionArgs {
args: args.clone(),
arg_fields: arg_fields.clone(),
diff --git a/datafusion/functions/src/string/chr.rs b/datafusion/functions/src/string/chr.rs
index ba011b9436..2f432c838e 100644
--- a/datafusion/functions/src/string/chr.rs
+++ b/datafusion/functions/src/string/chr.rs
@@ -18,24 +18,21 @@
use std::any::Any;
use std::sync::Arc;
-use arrow::array::ArrayRef;
-use arrow::array::GenericStringBuilder;
+use arrow::array::{ArrayRef, GenericStringBuilder, Int64Array};
use arrow::datatypes::DataType;
use arrow::datatypes::DataType::Int64;
use arrow::datatypes::DataType::Utf8;
-use crate::utils::make_scalar_function;
use datafusion_common::cast::as_int64_array;
-use datafusion_common::{Result, exec_err};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, internal_err};
use datafusion_expr::{ColumnarValue, Documentation, Volatility};
use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl, Signature};
use datafusion_macros::user_doc;
/// Returns the character with the given code.
/// chr(65) = 'A'
-fn chr(args: &[ArrayRef]) -> Result<ArrayRef> {
- let integer_array = as_int64_array(&args[0])?;
-
+fn chr_array(integer_array: &Int64Array) -> Result<ArrayRef> {
let mut builder = GenericStringBuilder::<i32>::with_capacity(
integer_array.len(),
// 1 byte per character, assuming that is the common case
@@ -56,15 +53,11 @@ fn chr(args: &[ArrayRef]) -> Result<ArrayRef> {
return exec_err!("invalid Unicode scalar value: {integer}");
}
- None => {
- builder.append_null();
- }
+ None => builder.append_null(),
}
}
- let result = builder.finish();
-
- Ok(Arc::new(result) as ArrayRef)
+ Ok(Arc::new(builder.finish()) as ArrayRef)
}
#[user_doc(
@@ -119,7 +112,32 @@ impl ScalarUDFImpl for ChrFunc {
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
- make_scalar_function(chr, vec![])(&args.args)
+ let [arg] = take_function_args(self.name(), args.args)?;
+
+ match arg {
+ ColumnarValue::Scalar(ScalarValue::Int64(Some(code_point))) => {
+ if let Ok(u) = u32::try_from(code_point)
+ && let Some(c) = core::char::from_u32(u)
+ {
+ Ok(ColumnarValue::Scalar(ScalarValue::Utf8(Some(
+ c.to_string(),
+ ))))
+ } else {
+ exec_err!("invalid Unicode scalar value: {code_point}")
+ }
+ }
+ ColumnarValue::Scalar(ScalarValue::Int64(None)) => {
+ Ok(ColumnarValue::Scalar(ScalarValue::Utf8(None)))
+ }
+ ColumnarValue::Array(array) => {
+ let integer_array = as_int64_array(&array)?;
+ Ok(ColumnarValue::Array(chr_array(integer_array)?))
+ }
+ other => internal_err!(
+ "Unexpected data type {:?} for function chr",
+ other.data_type()
+ ),
+ }
}
fn documentation(&self) -> Option<&Documentation> {
@@ -130,13 +148,27 @@ impl ScalarUDFImpl for ChrFunc {
#[cfg(test)]
mod tests {
use super::*;
+
use arrow::array::{Array, Int64Array, StringArray};
+ use arrow::datatypes::Field;
use datafusion_common::assert_contains;
+ use datafusion_common::config::ConfigOptions;
+ use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};
+
+ fn invoke_chr(arg: ColumnarValue, number_rows: usize) -> Result<ColumnarValue> {
+ ChrFunc::new().invoke_with_args(ScalarFunctionArgs {
+ args: vec![arg],
+ arg_fields: vec![Field::new("a", Int64, true).into()],
+ number_rows,
+ return_field: Field::new("f", Utf8, true).into(),
+ config_options: Arc::new(ConfigOptions::default()),
+ })
+ }
#[test]
fn test_chr_normal() {
let input = Arc::new(Int64Array::from(vec![
- Some(0), // null
+ Some(0), // \u{0000}
Some(65), // A
Some(66), // B
Some(67), // C
@@ -149,8 +181,13 @@ mod tests {
Some(9), // tab
Some(0x10FFFF), // 0x10FFFF, the largest Unicode code point
]));
- let result = chr(&[input]).unwrap();
- let string_array = result.as_any().downcast_ref::<StringArray>().unwrap();
+
+ let result = invoke_chr(ColumnarValue::Array(input), 12).unwrap();
+ let ColumnarValue::Array(arr) = result else {
+ panic!("Expected array");
+ };
+ let string_array = arr.as_any().downcast_ref::<StringArray>().unwrap();
+
let expected = [
"\u{0000}",
"A",
@@ -174,55 +211,48 @@ mod tests {
#[test]
fn test_chr_error() {
- // invalid Unicode code points (too large)
let input = Arc::new(Int64Array::from(vec![i64::MAX]));
- let result = chr(&[input]);
+ let result = invoke_chr(ColumnarValue::Array(input), 1);
assert!(result.is_err());
assert_contains!(
result.err().unwrap().to_string(),
"invalid Unicode scalar value: 9223372036854775807"
);
- // invalid Unicode code points (too large) case 2
let input = Arc::new(Int64Array::from(vec![0x10FFFF + 1]));
- let result = chr(&[input]);
+ let result = invoke_chr(ColumnarValue::Array(input), 1);
assert!(result.is_err());
assert_contains!(
result.err().unwrap().to_string(),
"invalid Unicode scalar value: 1114112"
);
- // invalid Unicode code points (surrogate code point)
- // link: <https://learn.microsoft.com/en-us/globalization/encoding/unicode-standard#surrogate-pairs>
let input = Arc::new(Int64Array::from(vec![0xD800 + 1]));
- let result = chr(&[input]);
+ let result = invoke_chr(ColumnarValue::Array(input), 1);
assert!(result.is_err());
assert_contains!(
result.err().unwrap().to_string(),
"invalid Unicode scalar value: 55297"
);
- // negative input
- let input = Arc::new(Int64Array::from(vec![i64::MIN + 2i64])); // will be 2 if cast to u32
- let result = chr(&[input]);
+ let input = Arc::new(Int64Array::from(vec![i64::MIN + 2i64]));
+ let result = invoke_chr(ColumnarValue::Array(input), 1);
assert!(result.is_err());
assert_contains!(
result.err().unwrap().to_string(),
"invalid Unicode scalar value: -9223372036854775806"
);
- // negative input case 2
let input = Arc::new(Int64Array::from(vec![-1]));
- let result = chr(&[input]);
+ let result = invoke_chr(ColumnarValue::Array(input), 1);
assert!(result.is_err());
assert_contains!(
result.err().unwrap().to_string(),
"invalid Unicode scalar value: -1"
);
- // one error with valid values after
- let input = Arc::new(Int64Array::from(vec![65, -1, 66])); // A, -1, B
- let result = chr(&[input]);
+ let input = Arc::new(Int64Array::from(vec![65, -1, 66]));
+ let result = invoke_chr(ColumnarValue::Array(input), 3);
assert!(result.is_err());
assert_contains!(
result.err().unwrap().to_string(),
@@ -232,10 +262,36 @@ mod tests {
#[test]
fn test_chr_empty() {
- // empty input array
let input = Arc::new(Int64Array::from(Vec::<i64>::new()));
- let result = chr(&[input]).unwrap();
- let string_array = result.as_any().downcast_ref::<StringArray>().unwrap();
+ let result = invoke_chr(ColumnarValue::Array(input), 0).unwrap();
+ let ColumnarValue::Array(arr) = result else {
+ panic!("Expected array");
+ };
+ let string_array = arr.as_any().downcast_ref::<StringArray>().unwrap();
assert_eq!(string_array.len(), 0);
}
+
+ #[test]
+ fn test_chr_scalar() {
+ let result =
+ invoke_chr(ColumnarValue::Scalar(ScalarValue::Int64(Some(65))), 1).unwrap();
+
+ match result {
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some(s))) => {
+ assert_eq!(s, "A");
+ }
+ other => panic!("Unexpected result: {other:?}"),
+ }
+ }
+
+ #[test]
+ fn test_chr_scalar_null() {
+ let result =
+ invoke_chr(ColumnarValue::Scalar(ScalarValue::Int64(None)), 1).unwrap();
+
+ match result {
+ ColumnarValue::Scalar(ScalarValue::Utf8(None)) => {}
+ other => panic!("Unexpected result: {other:?}"),
+ }
+ }
}
diff --git a/datafusion/sqllogictest/test_files/expr.slt b/datafusion/sqllogictest/test_files/expr.slt
index 00b5816327..90fe05815f 100644
--- a/datafusion/sqllogictest/test_files/expr.slt
+++ b/datafusion/sqllogictest/test_files/expr.slt
@@ -432,6 +432,16 @@ SELECT chr(CAST(0 AS int))
statement error DataFusion error: Execution error: invalid Unicode scalar value: 9223372036854775807
SELECT chr(CAST(9223372036854775807 AS bigint))
+statement error DataFusion error: Execution error: invalid Unicode scalar value: 1114112
+SELECT chr(CAST(1114112 AS bigint))
+
+statement error DataFusion error: Execution error: invalid Unicode scalar value: -1
+SELECT chr(CAST(-1 AS bigint))
+
+# surrogate code point (invalid scalar value)
+statement error DataFusion error: Execution error: invalid Unicode scalar value: 55297
+SELECT chr(CAST(55297 AS bigint))
+
query T
SELECT concat('a','b','c')
----