Skip to content

Apache DataFusion SQL Query Engine

perf: Optimize `upper`, `lower`, `translate`, `replace` (#26377)

Neil Conwaycd90b6dd9f85
diff --git a/datafusion/functions/benches/misc/main.rs b/datafusion/functions/benches/misc/main.rs
index 57a206790e..0970e9e419 100644
--- a/datafusion/functions/benches/misc/main.rs
+++ b/datafusion/functions/benches/misc/main.rs
@@ -29,11 +29,5 @@ mod get_field;
mod helper;
mod nullif;
mod overlay;
-mod upper;
-criterion_main!(
- get_field::benches,
- nullif::benches,
- overlay::benches,
- upper::benches,
-);
+criterion_main!(get_field::benches, nullif::benches, overlay::benches);
diff --git a/datafusion/functions/benches/misc/upper.rs b/datafusion/functions/benches/misc/upper.rs
deleted file mode 100644
index 1941547155..0000000000
--- a/datafusion/functions/benches/misc/upper.rs
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use arrow::datatypes::{DataType, Field};
-use arrow::util::bench_util::create_string_array_with_len;
-use criterion::{Criterion, criterion_group};
-use datafusion_common::config::ConfigOptions;
-use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
-use datafusion_functions::string;
-use std::hint::black_box;
-use std::sync::Arc;
-
-/// Create an array of args containing a StringArray, where all the values in the
-/// StringArray are ASCII.
-/// * `size` - the length of the StringArray, and
-/// * `str_len` - the length of the strings within the StringArray.
-fn create_args(size: usize, str_len: usize) -> Vec<ColumnarValue> {
- let array = Arc::new(create_string_array_with_len::<i32>(size, 0.2, str_len));
- vec![ColumnarValue::Array(array)]
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
- let upper = string::upper();
- let config_options = Arc::new(ConfigOptions::default());
-
- for size in [1024, 4096, 8192] {
- let args = create_args(size, 32);
- c.bench_function("upper_all_values_are_ascii", |b| {
- b.iter(|| {
- let args_cloned = args.clone();
- black_box(upper.invoke_with_args(ScalarFunctionArgs {
- args: args_cloned,
- arg_fields: vec![Field::new("a", DataType::Utf8, true).into()],
- number_rows: size,
- return_field: Field::new("f", DataType::Utf8, true).into(),
- config_options: Arc::clone(&config_options),
- }))
- })
- });
- }
-}
-
-criterion_group!(benches, criterion_benchmark);
diff --git a/datafusion/functions/benches/string_expressions/case_conversion_inputs.rs b/datafusion/functions/benches/string_expressions/case_conversion_inputs.rs
new file mode 100644
index 0000000000..72c6d70cca
--- /dev/null
+++ b/datafusion/functions/benches/string_expressions/case_conversion_inputs.rs
@@ -0,0 +1,213 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Inputs shared by the `upper` and `lower` benchmarks, so both functions are
+//! measured on the same arrays.
+
+use std::hint::black_box;
+use std::sync::Arc;
+
+use crate::helper::gen_string_array;
+use arrow::array::{Array, ArrayRef, LargeStringArray, StringArray, StringViewArray};
+use arrow::datatypes::Field;
+use arrow::util::bench_util::{
+ create_string_array_with_len, create_string_view_array_with_len,
+};
+use criterion::Criterion;
+use datafusion_common::config::ConfigOptions;
+use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDF};
+
+/// Rows in each array built by [`build_values`].
+const SIZE: usize = 4096;
+
+const WORDS: [&str; 8] = [
+ "café",
+ "straße",
+ "αλφα",
+ "こんにちは",
+ "münchen",
+ "naïve",
+ " órdenes",
+ "tschüß",
+];
+
+/// Builds [`SIZE`] values: every tenth is NULL and the rest come from `value`.
+fn build_values(value: impl Fn(usize) -> String) -> Vec<Option<String>> {
+ (0..SIZE).map(|i| (i % 10 != 0).then(|| value(i))).collect()
+}
+
+/// Three words from [`WORDS`], so the value is non-ASCII.
+fn word_value(row: usize) -> String {
+ let a = WORDS[row % WORDS.len()];
+ let b = WORDS[(row * 7 + 3) % WORDS.len()];
+ format!("{a} {b} {a}")
+}
+
+/// ASCII text of exactly `len` bytes that differs from row to row.
+fn ascii_value(row: usize, len: usize) -> String {
+ let base = format!("datafusion row {row} ");
+ base.repeat(len / base.len() + 1)[..len].to_string()
+}
+
+/// Returns the named input arrays.
+///
+/// Entirely ASCII, which take the all-ASCII array fast path:
+///
+/// - `all_ascii_utf8_rows*`: 32-byte values, 20% NULL.
+/// - `all_ascii_utf8_sliced`: 128 rows sliced from the middle of a larger
+/// array, so the input's buffers are much larger than the rows it uses.
+/// - `all_ascii_utf8view_*`: values of `len` bytes, or with `_mixed`, half
+/// under 12 bytes (stored inline in the view) and half of 12 to
+/// `max(30, len)` bytes.
+///
+/// Not entirely ASCII, so each row is converted on its own:
+///
+/// - `some_non_ascii_utf8view_*`: 30-character values, about 10% of them
+/// non-ASCII.
+/// - `non_ascii_words_*`: every value is three words from [`WORDS`].
+/// - `mostly_ascii_*`: ASCII values plus one non-ASCII value in the middle.
+/// - `late_non_ascii_*`: every value is ASCII text followed by one non-ASCII
+/// character.
+pub fn inputs() -> Vec<(String, ArrayRef)> {
+ let mut inputs: Vec<(String, ArrayRef)> = Vec::new();
+
+ for size in [1024, 4096, 8192] {
+ inputs.push((
+ format!("all_ascii_utf8_rows{size}"),
+ Arc::new(create_string_array_with_len::<i32>(size, 0.2, 32)),
+ ));
+ }
+
+ let parent_size = 65536;
+ let slice_len = 128;
+ let parent = create_string_array_with_len::<i32>(parent_size, 0.2, 32);
+ inputs.push((
+ "all_ascii_utf8_sliced".to_string(),
+ Arc::new(parent.slice((parent_size - slice_len) / 2, slice_len)),
+ ));
+
+ for size in [4096, 8192] {
+ for str_len in [10, 64, 128] {
+ for null_density in [0.0, 0.1] {
+ for mixed in [false, true] {
+ let suffix = if mixed { "_mixed" } else { "" };
+ inputs.push((
+ format!(
+ "all_ascii_utf8view_rows{size}_len{str_len}_nulls{null_density}{suffix}"
+ ),
+ Arc::new(create_string_view_array_with_len(
+ size,
+ null_density,
+ str_len,
+ mixed,
+ )),
+ ));
+ }
+ }
+ }
+ }
+
+ for size in [4096, 8192] {
+ for null_density in [0.0, 0.1] {
+ let array = gen_string_array(size, 30, null_density, 0.1, true)
+ .remove(0)
+ .into_array(size)
+ .unwrap();
+ inputs.push((
+ format!("some_non_ascii_utf8view_rows{size}_nulls{null_density}"),
+ array,
+ ));
+ }
+ }
+
+ let words = build_values(word_value);
+ inputs.push((
+ "non_ascii_words_utf8".to_string(),
+ Arc::new(words.iter().map(Option::as_deref).collect::<StringArray>()),
+ ));
+ inputs.push((
+ "non_ascii_words_large_utf8".to_string(),
+ Arc::new(
+ words
+ .iter()
+ .map(Option::as_deref)
+ .collect::<LargeStringArray>(),
+ ),
+ ));
+ inputs.push((
+ "non_ascii_words_utf8view".to_string(),
+ Arc::new(
+ words
+ .iter()
+ .map(Option::as_deref)
+ .collect::<StringViewArray>(),
+ ),
+ ));
+
+ for str_len in [8, 32, 128] {
+ let mostly_ascii = build_values(|i| {
+ if i == SIZE / 2 {
+ "café".to_string()
+ } else {
+ ascii_value(i, str_len)
+ }
+ });
+ let late_non_ascii = build_values(|i| ascii_value(i, str_len) + "é");
+ for (shape, values) in [
+ ("mostly_ascii", mostly_ascii),
+ ("late_non_ascii", late_non_ascii),
+ ] {
+ inputs.push((
+ format!("{shape}_utf8_len{str_len}"),
+ Arc::new(values.iter().map(Option::as_deref).collect::<StringArray>()),
+ ));
+ inputs.push((
+ format!("{shape}_utf8view_len{str_len}"),
+ Arc::new(
+ values
+ .iter()
+ .map(Option::as_deref)
+ .collect::<StringViewArray>(),
+ ),
+ ));
+ }
+ }
+
+ inputs
+}
+
+/// Benchmarks `func` on `array` under the name `name`.
+pub fn bench_array(c: &mut Criterion, func: &ScalarUDF, name: &str, array: &ArrayRef) {
+ let data_type = array.data_type().clone();
+ let arg_fields = vec![Field::new("a", data_type.clone(), true).into()];
+ let return_field = Field::new("f", data_type, true).into();
+ let config_options = Arc::new(ConfigOptions::default());
+ c.bench_function(name, |b| {
+ b.iter(|| {
+ black_box(
+ func.invoke_with_args(ScalarFunctionArgs {
+ args: vec![ColumnarValue::Array(Arc::clone(array))],
+ arg_fields: arg_fields.clone(),
+ number_rows: array.len(),
+ return_field: Arc::clone(&return_field),
+ config_options: Arc::clone(&config_options),
+ })
+ .unwrap(),
+ )
+ })
+ });
+}
diff --git a/datafusion/functions/benches/string_expressions/lower.rs b/datafusion/functions/benches/string_expressions/lower.rs
index 2f73106157..6e1a726452 100644
--- a/datafusion/functions/benches/string_expressions/lower.rs
+++ b/datafusion/functions/benches/string_expressions/lower.rs
@@ -15,288 +15,14 @@
// specific language governing permissions and limitations
// under the License.
-use arrow::array::{Array, ArrayRef, StringArray, StringViewBuilder};
-use arrow::datatypes::{DataType, Field};
-use arrow::util::bench_util::{
- create_string_array_with_len, create_string_view_array_with_len,
-};
+use crate::case_conversion_inputs;
use criterion::{Criterion, criterion_group};
-use datafusion_common::config::ConfigOptions;
-use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
use datafusion_functions::string;
-use std::hint::black_box;
-use std::sync::Arc;
-
-/// Create an array of args containing a StringArray, where all the values in the
-/// StringArray are ASCII.
-/// * `size` - the length of the StringArray, and
-/// * `str_len` - the length of the strings within the StringArray.
-fn create_args1(size: usize, str_len: usize) -> Vec<ColumnarValue> {
- let array = Arc::new(create_string_array_with_len::<i32>(size, 0.2, str_len));
- vec![ColumnarValue::Array(array)]
-}
-
-/// Create an array of args containing a StringArray, where the first value in the
-/// StringArray is non-ASCII.
-/// * `size` - the length of the StringArray, and
-/// * `str_len` - the length of the strings within the StringArray.
-fn create_args2(size: usize) -> Vec<ColumnarValue> {
- let mut items = Vec::with_capacity(size);
- items.push("农历新年".to_string());
- for i in 1..size {
- items.push(format!("DATAFUSION {i}"));
- }
- let array = Arc::new(StringArray::from(items)) as ArrayRef;
- vec![ColumnarValue::Array(array)]
-}
-
-/// Create an array of args containing a StringArray, where the middle value of the
-/// StringArray is non-ASCII.
-/// * `size` - the length of the StringArray, and
-/// * `str_len` - the length of the strings within the StringArray.
-fn create_args3(size: usize) -> Vec<ColumnarValue> {
- let mut items = Vec::with_capacity(size);
- let half = size / 2;
- for i in 0..half {
- items.push(format!("DATAFUSION {i}"));
- }
- items.push("Ⱦ".to_string());
- for i in half + 1..size {
- items.push(format!("DATAFUSION {i}"));
- }
- let array = Arc::new(StringArray::from(items)) as ArrayRef;
- vec![ColumnarValue::Array(array)]
-}
-
-/// Create an array of args containing StringViews, where all the values in the
-/// StringViews are ASCII.
-/// * `size` - the length of the StringViews, and
-/// * `str_len` - the length of the strings within the array.
-/// * `null_density` - the density of null values in the array.
-/// * `mixed` - whether the array is mixed between inlined and referenced strings.
-fn create_args4(
- size: usize,
- str_len: usize,
- null_density: f32,
- mixed: bool,
-) -> Vec<ColumnarValue> {
- let array = Arc::new(create_string_view_array_with_len(
- size,
- null_density,
- str_len,
- mixed,
- ));
-
- vec![ColumnarValue::Array(array)]
-}
-
-/// Create an array of args containing a StringViewArray, where some of the values in the
-/// array are non-ASCII.
-/// * `size` - the length of the StringArray, and
-/// * `non_ascii_density` - the density of non-ASCII values in the array.
-/// * `null_density` - the density of null values in the array.
-fn create_args5(
- size: usize,
- non_ascii_density: f32,
- null_density: f32,
-) -> Vec<ColumnarValue> {
- let mut string_view_builder = StringViewBuilder::with_capacity(size);
- for _ in 0..size {
- // sample null_density to determine if the value should be null
- if rand::random::<f32>() < null_density {
- string_view_builder.append_null();
- continue;
- }
-
- // sample non_ascii_density to determine if the value should be non-ASCII
- if rand::random::<f32>() < non_ascii_density {
- string_view_builder.append_value("农历新年农历新年农历新年农历新年农历新年");
- } else {
- string_view_builder.append_value("DATAFUSIONDATAFUSIONDATAFUSION");
- }
- }
-
- let array = Arc::new(string_view_builder.finish()) as ArrayRef;
- vec![ColumnarValue::Array(array)]
-}
fn criterion_benchmark(c: &mut Criterion) {
let lower = string::lower();
- let config_options = Arc::new(ConfigOptions::default());
-
- for size in [1024, 4096, 8192] {
- let args = create_args1(size, 32);
- let arg_fields = args
- .iter()
- .enumerate()
- .map(|(idx, arg)| {
- Field::new(format!("arg_{idx}"), arg.data_type(), true).into()
- })
- .collect::<Vec<_>>();
-
- c.bench_function(&format!("lower_all_values_are_ascii: {size}"), |b| {
- b.iter(|| {
- let args_cloned = args.clone();
- black_box(lower.invoke_with_args(ScalarFunctionArgs {
- args: args_cloned,
- arg_fields: arg_fields.clone(),
- number_rows: size,
- return_field: Field::new("f", DataType::Utf8, true).into(),
- config_options: Arc::clone(&config_options),
- }))
- })
- });
-
- let args = create_args2(size);
- let arg_fields = args
- .iter()
- .enumerate()
- .map(|(idx, arg)| {
- Field::new(format!("arg_{idx}"), arg.data_type(), true).into()
- })
- .collect::<Vec<_>>();
-
- c.bench_function(&format!("lower_the_first_value_is_nonascii: {size}"), |b| {
- b.iter(|| {
- let args_cloned = args.clone();
- black_box(lower.invoke_with_args(ScalarFunctionArgs {
- args: args_cloned,
- arg_fields: arg_fields.clone(),
- number_rows: size,
- return_field: Field::new("f", DataType::Utf8, true).into(),
- config_options: Arc::clone(&config_options),
- }))
- })
- });
-
- let args = create_args3(size);
- let arg_fields = args
- .iter()
- .enumerate()
- .map(|(idx, arg)| {
- Field::new(format!("arg_{idx}"), arg.data_type(), true).into()
- })
- .collect::<Vec<_>>();
-
- c.bench_function(
- &format!("lower_the_middle_value_is_nonascii: {size}"),
- |b| {
- b.iter(|| {
- let args_cloned = args.clone();
- black_box(lower.invoke_with_args(ScalarFunctionArgs {
- args: args_cloned,
- arg_fields: arg_fields.clone(),
- number_rows: size,
- return_field: Field::new("f", DataType::Utf8, true).into(),
- config_options: Arc::clone(&config_options),
- }))
- })
- },
- );
- }
-
- {
- let parent_size = 65536;
- let slice_len = 128;
- let str_len = 32;
- let parent = Arc::new(create_string_array_with_len::<i32>(
- parent_size,
- 0.2,
- str_len,
- )) as ArrayRef;
- let offset = (parent_size - slice_len) / 2;
- let sliced = parent.slice(offset, slice_len);
- let args = vec![ColumnarValue::Array(sliced)];
- let arg_fields = args
- .iter()
- .enumerate()
- .map(|(idx, arg)| {
- Field::new(format!("arg_{idx}"), arg.data_type(), true).into()
- })
- .collect::<Vec<_>>();
-
- c.bench_function(
- &format!("lower_sliced_ascii: parent={parent_size}, slice={slice_len}, str_len={str_len}"),
- |b| {
- b.iter(|| {
- let args_cloned = args.clone();
- black_box(lower.invoke_with_args(ScalarFunctionArgs {
- args: args_cloned,
- arg_fields: arg_fields.clone(),
- number_rows: slice_len,
- return_field: Field::new("f", DataType::Utf8, true).into(),
- config_options: Arc::clone(&config_options),
- }))
- })
- },
- );
- }
-
- let sizes = [4096, 8192];
- let str_lens = [10, 64, 128];
- let mixes = [true, false];
- let null_densities = [0.0f32, 0.1f32];
-
- for null_density in &null_densities {
- for &mixed in &mixes {
- for &str_len in &str_lens {
- for &size in &sizes {
- let args = create_args4(size, str_len, *null_density, mixed);
- let arg_fields = args
- .iter()
- .enumerate()
- .map(|(idx, arg)| {
- Field::new(format!("arg_{idx}"), arg.data_type(), true).into()
- })
- .collect::<Vec<_>>();
-
- c.bench_function(
- &format!("lower_all_values_are_ascii_string_views: size: {size}, str_len: {str_len}, null_density: {null_density}, mixed: {mixed}"),
- |b| b.iter(|| {
- let args_cloned = args.clone();
- black_box(lower.invoke_with_args(ScalarFunctionArgs{
- args: args_cloned,
- arg_fields: arg_fields.clone(),
- number_rows: size,
- return_field: Field::new("f", DataType::Utf8, true).into(),
- config_options: Arc::clone(&config_options),
- }))
- }),
- );
-
- let args = create_args4(size, str_len, *null_density, mixed);
- c.bench_function(
- &format!("lower_all_values_are_ascii_string_views: size: {size}, str_len: {str_len}, null_density: {null_density}, mixed: {mixed}"),
- |b| b.iter(|| {
- let args_cloned = args.clone();
- black_box(lower.invoke_with_args(ScalarFunctionArgs{
- args: args_cloned,
- arg_fields: arg_fields.clone(),
- number_rows: size,
- return_field: Field::new("f", DataType::Utf8, true).into(),
- config_options: Arc::clone(&config_options),
- }))
- }),
- );
-
- let args = create_args5(size, 0.1, *null_density);
- c.bench_function(
- &format!("lower_some_values_are_nonascii_string_views: size: {}, str_len: {}, non_ascii_density: {}, null_density: {}, mixed: {}",
- size, str_len, 0.1, null_density, mixed),
- |b| b.iter(|| {
- let args_cloned = args.clone();
- black_box(lower.invoke_with_args(ScalarFunctionArgs{
- args: args_cloned,
- arg_fields: arg_fields.clone(),
- number_rows: size,
- return_field: Field::new("f", DataType::Utf8, true).into(),
- config_options: Arc::clone(&config_options),
- }))
- }),
- );
- }
- }
- }
+ for (name, array) in case_conversion_inputs::inputs() {
+ case_conversion_inputs::bench_array(c, &lower, &format!("lower_{name}"), &array);
}
}
diff --git a/datafusion/functions/benches/string_expressions/main.rs b/datafusion/functions/benches/string_expressions/main.rs
index be0276ac90..a928d9607a 100644
--- a/datafusion/functions/benches/string_expressions/main.rs
+++ b/datafusion/functions/benches/string_expressions/main.rs
@@ -25,6 +25,7 @@
use criterion::criterion_main;
mod ascii;
+mod case_conversion_inputs;
mod chr;
mod concat;
mod concat_ws;
@@ -40,7 +41,7 @@ mod split_part;
mod starts_with;
mod to_hex;
mod trim;
-mod upper_unicode;
+mod upper;
mod uuid;
criterion_main!(
@@ -58,6 +59,6 @@ criterion_main!(
starts_with::benches,
to_hex::benches,
trim::benches,
- upper_unicode::benches,
+ upper::benches,
uuid::benches,
);
diff --git a/datafusion/functions/benches/string_expressions/upper.rs b/datafusion/functions/benches/string_expressions/upper.rs
new file mode 100644
index 0000000000..b1466f9527
--- /dev/null
+++ b/datafusion/functions/benches/string_expressions/upper.rs
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::case_conversion_inputs;
+use criterion::{Criterion, criterion_group};
+use datafusion_functions::string;
+
+fn criterion_benchmark(c: &mut Criterion) {
+ let upper = string::upper();
+ for (name, array) in case_conversion_inputs::inputs() {
+ case_conversion_inputs::bench_array(c, &upper, &format!("upper_{name}"), &array);
+ }
+}
+
+criterion_group!(benches, criterion_benchmark);
diff --git a/datafusion/functions/benches/string_expressions/upper_unicode.rs b/datafusion/functions/benches/string_expressions/upper_unicode.rs
deleted file mode 100644
index 9243da2912..0000000000
--- a/datafusion/functions/benches/string_expressions/upper_unicode.rs
+++ /dev/null
@@ -1,89 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Benchmarks `upper` on non-ASCII input, which exercises the
-//! character-streaming case-conversion path (not the ASCII fast path).
-
-use std::hint::black_box;
-use std::sync::Arc;
-
-use arrow::array::{ArrayRef, LargeStringArray, StringArray};
-use arrow::datatypes::{DataType, Field};
-use criterion::{Criterion, criterion_group};
-use datafusion_common::config::ConfigOptions;
-use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDF};
-use datafusion_functions::string;
-
-// A pool of non-ASCII words so `is_ascii()` is false and the Unicode path runs.
-const WORDS: [&str; 8] = [
- "café",
- "straße",
- "αλφα",
- "こんにちは",
- "münchen",
- "naïve",
- " órdenes",
- "tschüß",
-];
-
-fn build_values(size: usize) -> Vec<Option<String>> {
- (0..size)
- .map(|i| {
- if i % 10 == 0 {
- None
- } else {
- // Concatenate a few words for a longer, mixed value.
- let a = WORDS[i % WORDS.len()];
- let b = WORDS[(i * 7 + 3) % WORDS.len()];
- Some(format!("{a} {b} {a}"))
- }
- })
- .collect()
-}
-
-fn invoke(func: &ScalarUDF, array: ArrayRef, dt: DataType) {
- let len = array.len();
- let config_options = Arc::new(ConfigOptions::default());
- black_box(
- func.invoke_with_args(ScalarFunctionArgs {
- args: vec![ColumnarValue::Array(array)],
- arg_fields: vec![Field::new("a", dt.clone(), true).into()],
- number_rows: len,
- return_field: Field::new("f", dt, true).into(),
- config_options,
- })
- .unwrap(),
- );
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
- let upper = string::upper();
- let size = 4096;
- let values = build_values(size);
-
- let utf8: ArrayRef = Arc::new(StringArray::from(values.clone()));
- let large: ArrayRef = Arc::new(LargeStringArray::from(values));
-
- c.bench_function("upper_unicode_utf8", |b| {
- b.iter(|| invoke(&upper, Arc::clone(&utf8), DataType::Utf8))
- });
- c.bench_function("upper_unicode_large_utf8", |b| {
- b.iter(|| invoke(&upper, Arc::clone(&large), DataType::LargeUtf8))
- });
-}
-
-criterion_group!(benches, criterion_benchmark);
diff --git a/datafusion/functions/benches/unicode_expressions/translate.rs b/datafusion/functions/benches/unicode_expressions/translate.rs
index 90fdf4970f..b2cccbc7bf 100644
--- a/datafusion/functions/benches/unicode_expressions/translate.rs
+++ b/datafusion/functions/benches/unicode_expressions/translate.rs
@@ -16,8 +16,10 @@
// under the License.
use arrow::array::{GenericStringArray, OffsetSizeTrait};
-use arrow::datatypes::{DataType, Field};
-use arrow::util::bench_util::create_string_array_with_len;
+use arrow::datatypes::Field;
+use arrow::util::bench_util::{
+ create_string_array_with_len, create_string_view_array_with_len,
+};
use criterion::{Criterion, SamplingMode, criterion_group};
use datafusion_common::config::ConfigOptions;
use datafusion_common::{DataFusionError, ScalarValue};
@@ -101,10 +103,26 @@ fn create_args_scalar_from_to<O: OffsetSizeTrait>(
]
}
+/// Like [`create_args_scalar_from_to`], with the strings as a `StringViewArray`.
+fn create_args_scalar_from_to_utf8view(
+ size: usize,
+ str_len: usize,
+) -> Vec<ColumnarValue> {
+ let string_array =
+ Arc::new(create_string_view_array_with_len(size, 0.1, str_len, false));
+
+ vec![
+ ColumnarValue::Array(string_array),
+ ColumnarValue::Scalar(ScalarValue::from("aeiou")),
+ ColumnarValue::Scalar(ScalarValue::from("AEIOU")),
+ ]
+}
+
fn invoke_translate_with_args(
args: Vec<ColumnarValue>,
number_rows: usize,
) -> Result<ColumnarValue, DataFusionError> {
+ let return_type = args[0].data_type();
let arg_fields = args
.iter()
.enumerate()
@@ -116,7 +134,7 @@ fn invoke_translate_with_args(
args,
arg_fields,
number_rows,
- return_field: Field::new("f", DataType::Utf8, true).into(),
+ return_field: Field::new("f", return_type, true).into(),
config_options: Arc::clone(&config_options),
})
}
@@ -155,6 +173,17 @@ fn criterion_benchmark(c: &mut Criterion) {
black_box(invoke_translate_with_args(args_cloned, size))
})
});
+
+ let args = create_args_scalar_from_to_utf8view(size, str_len);
+ group.bench_function(
+ format!("scalar_from_to_utf8view [str_len={str_len}]"),
+ |b| {
+ b.iter(|| {
+ let args_cloned = args.clone();
+ black_box(invoke_translate_with_args(args_cloned, size))
+ })
+ },
+ );
}
group.finish();
diff --git a/datafusion/functions/src/string/common.rs b/datafusion/functions/src/string/common.rs
index 968790738d..73d66045a0 100644
--- a/datafusion/functions/src/string/common.rs
+++ b/datafusion/functions/src/string/common.rs
@@ -21,7 +21,7 @@ use std::sync::Arc;
use crate::strings::{
GenericStringArrayBuilder, STRING_VIEW_INIT_BLOCK_SIZE, STRING_VIEW_MAX_BLOCK_SIZE,
- StringViewArrayBuilder, StringWriter, substr_view,
+ StringViewArrayBuilder, substr_view,
};
use arrow::array::{
Array, ArrayRef, AsArray, GenericStringArray, NullBufferBuilder, OffsetSizeTrait,
@@ -390,38 +390,56 @@ fn unicode_case(s: &str, lower: bool) -> String {
}
}
-/// Like [`unicode_case`], but uppercases into `buf` so its allocation is reused
-/// across rows. Lowercasing still allocates, see [`write_unicode_case`].
+/// Appends `s` to `builder` with its case converted.
+///
+/// An ASCII value is converted byte by byte straight into the builder. Other
+/// values go through `str::to_lowercase` / `str::to_uppercase`, which allocate
+/// a `String` but convert any leading run of ASCII bytes in bulk. Checking
+/// `is_ascii` first means a non-ASCII value is read up to its first non-ASCII
+/// byte before it is converted; that extra read is cheap compared with what an
+/// ASCII value saves by being written straight into the builder instead of
+/// through a temporary `String`.
+///
+/// Lowercasing needs the whole value: `Σ` becomes `ς` at the end of a word and
+/// `σ` elsewhere, so converting one character at a time, or converting the
+/// ASCII prefix separately, would give the wrong result.
#[inline]
-fn unicode_case_into<'a>(s: &str, lower: bool, buf: &'a mut String) -> &'a str {
- if lower {
- *buf = s.to_lowercase();
- } else {
- buf.clear();
- buf.extend(s.chars().flat_map(char::to_uppercase));
+fn append_case_converted<O: OffsetSizeTrait>(
+ builder: &mut GenericStringArrayBuilder<O>,
+ s: &str,
+ lower: bool,
+) -> Result<()> {
+ if !s.is_ascii() {
+ return builder.try_append_value(&unicode_case(s, lower));
+ }
+ // SAFETY: ASCII case conversion maps ASCII bytes to ASCII bytes, so the
+ // output is valid UTF-8.
+ unsafe {
+ if lower {
+ builder.try_append_byte_map(s.as_bytes(), |b| b.to_ascii_lowercase())
+ } else {
+ builder.try_append_byte_map(s.as_bytes(), |b| b.to_ascii_uppercase())
+ }
}
- buf
}
-/// Writes the case-converted form of `s` directly into `w`.
-///
-/// Uppercasing is a context-free character mapping, so each character is
-/// mapped and streamed straight into the output buffer, avoiding the
-/// intermediate `String` that `str::to_uppercase` allocates per row.
-///
-/// Lowercasing is *not* context-free — `str::to_lowercase` applies the
-/// special Greek final-sigma rule (Σ becomes ς at the end of a word but σ
-/// elsewhere), which a per-character mapping cannot reproduce — so it keeps
-/// using `str::to_lowercase`.
+/// Like [`append_case_converted`], for a `StringViewArray`.
#[inline]
-fn write_unicode_case(w: &mut impl StringWriter, s: &str, lower: bool) {
- if lower {
- w.write_str(&s.to_lowercase());
- } else {
- for c in s.chars() {
- for upper in c.to_uppercase() {
- w.write_char(upper);
- }
+fn append_case_converted_view(
+ builder: &mut StringViewArrayBuilder,
+ s: &str,
+ lower: bool,
+) -> Result<()> {
+ if !s.is_ascii() {
+ return builder.try_append_value(&unicode_case(s, lower));
+ }
+ // SAFETY: ASCII case conversion maps ASCII bytes to ASCII bytes, so the
+ // output is valid UTF-8.
+ unsafe {
+ if lower {
+ builder.try_append_byte_map(s.as_bytes(), |b| b.to_ascii_lowercase())
+ } else {
+ builder.try_append_byte_map(s.as_bytes(), |b| b.to_ascii_uppercase())
}
}
}
@@ -496,7 +514,6 @@ fn case_conversion_utf8view(array: &ArrayRef, lower: bool) -> Result<ArrayRef> {
// Null-preserving: reuse the input null buffer as the output null buffer.
let nulls = string_array.nulls().cloned();
let mut builder = StringViewArrayBuilder::with_capacity(item_len);
- let mut buf = String::new();
if let Some(ref n) = nulls {
for i in 0..item_len {
@@ -505,14 +522,14 @@ fn case_conversion_utf8view(array: &ArrayRef, lower: bool) -> Result<ArrayRef> {
} else {
// SAFETY: `n.is_null(i)` was false in the branch above.
let s = unsafe { string_array.value_unchecked(i) };
- builder.try_append_value(unicode_case_into(s, lower, &mut buf))?;
+ append_case_converted_view(&mut builder, s, lower)?;
}
}
} else {
for i in 0..item_len {
// SAFETY: no null buffer means every index is valid.
let s = unsafe { string_array.value_unchecked(i) };
- builder.try_append_value(unicode_case_into(s, lower, &mut buf))?;
+ append_case_converted_view(&mut builder, s, lower)?;
}
}
@@ -554,14 +571,14 @@ fn case_conversion_array<O: OffsetSizeTrait>(
} else {
// SAFETY: `n.is_null(i)` was false in the branch above.
let s = unsafe { string_array.value_unchecked(i) };
- builder.try_append_with(|w| write_unicode_case(w, s, lower))?;
+ append_case_converted(&mut builder, s, lower)?;
}
}
} else {
for i in 0..item_len {
// SAFETY: no null buffer means every index is valid.
let s = unsafe { string_array.value_unchecked(i) };
- builder.try_append_with(|w| write_unicode_case(w, s, lower))?;
+ append_case_converted(&mut builder, s, lower)?;
}
}
Ok(Arc::new(builder.finish(nulls)?))
diff --git a/datafusion/functions/src/string/lower.rs b/datafusion/functions/src/string/lower.rs
index 18e718526f..ed6e176c2a 100644
--- a/datafusion/functions/src/string/lower.rs
+++ b/datafusion/functions/src/string/lower.rs
@@ -142,27 +142,6 @@ mod tests {
Ok(())
}
- #[test]
- fn lower_maybe_optimization() -> Result<()> {
- let input = Arc::new(StringArray::from(vec![
- Some("农历新年"),
- None,
- Some("DATAFUSION"),
- Some("0123456789"),
- Some(""),
- ])) as ArrayRef;
-
- let expected = Arc::new(StringArray::from(vec![
- Some("农历新年"),
- None,
- Some("datafusion"),
- Some("0123456789"),
- Some(""),
- ])) as ArrayRef;
-
- to_lower(input, expected)
- }
-
#[test]
fn lower_full_optimization() -> Result<()> {
let input = Arc::new(StringArray::from(vec![
@@ -184,56 +163,6 @@ mod tests {
to_lower(input, expected)
}
- #[test]
- fn lower_partial_optimization() -> Result<()> {
- let input = Arc::new(StringArray::from(vec![
- Some("ARROW"),
- None,
- Some("DATAFUSION"),
- Some("@_"),
- Some("0123456789"),
- Some(""),
- Some("\t\n"),
- Some("ὈΔΥΣΣΕΎΣ"),
- Some("TSCHÜSS"),
- Some("Ⱦ"), // ⱦ: length change
- Some("农历新年"),
- ])) as ArrayRef;
-
- let expected = Arc::new(StringArray::from(vec![
- Some("arrow"),
- None,
- Some("datafusion"),
- Some("@_"),
- Some("0123456789"),
- Some(""),
- Some("\t\n"),
- Some("ὀδυσσεύς"),
- Some("tschüss"),
- Some("ⱦ"),
- Some("农历新年"),
- ])) as ArrayRef;
-
- to_lower(input, expected)
- }
-
- #[test]
- fn lower_utf8view() -> Result<()> {
- let input = Arc::new(StringViewArray::from(vec![
- Some("ARROW"),
- None,
- Some("TSCHÜSS"),
- ])) as ArrayRef;
-
- let expected = Arc::new(StringViewArray::from(vec![
- Some("arrow"),
- None,
- Some("tschüss"),
- ])) as ArrayRef;
-
- to_lower(input, expected)
- }
-
#[test]
fn lower_ascii_utf8view() -> Result<()> {
// Mix of inlined (≤12 bytes) and referenced (>12 bytes) strings, plus
diff --git a/datafusion/functions/src/string/upper.rs b/datafusion/functions/src/string/upper.rs
index c7aa430d97..27a50fcfce 100644
--- a/datafusion/functions/src/string/upper.rs
+++ b/datafusion/functions/src/string/upper.rs
@@ -141,27 +141,6 @@ mod tests {
Ok(())
}
- #[test]
- fn upper_maybe_optimization() -> Result<()> {
- let input = Arc::new(StringArray::from(vec![
- Some("农历新年"),
- None,
- Some("datafusion"),
- Some("0123456789"),
- Some(""),
- ])) as ArrayRef;
-
- let expected = Arc::new(StringArray::from(vec![
- Some("农历新年"),
- None,
- Some("DATAFUSION"),
- Some("0123456789"),
- Some(""),
- ])) as ArrayRef;
-
- to_upper(input, expected)
- }
-
#[test]
fn upper_full_optimization() -> Result<()> {
let input = Arc::new(StringArray::from(vec![
@@ -183,56 +162,6 @@ mod tests {
to_upper(input, expected)
}
- #[test]
- fn upper_partial_optimization() -> Result<()> {
- let input = Arc::new(StringArray::from(vec![
- Some("arrow"),
- None,
- Some("datafusion"),
- Some("@_"),
- Some("0123456789"),
- Some(""),
- Some("\t\n"),
- Some("ὀδυσσεύς"),
- Some("tschüß"),
- Some("ⱦ"), // Ⱦ: length change
- Some("农历新年"),
- ])) as ArrayRef;
-
- let expected = Arc::new(StringArray::from(vec![
- Some("ARROW"),
- None,
- Some("DATAFUSION"),
- Some("@_"),
- Some("0123456789"),
- Some(""),
- Some("\t\n"),
- Some("ὈΔΥΣΣΕΎΣ"),
- Some("TSCHÜSS"),
- Some("Ⱦ"),
- Some("农历新年"),
- ])) as ArrayRef;
-
- to_upper(input, expected)
- }
-
- #[test]
- fn upper_utf8view() -> Result<()> {
- let input = Arc::new(StringViewArray::from(vec![
- Some("arrow"),
- None,
- Some("tschüß"),
- ])) as ArrayRef;
-
- let expected = Arc::new(StringViewArray::from(vec![
- Some("ARROW"),
- None,
- Some("TSCHÜSS"),
- ])) as ArrayRef;
-
- to_upper(input, expected)
- }
-
#[test]
fn upper_ascii_utf8view() -> Result<()> {
// Mix of inlined (≤12 bytes) and referenced (>12 bytes) strings, plus
diff --git a/datafusion/functions/src/strings.rs b/datafusion/functions/src/strings.rs
index ed99680ec7..671bca0d4a 100644
--- a/datafusion/functions/src/strings.rs
+++ b/datafusion/functions/src/strings.rs
@@ -422,8 +422,8 @@ impl<O: OffsetSizeTrait> GenericStringArrayBuilder<O> {
/// See [`BulkNullStringArrayBuilder::append_value`].
///
- /// Note: new call sites that need recoverable overflow handling should
- /// prefer [`Self::try_append_value`].
+ /// Prefer [`Self::try_append_value`], which returns an error instead of
+ /// panicking.
///
/// # Panics
///
@@ -436,8 +436,8 @@ impl<O: OffsetSizeTrait> GenericStringArrayBuilder<O> {
/// See [`BulkNullStringArrayBuilder::append_placeholder`].
///
- /// Note: new call sites that need recoverable overflow handling should
- /// prefer [`Self::try_append_placeholder`].
+ /// Prefer [`Self::try_append_placeholder`], which returns an error instead
+ /// of panicking.
#[inline]
pub fn append_placeholder(&mut self) {
self.try_append_placeholder()
@@ -462,14 +462,20 @@ impl<O: OffsetSizeTrait> GenericStringArrayBuilder<O> {
mut map: F,
) -> Result<()> {
self.try_append_bytes(src.len(), |value_buffer| {
- value_buffer.extend(src.iter().map(|&b| map(b)));
+ // Copy, then map the bytes in place: a plain loop over a slice,
+ // which the compiler can vectorize.
+ let old_len = value_buffer.len();
+ value_buffer.extend_from_slice(src);
+ for b in &mut value_buffer.as_slice_mut()[old_len..] {
+ *b = map(*b);
+ }
})
}
/// See [`BulkNullStringArrayBuilder::append_byte_map`].
///
- /// Note: new call sites that need recoverable overflow handling should
- /// prefer [`Self::try_append_byte_map`].
+ /// Prefer [`Self::try_append_byte_map`], which returns an error instead of
+ /// panicking.
///
/// # Safety
///
@@ -517,8 +523,8 @@ impl<O: OffsetSizeTrait> GenericStringArrayBuilder<O> {
/// See [`BulkNullStringArrayBuilder::append_with`].
///
- /// Note: new call sites that need recoverable overflow handling should
- /// prefer [`Self::try_append_with`].
+ /// Prefer [`Self::try_append_with`], which returns an error instead of
+ /// panicking.
///
/// # Panics
///
@@ -731,8 +737,8 @@ impl StringViewArrayBuilder {
/// See [`BulkNullStringArrayBuilder::append_value`].
///
- /// Note: new call sites that need recoverable overflow handling should
- /// prefer [`Self::try_append_value`].
+ /// Prefer [`Self::try_append_value`], which returns an error instead of
+ /// panicking.
///
/// # Panics
///
@@ -757,9 +763,6 @@ impl StringViewArrayBuilder {
}
/// See [`BulkNullStringArrayBuilder::append_placeholder`].
- ///
- /// Note: new call sites that need recoverable overflow handling should
- /// prefer [`Self::try_append_placeholder`].
#[inline]
pub fn append_placeholder(&mut self) {
// Zero-length inline view — `length` field is 0, no buffer ref.
@@ -829,38 +832,78 @@ impl StringViewArrayBuilder {
Self::make_long_view_checked(length, buffer_index, offset, prefix_bytes)
}
- /// See [`BulkNullStringArrayBuilder::append_byte_map`].
+ /// Fallible variant of [`Self::append_byte_map`].
///
/// # Safety
///
/// The bytes produced by applying `map` to each byte of `src`, in order,
/// must form valid UTF-8.
///
- /// # Panics
+ /// # Errors
///
- /// Panics under the same conditions as [`Self::append_value`]: if
- /// `src.len()`, the in-progress buffer offset, or the number of completed
- /// buffers exceeds `i32::MAX`.
+ /// Returns an error under the same conditions as
+ /// [`Self::try_append_value`].
#[inline]
- pub unsafe fn append_byte_map<F: FnMut(u8) -> u8>(&mut self, src: &[u8], mut map: F) {
- let length: u32 =
- i32::try_from(src.len()).expect("value length exceeds i32::MAX") as u32;
- if length <= 12 {
- let mut bytes = [0u8; 12];
- for (d, &b) in bytes[..src.len()].iter_mut().zip(src) {
+ pub unsafe fn try_append_byte_map<F: FnMut(u8) -> u8>(
+ &mut self,
+ src: &[u8],
+ mut map: F,
+ ) -> Result<()> {
+ if src.len() <= MAX_INLINE_LEN {
+ // An inline view holds the length in its low 4 bytes, followed by
+ // the value zero-padded to 12 bytes. Writing the mapped bytes
+ // straight into it is faster than going through `inline_view`.
+ let mut view = (src.len() as u128).to_le_bytes();
+ for (d, &b) in view[4..].iter_mut().zip(src) {
*d = map(b);
}
- self.views.push(make_view(&bytes[..src.len()], 0, 0));
- return;
+ self.views.push(u128::from_le_bytes(view));
+ return Ok(());
}
- self.ensure_long_capacity(length);
+ let length: u32 = i32::try_from(src.len())
+ .map_err(|_| string_view_overflow_error("value length"))?
+ as u32;
+ self.try_ensure_long_capacity(length)?;
let cursor = self.in_progress.len();
- let offset: u32 = i32::try_from(cursor).expect("offset exceeds i32::MAX") as u32;
+ let offset: u32 = i32::try_from(cursor)
+ .map_err(|_| string_view_overflow_error("offset"))?
+ as u32;
+ let buffer_index: u32 = i32::try_from(self.completed.len())
+ .map_err(|_| string_view_overflow_error("buffer count"))?
+ as u32;
self.in_progress.extend(src.iter().map(|&b| map(b)));
- self.views
- .push(self.make_long_view(length, offset, &self.in_progress[cursor..]));
+ // The prefix comes from the mapped bytes, not from `src`.
+ self.views.push(Self::make_long_view_checked(
+ length,
+ buffer_index,
+ offset,
+ &self.in_progress[cursor..],
+ ));
+ Ok(())
+ }
+
+ /// See [`BulkNullStringArrayBuilder::append_byte_map`].
+ ///
+ /// Prefer [`Self::try_append_byte_map`], which returns an error instead of
+ /// panicking.
+ ///
+ /// # Safety
+ ///
+ /// The bytes produced by applying `map` to each byte of `src`, in order,
+ /// must form valid UTF-8.
+ ///
+ /// # Panics
+ ///
+ /// Panics under the same conditions as [`Self::append_value`]: if
+ /// `src.len()`, the in-progress buffer offset, or the number of completed
+ /// buffers exceeds `i32::MAX`.
+ #[inline]
+ pub unsafe fn append_byte_map<F: FnMut(u8) -> u8>(&mut self, src: &[u8], map: F) {
+ // SAFETY: caller upholds this method's UTF-8 contract.
+ unsafe { self.try_append_byte_map(src, map) }
+ .expect("byte array offset overflow");
}
/// See [`BulkNullStringArrayBuilder::append_with`].
@@ -1753,16 +1796,28 @@ mod tests {
#[test]
fn string_view_builder_try_append_success_path() {
- let mut builder = StringViewArrayBuilder::with_capacity(3);
+ let mut builder = StringViewArrayBuilder::with_capacity(5);
builder.try_append_value("abc").unwrap();
builder.try_append_placeholder().unwrap();
builder.try_append_value("a long string value").unwrap();
+ // SAFETY: ASCII inputs and ASCII outputs.
+ unsafe {
+ builder
+ .try_append_byte_map(b"de", |b| b.to_ascii_uppercase())
+ .unwrap();
+ builder
+ .try_append_byte_map(b"a long mapped value", |b| b.to_ascii_uppercase())
+ .unwrap();
+ }
- let nulls = Some(NullBuffer::from(vec![true, false, true]));
+ let nulls = Some(NullBuffer::from(vec![true, false, true, true, true]));
let array = builder.finish(nulls).unwrap();
+ array.to_data().validate_full().unwrap();
assert_eq!(array.value(0), "abc");
assert!(array.is_null(1));
assert_eq!(array.value(2), "a long string value");
+ assert_eq!(array.value(3), "DE");
+ assert_eq!(array.value(4), "A LONG MAPPED VALUE");
}
#[test]
diff --git a/datafusion/sqllogictest/test_files/functions.slt b/datafusion/sqllogictest/test_files/functions.slt
index fbabbb07fe..d2f9f1f1a2 100644
--- a/datafusion/sqllogictest/test_files/functions.slt
+++ b/datafusion/sqllogictest/test_files/functions.slt
@@ -486,6 +486,44 @@ BAR Dictionary(Int32, Utf8)
statement ok
DROP TABLE upper_dictionary_test
+# The array is not all ASCII, so each row is converted on its own: ASCII rows,
+# and rows with non-ASCII characters at the start, middle and end, including
+# values longer than the 12 bytes a Utf8View can store inline.
+statement ok
+CREATE TABLE upper_mixed_test AS
+SELECT column1 AS id,
+ arrow_cast(column2, 'Utf8') AS utf8,
+ arrow_cast(column2, 'LargeUtf8') AS large_utf8,
+ arrow_cast(column2, 'Utf8View') AS utf8view
+FROM (VALUES
+(1, 'datafusion'),
+(2, 'a long ascii-only value'),
+(3, ''),
+(4, NULL),
+(5, 'éa'),
+(6, 'über alles, then ascii'),
+(7, 'straße'),
+(8, 'a long value ending in ß'),
+(9, 'ⱦx'),
+(10, '农历新年 new year'));
+
+query TTT
+SELECT upper(utf8), upper(large_utf8), upper(utf8view) FROM upper_mixed_test ORDER BY id
+----
+DATAFUSION DATAFUSION DATAFUSION
+A LONG ASCII-ONLY VALUE A LONG ASCII-ONLY VALUE A LONG ASCII-ONLY VALUE
+(empty) (empty) (empty)
+NULL NULL NULL
+ÉA ÉA ÉA
+ÜBER ALLES, THEN ASCII ÜBER ALLES, THEN ASCII ÜBER ALLES, THEN ASCII
+STRASSE STRASSE STRASSE
+A LONG VALUE ENDING IN SS A LONG VALUE ENDING IN SS A LONG VALUE ENDING IN SS
+ȾX ȾX ȾX
+农历新年 NEW YEAR 农历新年 NEW YEAR 农历新年 NEW YEAR
+
+statement ok
+DROP TABLE upper_mixed_test
+
# upper and lower preserve input nullability in the planned schema
query TTT
DESCRIBE SELECT upper(a) AS ua, lower(a) AS la, upper(b) AS ub
@@ -708,6 +746,47 @@ bar Dictionary(Int32, Utf8)
statement ok
DROP TABLE lower_dictionary_test
+# The array is not all ASCII, so each row is converted on its own: ASCII rows,
+# and rows with non-ASCII characters at the start and end, including values
+# longer than the 12 bytes a Utf8View can store inline. A sigma lowercases to ς
+# at the end of a word and to σ elsewhere.
+statement ok
+CREATE TABLE lower_mixed_test AS
+SELECT column1 AS id,
+ arrow_cast(column2, 'Utf8') AS utf8,
+ arrow_cast(column2, 'LargeUtf8') AS large_utf8,
+ arrow_cast(column2, 'Utf8View') AS utf8view
+FROM (VALUES
+(1, 'DATAFUSION'),
+(2, 'A LONG ASCII-ONLY VALUE'),
+(3, ''),
+(4, NULL),
+(5, 'ÉA'),
+(6, 'ÜBER ALLES, THEN ASCII'),
+(7, 'A LONG VALUE ENDING IN ΟΔΟΣ'),
+(8, 'ABΣ'),
+(9, 'ȾX'),
+(10, '农历新年 NEW YEAR'),
+(11, 'ὈΔΥΣΣΕΎΣ'));
+
+query TTT
+SELECT lower(utf8), lower(large_utf8), lower(utf8view) FROM lower_mixed_test ORDER BY id
+----
+datafusion datafusion datafusion
+a long ascii-only value a long ascii-only value a long ascii-only value
+(empty) (empty) (empty)
+NULL NULL NULL
+éa éa éa
+über alles, then ascii über alles, then ascii über alles, then ascii
+a long value ending in οδος a long value ending in οδος a long value ending in οδος
+abς abς abς
+ⱦx ⱦx ⱦx
+农历新年 new year 农历新年 new year 农历新年 new year
+ὀδυσσεύς ὀδυσσεύς ὀδυσσεύς
+
+statement ok
+DROP TABLE lower_mixed_test
+
query T
SELECT ltrim(' foo')
----