Apache DataFusion SQL Query Engine
perf: Optimize translate() UDF for scalar inputs (#20305)
Neil Conwayc3f080774cd5
diff --git a/datafusion/functions/benches/translate.rs b/datafusion/functions/benches/translate.rsindex f63faacd39..d0568ba0f5 100644--- a/datafusion/functions/benches/translate.rs+++ b/datafusion/functions/benches/translate.rs@@ -19,17 +19,19 @@ use arrow::array::OffsetSizeTrait;use arrow::datatypes::{DataType, Field};use arrow::util::bench_util::create_string_array_with_len;use criterion::{Criterion, SamplingMode, criterion_group, criterion_main};-use datafusion_common::DataFusionError;use datafusion_common::config::ConfigOptions;+use datafusion_common::{DataFusionError, ScalarValue};use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};use datafusion_functions::unicode;use std::hint::black_box;use std::sync::Arc;use std::time::Duration;-fn create_args<O: OffsetSizeTrait>(size: usize, str_len: usize) -> Vec<ColumnarValue> {+fn create_args_array_from_to<O: OffsetSizeTrait>(+ size: usize,+ str_len: usize,+) -> Vec<ColumnarValue> {let string_array = Arc::new(create_string_array_with_len::<O>(size, 0.1, str_len));- // Create simple from/to strings for translationlet from_array = Arc::new(create_string_array_with_len::<O>(size, 0.1, 3));let to_array = Arc::new(create_string_array_with_len::<O>(size, 0.1, 2));@@ -40,6 +42,19 @@ fn create_args<O: OffsetSizeTrait>(size: usize, str_len: usize) -> Vec<ColumnarV]}+fn create_args_scalar_from_to<O: OffsetSizeTrait>(+ size: usize,+ str_len: usize,+) -> Vec<ColumnarValue> {+ let string_array = Arc::new(create_string_array_with_len::<O>(size, 0.1, str_len));++ vec![+ ColumnarValue::Array(string_array),+ ColumnarValue::Scalar(ScalarValue::from("aeiou")),+ ColumnarValue::Scalar(ScalarValue::from("AEIOU")),+ ]+}+fn invoke_translate_with_args(args: Vec<ColumnarValue>,number_rows: usize,@@ -67,17 +82,22 @@ fn criterion_benchmark(c: &mut Criterion) {group.sample_size(10);group.measurement_time(Duration::from_secs(10));- for str_len in [8, 32] {- let args = create_args::<i32>(size, str_len);- group.bench_function(- format!("translate_string [size={size}, str_len={str_len}]"),- |b| {- b.iter(|| {- let args_cloned = args.clone();- black_box(invoke_translate_with_args(args_cloned, size))- })- },- );+ for str_len in [8, 32, 128, 1024] {+ let args = create_args_array_from_to::<i32>(size, str_len);+ group.bench_function(format!("array_from_to [str_len={str_len}]"), |b| {+ b.iter(|| {+ let args_cloned = args.clone();+ black_box(invoke_translate_with_args(args_cloned, size))+ })+ });++ let args = create_args_scalar_from_to::<i32>(size, str_len);+ group.bench_function(format!("scalar_from_to [str_len={str_len}]"), |b| {+ b.iter(|| {+ let args_cloned = args.clone();+ black_box(invoke_translate_with_args(args_cloned, size))+ })+ });}group.finish();diff --git a/datafusion/functions/src/unicode/translate.rs b/datafusion/functions/src/unicode/translate.rsindex f97c0ed5c2..e86eaf8111 100644--- a/datafusion/functions/src/unicode/translate.rs+++ b/datafusion/functions/src/unicode/translate.rs@@ -35,8 +35,8 @@ use datafusion_macros::user_doc;#[user_doc(doc_section(label = "String Functions"),- description = "Translates characters in a string to specified translation characters.",- syntax_example = "translate(str, chars, translation)",+ description = "Performs character-wise substitution based on a mapping.",+ syntax_example = "translate(str, from, to)",sql_example = r#"```sql> select translate('twice', 'wic', 'her');+--------------------------------------------------+@@ -46,10 +46,10 @@ use datafusion_macros::user_doc;+--------------------------------------------------+```"#,standard_argument(name = "str", prefix = "String"),- argument(name = "chars", description = "Characters to translate."),+ argument(name = "from", description = "The characters to be replaced."),argument(- name = "translation",- description = "Translation characters. Translation characters replace only characters at the same position in the **chars** string."+ name = "to",+ description = "The characters to replace them with. Each character in **from** that is found in **str** is replaced by the character at the same index in **to**. Any characters in **from** that don't have a corresponding character in **to** are removed. If a character appears more than once in **from**, the first occurrence determines the mapping."))]#[derive(Debug, PartialEq, Eq, Hash)]@@ -71,6 +71,7 @@ impl TranslateFunc {vec![Exact(vec![Utf8View, Utf8, Utf8]),Exact(vec![Utf8, Utf8, Utf8]),+ Exact(vec![LargeUtf8, Utf8, Utf8]),],Volatility::Immutable,),@@ -99,6 +100,61 @@ impl ScalarUDFImpl for TranslateFunc {&self,args: datafusion_expr::ScalarFunctionArgs,) -> Result<ColumnarValue> {+ // When from and to are scalars, pre-build the translation map once+ if let (Some(from_str), Some(to_str)) = (+ try_as_scalar_str(&args.args[1]),+ try_as_scalar_str(&args.args[2]),+ ) {+ let to_graphemes: Vec<&str> = to_str.graphemes(true).collect();++ let mut from_map: HashMap<&str, usize> = HashMap::new();+ for (index, c) in from_str.graphemes(true).enumerate() {+ // Ignore characters that already exist in from_map+ from_map.entry(c).or_insert(index);+ }++ let ascii_table = build_ascii_translate_table(from_str, to_str);++ let string_array = args.args[0].to_array_of_size(args.number_rows)?;++ let result = match string_array.data_type() {+ DataType::Utf8View => {+ let arr = string_array.as_string_view();+ translate_with_map::<i32, _>(+ arr,+ &from_map,+ &to_graphemes,+ ascii_table.as_ref(),+ )+ }+ DataType::Utf8 => {+ let arr = string_array.as_string::<i32>();+ translate_with_map::<i32, _>(+ arr,+ &from_map,+ &to_graphemes,+ ascii_table.as_ref(),+ )+ }+ DataType::LargeUtf8 => {+ let arr = string_array.as_string::<i64>();+ translate_with_map::<i64, _>(+ arr,+ &from_map,+ &to_graphemes,+ ascii_table.as_ref(),+ )+ }+ other => {+ return exec_err!(+ "Unsupported data type {other:?} for function translate"+ );+ }+ }?;++ return Ok(ColumnarValue::Array(result));+ }+make_scalar_function(invoke_translate, vec![])(&args.args)}@@ -107,6 +163,14 @@ impl ScalarUDFImpl for TranslateFunc {}}+/// If `cv` is a non-null scalar string, return its value.+fn try_as_scalar_str(cv: &ColumnarValue) -> Option<&str> {+ match cv {+ ColumnarValue::Scalar(s) => s.try_as_str().flatten(),+ _ => None,+ }+}+fn invoke_translate(args: &[ArrayRef]) -> Result<ArrayRef> {match args[0].data_type() {DataType::Utf8View => {@@ -123,8 +187,8 @@ fn invoke_translate(args: &[ArrayRef]) -> Result<ArrayRef> {}DataType::LargeUtf8 => {let string_array = args[0].as_string::<i64>();- let from_array = args[1].as_string::<i64>();- let to_array = args[2].as_string::<i64>();+ let from_array = args[1].as_string::<i32>();+ let to_array = args[2].as_string::<i32>();translate::<i64, _, _>(string_array, from_array, to_array)}other => {@@ -170,7 +234,7 @@ where// Build from_map using reusable bufferfrom_graphemes.extend(from.graphemes(true));for (index, c) in from_graphemes.iter().enumerate() {- // Ignore characters that already exist in from_map, else insert+ // Ignore characters that already exist in from_mapfrom_map.entry(*c).or_insert(index);}@@ -199,6 +263,97 @@ whereOk(Arc::new(result) as ArrayRef)}+/// Sentinel value in the ASCII translate table indicating the character should+/// be deleted (the `from` character has no corresponding `to` character). Any+/// value > 127 works since valid ASCII is 0–127.+const ASCII_DELETE: u8 = 0xFF;++/// If `from` and `to` are both ASCII, build a fixed-size lookup table for+/// translation. Each entry maps an input byte to its replacement byte, or to+/// [`ASCII_DELETE`] if the character should be removed. Returns `None` if+/// either string contains non-ASCII characters.+fn build_ascii_translate_table(from: &str, to: &str) -> Option<[u8; 128]> {+ if !from.is_ascii() || !to.is_ascii() {+ return None;+ }+ let mut table = [0u8; 128];+ for i in 0..128u8 {+ table[i as usize] = i;+ }+ let to_bytes = to.as_bytes();+ let mut seen = [false; 128];+ for (i, from_byte) in from.bytes().enumerate() {+ let idx = from_byte as usize;+ if !seen[idx] {+ seen[idx] = true;+ if i < to_bytes.len() {+ table[idx] = to_bytes[i];+ } else {+ table[idx] = ASCII_DELETE;+ }+ }+ }+ Some(table)+}++/// Optimized translate for constant `from` and `to` arguments: uses a pre-built+/// translation map instead of rebuilding it for every row. When an ASCII byte+/// lookup table is provided, ASCII input rows use the lookup table; non-ASCII+/// inputs fallback to using the map.+fn translate_with_map<'a, T: OffsetSizeTrait, V>(+ string_array: V,+ from_map: &HashMap<&str, usize>,+ to_graphemes: &[&str],+ ascii_table: Option<&[u8; 128]>,+) -> Result<ArrayRef>+where+ V: ArrayAccessor<Item = &'a str>,+{+ let mut result_graphemes: Vec<&str> = Vec::new();+ let mut ascii_buf: Vec<u8> = Vec::new();++ let result = ArrayIter::new(string_array)+ .map(|string| {+ string.map(|s| {+ // Fast path: byte-level table lookup for ASCII strings+ if let Some(table) = ascii_table+ && s.is_ascii()+ {+ ascii_buf.clear();+ for &b in s.as_bytes() {+ let mapped = table[b as usize];+ if mapped != ASCII_DELETE {+ ascii_buf.push(mapped);+ }+ }+ // SAFETY: all bytes are ASCII, hence valid UTF-8.+ return unsafe {+ std::str::from_utf8_unchecked(&ascii_buf).to_owned()+ };+ }++ // Slow path: grapheme-based translation+ result_graphemes.clear();++ for c in s.graphemes(true) {+ match from_map.get(c) {+ Some(n) => {+ if let Some(replacement) = to_graphemes.get(*n) {+ result_graphemes.push(*replacement);+ }+ }+ None => result_graphemes.push(c),+ }+ }++ result_graphemes.concat()+ })+ })+ .collect::<GenericStringArray<T>>();++ Ok(Arc::new(result) as ArrayRef)+}+#[cfg(test)]mod tests {use arrow::array::{Array, StringArray};@@ -284,6 +439,21 @@ mod tests {Utf8,StringArray);+ // Non-ASCII input with ASCII scalar from/to: exercises the+ // grapheme fallback within translate_with_map.+ test_function!(+ TranslateFunc::new(),+ vec![+ ColumnarValue::Scalar(ScalarValue::from("café")),+ ColumnarValue::Scalar(ScalarValue::from("ae")),+ ColumnarValue::Scalar(ScalarValue::from("AE"))+ ],+ Ok(Some("cAfé")),+ &str,+ Utf8,+ StringArray+ );+#[cfg(not(feature = "unicode_expressions"))]test_function!(TranslateFunc::new(),diff --git a/datafusion/sqllogictest/test_files/functions.slt b/datafusion/sqllogictest/test_files/functions.sltindex 6c87d618c7..35a32897d0 100644--- a/datafusion/sqllogictest/test_files/functions.slt+++ b/datafusion/sqllogictest/test_files/functions.slt@@ -239,6 +239,11 @@ SELECT translate('12345', '143', NULL)----NULL+query T+SELECT translate(arrow_cast('12345', 'LargeUtf8'), '143', 'ax')+----+a2x5+statement okCREATE TABLE test(c1 VARCHARdiff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.mdindex e09c4cb7cb..02b4b55fe6 100644--- a/docs/source/user-guide/sql/scalar_functions.md+++ b/docs/source/user-guide/sql/scalar_functions.md@@ -2068,17 +2068,17 @@ to_hex(int)### `translate`-Translates characters in a string to specified translation characters.+Performs character-wise substitution based on a mapping.```sql-translate(str, chars, translation)+translate(str, from, to)```#### Arguments- **str**: String expression to operate on. Can be a constant, column, or function, and any combination of operators.-- **chars**: Characters to translate.-- **translation**: Translation characters. Translation characters replace only characters at the same position in the **chars** string.+- **from**: The characters to be replaced.+- **to**: The characters to replace them with. Each character in **from** that is found in **str** is replaced by the character at the same index in **to**. Any characters in **from** that don't have a corresponding character in **to** are removed. If a character appears more than once in **from**, the first occurrence determines the mapping.#### Example