Apache DataFusion SQL Query Engine
perf: optimize encode in datafusion-functions (#23456)
Andy Grove5b60fcb1107b
diff --git a/datafusion/functions/benches/encoding.rs b/datafusion/functions/benches/encoding.rsindex 0b8f0c5c51..451baff518 100644--- a/datafusion/functions/benches/encoding.rs+++ b/datafusion/functions/benches/encoding.rs@@ -27,10 +27,35 @@ use std::sync::Arc;fn criterion_benchmark(c: &mut Criterion) {let decode = encoding::decode();+ let encode = encoding::encode();let config_options = Arc::new(ConfigOptions::default());for size in [1024, 4096, 8192] {let bin_array = Arc::new(create_binary_array::<i32>(size, 0.2));++ c.bench_function(&format!("hex_encode/{size}"), |b| {+ let method = ColumnarValue::Scalar("hex".into());+ let arg_fields = vec![+ Field::new("a", bin_array.data_type().to_owned(), true).into(),+ Field::new("b", method.data_type().to_owned(), true).into(),+ ];+ let args = vec![ColumnarValue::Array(bin_array.clone()), method];+ let return_field = Field::new("f", DataType::Utf8, true).into();++ b.iter(|| {+ black_box(+ encode+ .invoke_with_args(ScalarFunctionArgs {+ args: args.clone(),+ arg_fields: arg_fields.clone(),+ number_rows: size,+ return_field: Arc::clone(&return_field),+ config_options: Arc::clone(&config_options),+ })+ .unwrap(),+ )+ })+ });c.bench_function(&format!("base64_decode/{size}"), |b| {let method = ColumnarValue::Scalar("base64".into());let encoded = encoding::encode()diff --git a/datafusion/functions/src/encoding/inner.rs b/datafusion/functions/src/encoding/inner.rsindex 877acbb529..027ec8e5e5 100644--- a/datafusion/functions/src/encoding/inner.rs+++ b/datafusion/functions/src/encoding/inner.rs@@ -410,11 +410,7 @@ impl Encoding {.collect();Ok(Arc::new(array))}- Self::Hex => {- let array: GenericStringArray<OutputOffset> =- array.iter().map(|x| x.map(hex::encode)).collect();- Ok(Arc::new(array))- }+ Self::Hex => hex_encode_array::<_, OutputOffset>(array),}}@@ -459,6 +455,45 @@ impl Encoding {}}+/// Hex-encode a binary array into a string array, writing the lowercase hex+/// digits directly into a single pre-sized value buffer. Each input byte maps+/// to exactly two hex characters, so the output size is known up front and no+/// per-element `String` is allocated.+fn hex_encode_array<'a, InputBinaryArray, OutputOffset>(+ array: &InputBinaryArray,+) -> Result<ArrayRef>+where+ InputBinaryArray: BinaryArrayType<'a>,+ OutputOffset: OffsetSizeTrait,+{+ let total_input_bytes: usize = array.iter().flatten().map(|v| v.len()).sum();++ let mut values = vec![0u8; total_input_bytes * 2];+ let mut offsets = Vec::<OutputOffset>::with_capacity(array.len() + 1);+ offsets.push(OutputOffset::zero());++ let mut pos = 0usize;+ for v in array.iter() {+ if let Some(v) = v {+ let out_len = v.len() * 2;+ // The slice is sized to exactly `2 * v.len()`, which is the only+ // condition under which `encode_to_slice` can fail, so this cannot+ // error.+ hex::encode_to_slice(v, &mut values[pos..pos + out_len])+ .map_err(|e| exec_datafusion_err!("Failed to encode to hex: {e}"))?;+ pos += out_len;+ }+ offsets.push(OutputOffset::usize_as(pos));+ }++ let array = GenericStringArray::<OutputOffset>::try_new(+ OffsetBuffer::new(offsets.into()),+ Buffer::from_vec(values),+ array.nulls().cloned(),+ )?;+ Ok(Arc::new(array))+}+fn delegated_decode<'a, DecodeFunction, InputBinaryArray, OutputOffset>(decode: DecodeFunction,input: &InputBinaryArray,