Skip to content

Apache DataFusion SQL Query Engine

perf: optimize encode in datafusion-functions (#23456)

Andy Grove5b60fcb1107b
diff --git a/datafusion/functions/benches/encoding.rs b/datafusion/functions/benches/encoding.rs
index 0b8f0c5c51..451baff518 100644
--- a/datafusion/functions/benches/encoding.rs
+++ b/datafusion/functions/benches/encoding.rs
@@ -27,10 +27,35 @@ use std::sync::Arc;
fn criterion_benchmark(c: &mut Criterion) {
let decode = encoding::decode();
+ let encode = encoding::encode();
let config_options = Arc::new(ConfigOptions::default());
for size in [1024, 4096, 8192] {
let bin_array = Arc::new(create_binary_array::<i32>(size, 0.2));
+
+ c.bench_function(&format!("hex_encode/{size}"), |b| {
+ let method = ColumnarValue::Scalar("hex".into());
+ let arg_fields = vec![
+ Field::new("a", bin_array.data_type().to_owned(), true).into(),
+ Field::new("b", method.data_type().to_owned(), true).into(),
+ ];
+ let args = vec![ColumnarValue::Array(bin_array.clone()), method];
+ let return_field = Field::new("f", DataType::Utf8, true).into();
+
+ b.iter(|| {
+ black_box(
+ encode
+ .invoke_with_args(ScalarFunctionArgs {
+ args: args.clone(),
+ arg_fields: arg_fields.clone(),
+ number_rows: size,
+ return_field: Arc::clone(&return_field),
+ config_options: Arc::clone(&config_options),
+ })
+ .unwrap(),
+ )
+ })
+ });
c.bench_function(&format!("base64_decode/{size}"), |b| {
let method = ColumnarValue::Scalar("base64".into());
let encoded = encoding::encode()
diff --git a/datafusion/functions/src/encoding/inner.rs b/datafusion/functions/src/encoding/inner.rs
index 877acbb529..027ec8e5e5 100644
--- a/datafusion/functions/src/encoding/inner.rs
+++ b/datafusion/functions/src/encoding/inner.rs
@@ -410,11 +410,7 @@ impl Encoding {
.collect();
Ok(Arc::new(array))
}
- Self::Hex => {
- let array: GenericStringArray<OutputOffset> =
- array.iter().map(|x| x.map(hex::encode)).collect();
- Ok(Arc::new(array))
- }
+ Self::Hex => hex_encode_array::<_, OutputOffset>(array),
}
}
@@ -459,6 +455,45 @@ impl Encoding {
}
}
+/// Hex-encode a binary array into a string array, writing the lowercase hex
+/// digits directly into a single pre-sized value buffer. Each input byte maps
+/// to exactly two hex characters, so the output size is known up front and no
+/// per-element `String` is allocated.
+fn hex_encode_array<'a, InputBinaryArray, OutputOffset>(
+ array: &InputBinaryArray,
+) -> Result<ArrayRef>
+where
+ InputBinaryArray: BinaryArrayType<'a>,
+ OutputOffset: OffsetSizeTrait,
+{
+ let total_input_bytes: usize = array.iter().flatten().map(|v| v.len()).sum();
+
+ let mut values = vec![0u8; total_input_bytes * 2];
+ let mut offsets = Vec::<OutputOffset>::with_capacity(array.len() + 1);
+ offsets.push(OutputOffset::zero());
+
+ let mut pos = 0usize;
+ for v in array.iter() {
+ if let Some(v) = v {
+ let out_len = v.len() * 2;
+ // The slice is sized to exactly `2 * v.len()`, which is the only
+ // condition under which `encode_to_slice` can fail, so this cannot
+ // error.
+ hex::encode_to_slice(v, &mut values[pos..pos + out_len])
+ .map_err(|e| exec_datafusion_err!("Failed to encode to hex: {e}"))?;
+ pos += out_len;
+ }
+ offsets.push(OutputOffset::usize_as(pos));
+ }
+
+ let array = GenericStringArray::<OutputOffset>::try_new(
+ OffsetBuffer::new(offsets.into()),
+ Buffer::from_vec(values),
+ array.nulls().cloned(),
+ )?;
+ Ok(Arc::new(array))
+}
+
fn delegated_decode<'a, DecodeFunction, InputBinaryArray, OutputOffset>(
decode: DecodeFunction,
input: &InputBinaryArray,