Apache DataFusion SQL Query Engine
perf: Improve performance of `to_hex` (> 2x) (#19503)
Andy Grovee5ca510dcf03
diff --git a/datafusion/functions/benches/to_hex.rs b/datafusion/functions/benches/to_hex.rsindex 7346b89847..1c6757a291 100644--- a/datafusion/functions/benches/to_hex.rs+++ b/datafusion/functions/benches/to_hex.rs@@ -17,56 +17,102 @@extern crate criterion;+use arrow::array::Int64Array;use arrow::datatypes::{DataType, Field, Int32Type, Int64Type};use arrow::util::bench_util::create_primitive_array;-use criterion::{Criterion, criterion_group, criterion_main};+use criterion::{Criterion, SamplingMode, criterion_group, criterion_main};use datafusion_common::config::ConfigOptions;use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};use datafusion_functions::string;use std::hint::black_box;use std::sync::Arc;+use std::time::Duration;fn criterion_benchmark(c: &mut Criterion) {let hex = string::to_hex();- let size = 1024;- let i32_array = Arc::new(create_primitive_array::<Int32Type>(size, 0.2));- let batch_len = i32_array.len();- let i32_args = vec![ColumnarValue::Array(i32_array)];let config_options = Arc::new(ConfigOptions::default());- c.bench_function(&format!("to_hex i32 array: {size}"), |b| {- b.iter(|| {- let args_cloned = i32_args.clone();- black_box(- hex.invoke_with_args(ScalarFunctionArgs {- args: args_cloned,- arg_fields: vec![Field::new("a", DataType::Int32, false).into()],- number_rows: batch_len,- return_field: Field::new("f", DataType::Utf8, true).into(),- config_options: Arc::clone(&config_options),- })- .unwrap(),- )- })- });- let i64_array = Arc::new(create_primitive_array::<Int64Type>(size, 0.2));- let batch_len = i64_array.len();- let i64_args = vec![ColumnarValue::Array(i64_array)];- c.bench_function(&format!("to_hex i64 array: {size}"), |b| {- b.iter(|| {- let args_cloned = i64_args.clone();- black_box(- hex.invoke_with_args(ScalarFunctionArgs {- args: args_cloned,- arg_fields: vec![Field::new("a", DataType::Int64, false).into()],- number_rows: batch_len,- return_field: Field::new("f", DataType::Utf8, true).into(),- config_options: Arc::clone(&config_options),+ for size in [1024, 4096, 8192] {+ let mut group = c.benchmark_group(format!("to_hex size={size}"));+ group.sampling_mode(SamplingMode::Flat);+ group.sample_size(10);+ group.measurement_time(Duration::from_secs(10));++ // i32 array with random values+ let i32_array = Arc::new(create_primitive_array::<Int32Type>(size, 0.1));+ let batch_len = i32_array.len();+ let i32_args = vec![ColumnarValue::Array(i32_array)];++ group.bench_function("i32_random", |b| {+ b.iter(|| {+ let args_cloned = i32_args.clone();+ black_box(+ hex.invoke_with_args(ScalarFunctionArgs {+ args: args_cloned,+ arg_fields: vec![Field::new("a", DataType::Int32, true).into()],+ number_rows: batch_len,+ return_field: Field::new("f", DataType::Utf8, true).into(),+ config_options: Arc::clone(&config_options),+ })+ .unwrap(),+ )+ })+ });++ // i64 array with random values (produces longer hex strings)+ let i64_array = Arc::new(create_primitive_array::<Int64Type>(size, 0.1));+ let batch_len = i64_array.len();+ let i64_args = vec![ColumnarValue::Array(i64_array)];++ group.bench_function("i64_random", |b| {+ b.iter(|| {+ let args_cloned = i64_args.clone();+ black_box(+ hex.invoke_with_args(ScalarFunctionArgs {+ args: args_cloned,+ arg_fields: vec![Field::new("a", DataType::Int64, true).into()],+ number_rows: batch_len,+ return_field: Field::new("f", DataType::Utf8, true).into(),+ config_options: Arc::clone(&config_options),+ })+ .unwrap(),+ )+ })+ });++ // i64 array with large values (max length hex strings)+ let i64_large_array = Arc::new(Int64Array::from(+ (0..size)+ .map(|i| {+ if i % 10 == 0 {+ None+ } else {+ Some(i64::MAX - i as i64)+ }})- .unwrap(),- )- })- });+ .collect::<Vec<_>>(),+ ));+ let batch_len = i64_large_array.len();+ let i64_large_args = vec![ColumnarValue::Array(i64_large_array)];++ group.bench_function("i64_large_values", |b| {+ b.iter(|| {+ let args_cloned = i64_large_args.clone();+ black_box(+ hex.invoke_with_args(ScalarFunctionArgs {+ args: args_cloned,+ arg_fields: vec![Field::new("a", DataType::Int64, true).into()],+ number_rows: batch_len,+ return_field: Field::new("f", DataType::Utf8, true).into(),+ config_options: Arc::clone(&config_options),+ })+ .unwrap(),+ )+ })+ });++ group.finish();+ }}criterion_group!(benches, criterion_benchmark);diff --git a/datafusion/functions/src/string/to_hex.rs b/datafusion/functions/src/string/to_hex.rsindex fb34c96ad8..dd4f417426 100644--- a/datafusion/functions/src/string/to_hex.rs+++ b/datafusion/functions/src/string/to_hex.rs@@ -16,11 +16,11 @@// under the License.use std::any::Any;-use std::fmt::Write;use std::sync::Arc;use crate::utils::make_scalar_function;-use arrow::array::{ArrayRef, GenericStringBuilder};+use arrow::array::{Array, ArrayRef, StringArray};+use arrow::buffer::{Buffer, OffsetBuffer};use arrow::datatypes::DataType::{Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64, Utf8,};@@ -37,42 +37,142 @@ use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};use datafusion_expr_common::signature::TypeSignature::Exact;use datafusion_macros::user_doc;+/// Hex lookup table for fast conversion+const HEX_CHARS: &[u8; 16] = b"0123456789abcdef";+/// Converts the number to its equivalent hexadecimal representation./// to_hex(2147483647) = '7fffffff'fn to_hex<T: ArrowPrimitiveType>(args: &[ArrayRef]) -> Result<ArrayRef>where- T::Native: std::fmt::LowerHex,+ T::Native: ToHex,{let integer_array = as_primitive_array::<T>(&args[0])?;+ let len = integer_array.len();- let mut result = GenericStringBuilder::<i32>::with_capacity(- integer_array.len(),- // * 8 to convert to bits, / 4 bits per hex char- integer_array.len() * (T::Native::get_byte_width() * 8 / 4),- );+ // Max hex string length: 16 chars for u64/i64+ let max_hex_len = T::Native::get_byte_width() * 2;- for integer in integer_array {- if let Some(value) = integer {- if let Some(value_usize) = value.to_usize() {- write!(result, "{value_usize:x}")?;- } else if let Some(value_isize) = value.to_isize() {- write!(result, "{value_isize:x}")?;- } else {- return exec_err!(- "Unsupported data type {integer:?} for function to_hex"- );- }- result.append_value("");- } else {- result.append_null();- }+ // Pre-allocate buffers - avoid the builder API overhead+ let mut offsets: Vec<i32> = Vec::with_capacity(len + 1);+ let mut values: Vec<u8> = Vec::with_capacity(len * max_hex_len);++ // Reusable buffer for hex conversion+ let mut hex_buffer = [0u8; 16];++ // Start with offset 0+ offsets.push(0);++ // Process all values directly (including null slots - we write empty strings for nulls)+ // The null bitmap will mark which entries are actually null+ for value in integer_array.values() {+ let hex_len = value.write_hex_to_buffer(&mut hex_buffer);+ values.extend_from_slice(&hex_buffer[16 - hex_len..]);+ offsets.push(values.len() as i32);}- let result = result.finish();+ // Copy null bitmap from input (nulls pass through unchanged)+ let nulls = integer_array.nulls().cloned();++ // SAFETY: offsets are valid (monotonically increasing, last value equals values.len())+ // and values contains valid UTF-8 (only ASCII hex digits)+ let offsets =+ unsafe { OffsetBuffer::new_unchecked(Buffer::from_vec(offsets).into()) };+ let result = StringArray::new(offsets, Buffer::from_vec(values), nulls);Ok(Arc::new(result) as ArrayRef)}+/// Trait for converting integer types to hexadecimal in a buffer+trait ToHex: ArrowNativeType {+ /// Write hex representation to buffer and return the number of hex digits written.+ /// The hex digits are written right-aligned in the buffer (starting from position 16 - len).+ fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize;+}++/// Write unsigned value to hex buffer and return the number of digits written.+/// Digits are written right-aligned in the buffer.+#[inline]+fn write_unsigned_hex_to_buffer(value: u64, buffer: &mut [u8; 16]) -> usize {+ if value == 0 {+ buffer[15] = b'0';+ return 1;+ }++ // Write hex digits from right to left+ let mut pos = 16;+ let mut v = value;+ while v > 0 {+ pos -= 1;+ buffer[pos] = HEX_CHARS[(v & 0xf) as usize];+ v >>= 4;+ }++ 16 - pos+}++/// Write signed value to hex buffer (two's complement for negative) and return digit count+#[inline]+fn write_signed_hex_to_buffer(value: i64, buffer: &mut [u8; 16]) -> usize {+ // For negative values, use two's complement representation (same as casting to u64)+ write_unsigned_hex_to_buffer(value as u64, buffer)+}++impl ToHex for i8 {+ #[inline]+ fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize {+ write_signed_hex_to_buffer(self as i64, buffer)+ }+}++impl ToHex for i16 {+ #[inline]+ fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize {+ write_signed_hex_to_buffer(self as i64, buffer)+ }+}++impl ToHex for i32 {+ #[inline]+ fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize {+ write_signed_hex_to_buffer(self as i64, buffer)+ }+}++impl ToHex for i64 {+ #[inline]+ fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize {+ write_signed_hex_to_buffer(self, buffer)+ }+}++impl ToHex for u8 {+ #[inline]+ fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize {+ write_unsigned_hex_to_buffer(self as u64, buffer)+ }+}++impl ToHex for u16 {+ #[inline]+ fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize {+ write_unsigned_hex_to_buffer(self as u64, buffer)+ }+}++impl ToHex for u32 {+ #[inline]+ fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize {+ write_unsigned_hex_to_buffer(self as u64, buffer)+ }+}++impl ToHex for u64 {+ #[inline]+ fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize {+ write_unsigned_hex_to_buffer(self, buffer)+ }+}+#[user_doc(doc_section(label = "String Functions"),description = "Converts an integer to a hexadecimal string.",