Apache DataFusion SQL Query Engine
perf: Optimize scalar fast path of to_hex function (#20112)
Kumar Ujjawal57ff351566b5
diff --git a/datafusion/functions/benches/to_hex.rs b/datafusion/functions/benches/to_hex.rsindex 1c6757a291..356c3c6c05 100644--- a/datafusion/functions/benches/to_hex.rs+++ b/datafusion/functions/benches/to_hex.rs@@ -21,6 +21,7 @@ use arrow::array::Int64Array;use arrow::datatypes::{DataType, Field, Int32Type, Int64Type};use arrow::util::bench_util::create_primitive_array;use criterion::{Criterion, SamplingMode, criterion_group, criterion_main};+use datafusion_common::ScalarValue;use datafusion_common::config::ConfigOptions;use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};use datafusion_functions::string;@@ -32,6 +33,42 @@ fn criterion_benchmark(c: &mut Criterion) {let hex = string::to_hex();let config_options = Arc::new(ConfigOptions::default());+ c.bench_function("to_hex/scalar_i32", |b| {+ let args = vec![ColumnarValue::Scalar(ScalarValue::Int32(Some(2147483647)))];+ let arg_fields = vec![Field::new("a", DataType::Int32, true).into()];+ b.iter(|| {+ black_box(+ hex.invoke_with_args(ScalarFunctionArgs {+ args: args.clone(),+ arg_fields: arg_fields.clone(),+ number_rows: 1,+ return_field: Field::new("f", DataType::Utf8, true).into(),+ config_options: Arc::clone(&config_options),+ })+ .unwrap(),+ )+ })+ });++ c.bench_function("to_hex/scalar_i64", |b| {+ let args = vec![ColumnarValue::Scalar(ScalarValue::Int64(Some(+ 9223372036854775807,+ )))];+ let arg_fields = vec![Field::new("a", DataType::Int64, true).into()];+ b.iter(|| {+ black_box(+ hex.invoke_with_args(ScalarFunctionArgs {+ args: args.clone(),+ arg_fields: arg_fields.clone(),+ number_rows: 1,+ return_field: Field::new("f", DataType::Utf8, true).into(),+ config_options: Arc::clone(&config_options),+ })+ .unwrap(),+ )+ })+ });+for size in [1024, 4096, 8192] {let mut group = c.benchmark_group(format!("to_hex size={size}"));group.sampling_mode(SamplingMode::Flat);diff --git a/datafusion/functions/src/string/to_hex.rs b/datafusion/functions/src/string/to_hex.rsindex 891cbe2549..ed8ce07b87 100644--- a/datafusion/functions/src/string/to_hex.rs+++ b/datafusion/functions/src/string/to_hex.rs@@ -18,7 +18,6 @@use std::any::Any;use std::sync::Arc;-use crate::utils::make_scalar_function;use arrow::array::{Array, ArrayRef, StringArray};use arrow::buffer::{Buffer, OffsetBuffer};use arrow::datatypes::{@@ -26,7 +25,7 @@ use arrow::datatypes::{Int64Type, UInt8Type, UInt16Type, UInt32Type, UInt64Type,};use datafusion_common::cast::as_primitive_array;-use datafusion_common::{Result, ScalarValue, exec_err};+use datafusion_common::{Result, ScalarValue, exec_err, internal_err};use datafusion_expr::{Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,TypeSignatureClass, Volatility,@@ -38,11 +37,11 @@ const HEX_CHARS: &[u8; 16] = b"0123456789abcdef";/// Converts the number to its equivalent hexadecimal representation./// to_hex(2147483647) = '7fffffff'-fn to_hex<T: ArrowPrimitiveType>(args: &[ArrayRef]) -> Result<ArrayRef>+fn to_hex_array<T: ArrowPrimitiveType>(array: &ArrayRef) -> Result<ArrayRef>whereT::Native: ToHex,{- let integer_array = as_primitive_array::<T>(&args[0])?;+ let integer_array = as_primitive_array::<T>(array)?;let len = integer_array.len();// Max hex string length: 16 chars for u64/i64@@ -78,6 +77,14 @@ whereOk(Arc::new(result) as ArrayRef)}+#[inline]+fn to_hex_scalar<T: ToHex>(value: T) -> String {+ let mut hex_buffer = [0u8; 16];+ let hex_len = value.write_hex_to_buffer(&mut hex_buffer);+ // SAFETY: hex_buffer is ASCII hex digits+ unsafe { std::str::from_utf8_unchecked(&hex_buffer[16 - hex_len..]).to_string() }+}+/// Trait for converting integer types to hexadecimal in a buffertrait ToHex: ArrowNativeType {/// Write hex representation to buffer and return the number of hex digits written.@@ -223,33 +230,71 @@ impl ScalarUDFImpl for ToHexFunc {}fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {- match args.args[0].data_type() {- DataType::Null => Ok(ColumnarValue::Scalar(ScalarValue::Utf8(None))),- DataType::Int64 => {- make_scalar_function(to_hex::<Int64Type>, vec![])(&args.args)- }- DataType::UInt64 => {- make_scalar_function(to_hex::<UInt64Type>, vec![])(&args.args)- }- DataType::Int32 => {- make_scalar_function(to_hex::<Int32Type>, vec![])(&args.args)- }- DataType::UInt32 => {- make_scalar_function(to_hex::<UInt32Type>, vec![])(&args.args)- }- DataType::Int16 => {- make_scalar_function(to_hex::<Int16Type>, vec![])(&args.args)- }- DataType::UInt16 => {- make_scalar_function(to_hex::<UInt16Type>, vec![])(&args.args)- }- DataType::Int8 => {- make_scalar_function(to_hex::<Int8Type>, vec![])(&args.args)- }- DataType::UInt8 => {- make_scalar_function(to_hex::<UInt8Type>, vec![])(&args.args)+ let arg = &args.args[0];++ match arg {+ ColumnarValue::Scalar(ScalarValue::Int64(Some(v))) => Ok(+ ColumnarValue::Scalar(ScalarValue::Utf8(Some(to_hex_scalar(*v)))),+ ),+ ColumnarValue::Scalar(ScalarValue::UInt64(Some(v))) => Ok(+ ColumnarValue::Scalar(ScalarValue::Utf8(Some(to_hex_scalar(*v)))),+ ),+ ColumnarValue::Scalar(ScalarValue::Int32(Some(v))) => Ok(+ ColumnarValue::Scalar(ScalarValue::Utf8(Some(to_hex_scalar(*v)))),+ ),+ ColumnarValue::Scalar(ScalarValue::UInt32(Some(v))) => Ok(+ ColumnarValue::Scalar(ScalarValue::Utf8(Some(to_hex_scalar(*v)))),+ ),+ ColumnarValue::Scalar(ScalarValue::Int16(Some(v))) => Ok(+ ColumnarValue::Scalar(ScalarValue::Utf8(Some(to_hex_scalar(*v)))),+ ),+ ColumnarValue::Scalar(ScalarValue::UInt16(Some(v))) => Ok(+ ColumnarValue::Scalar(ScalarValue::Utf8(Some(to_hex_scalar(*v)))),+ ),+ ColumnarValue::Scalar(ScalarValue::Int8(Some(v))) => Ok(+ ColumnarValue::Scalar(ScalarValue::Utf8(Some(to_hex_scalar(*v)))),+ ),+ ColumnarValue::Scalar(ScalarValue::UInt8(Some(v))) => Ok(+ ColumnarValue::Scalar(ScalarValue::Utf8(Some(to_hex_scalar(*v)))),+ ),++ // NULL scalars+ ColumnarValue::Scalar(s) if s.is_null() => {+ Ok(ColumnarValue::Scalar(ScalarValue::Utf8(None)))}- other => exec_err!("Unsupported data type {other:?} for function to_hex"),++ ColumnarValue::Array(array) => match array.data_type() {+ DataType::Int64 => {+ Ok(ColumnarValue::Array(to_hex_array::<Int64Type>(array)?))+ }+ DataType::UInt64 => {+ Ok(ColumnarValue::Array(to_hex_array::<UInt64Type>(array)?))+ }+ DataType::Int32 => {+ Ok(ColumnarValue::Array(to_hex_array::<Int32Type>(array)?))+ }+ DataType::UInt32 => {+ Ok(ColumnarValue::Array(to_hex_array::<UInt32Type>(array)?))+ }+ DataType::Int16 => {+ Ok(ColumnarValue::Array(to_hex_array::<Int16Type>(array)?))+ }+ DataType::UInt16 => {+ Ok(ColumnarValue::Array(to_hex_array::<UInt16Type>(array)?))+ }+ DataType::Int8 => {+ Ok(ColumnarValue::Array(to_hex_array::<Int8Type>(array)?))+ }+ DataType::UInt8 => {+ Ok(ColumnarValue::Array(to_hex_array::<UInt8Type>(array)?))+ }+ other => exec_err!("Unsupported data type {other:?} for function to_hex"),+ },++ other => internal_err!(+ "Unexpected argument type {:?} for function to_hex",+ other.data_type()+ ),}}@@ -288,8 +333,8 @@ mod tests {let expected = $expected;let array = <$array_type>::from(input);- let array_ref = Arc::new(array);- let hex_result = to_hex::<$arrow_type>(&[array_ref])?;+ let array_ref: ArrayRef = Arc::new(array);+ let hex_result = to_hex_array::<$arrow_type>(&array_ref)?;let hex_array = as_string_array(&hex_result)?;let expected_array = StringArray::from(expected);diff --git a/datafusion/sqllogictest/test_files/expr.slt b/datafusion/sqllogictest/test_files/expr.sltindex 90fe05815f..4e078d1e69 100644--- a/datafusion/sqllogictest/test_files/expr.slt+++ b/datafusion/sqllogictest/test_files/expr.slt@@ -725,6 +725,27 @@ SELECT to_hex(CAST(NULL AS int))----NULL+query T+SELECT to_hex(0)+----+0++# negative values (two's complement encoding)+query T+SELECT to_hex(-1)+----+ffffffffffffffff++query T+SELECT to_hex(CAST(-1 AS INT))+----+ffffffffffffffff++query T+SELECT to_hex(CAST(255 AS TINYINT UNSIGNED))+----+ff+query TSELECT trim(' tom ')----