Apache DataFusion SQL Query Engine
refactor: share hex encoding across datafusion-common, functions, and spark (#23766)
Andy Grove8393fd32b715
diff --git a/datafusion/common/src/utils/hex.rs b/datafusion/common/src/utils/hex.rsnew file mode 100644index 0000000000..6d0811350e--- /dev/null+++ b/datafusion/common/src/utils/hex.rs@@ -0,0 +1,357 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Hex encoding of bytes and integers.+//!+//! [`encode_bytes`] and [`encode_bytes_into`] encode a byte slice into an+//! owned `String` or an appended `Vec<u8>`, respectively; [`encode_bytes_to_slice`]+//! writes into a caller-provided, pre-sized buffer. [`encode_u64`] encodes an+//! integer, trimming leading zeros. All four take a [`HexCase`] to choose+//! between lowercase and uppercase digits.++use crate::Result;+use crate::error::_internal_err;++/// Case of the emitted hex digits.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum HexCase {+ /// Digits `0123456789abcdef`.+ Lower,+ /// Digits `0123456789ABCDEF`.+ Upper,+}++const LOWER_DIGITS: &[u8; 16] = b"0123456789abcdef";+const UPPER_DIGITS: &[u8; 16] = b"0123456789ABCDEF";++/// Maps a full byte to its two hex digits, so encoding advances a whole byte+/// per iteration instead of a nibble.+const LOOKUP_LOWER: [[u8; 2]; 256] = build_lookup(LOWER_DIGITS);+const LOOKUP_UPPER: [[u8; 2]; 256] = build_lookup(UPPER_DIGITS);++const fn build_lookup(digits: &[u8; 16]) -> [[u8; 2]; 256] {+ let mut table = [[0u8; 2]; 256];+ let mut i = 0;+ while i < 256 {+ table[i][0] = digits[i >> 4];+ table[i][1] = digits[i & 0xF];+ i += 1;+ }+ table+}++impl HexCase {+ #[inline]+ const fn lookup(self) -> &'static [[u8; 2]; 256] {+ match self {+ HexCase::Lower => &LOOKUP_LOWER,+ HexCase::Upper => &LOOKUP_UPPER,+ }+ }++ #[inline]+ const fn digits(self) -> &'static [u8; 16] {+ match self {+ HexCase::Lower => LOWER_DIGITS,+ HexCase::Upper => UPPER_DIGITS,+ }+ }+}++/// Appends the hex encoding of `bytes` to `out`.+///+/// Allocates only through `out`'s own growth. Callers that must bound or guard+/// that growth should reserve capacity in `out` before calling.+#[inline(always)]+pub fn encode_bytes_into(bytes: &[u8], case: HexCase, out: &mut Vec<u8>) {+ let lookup = case.lookup();+ for &byte in bytes {+ out.extend_from_slice(&lookup[byte as usize]);+ }+}++/// Writes the hex encoding of `bytes` into `out`.+///+/// This is for callers that already own a pre-sized buffer (for example a+/// slice of a larger, pre-allocated output array) and want to write directly+/// into it rather than appending to a `Vec`.+///+/// Returns an internal error if `out` is not exactly `2 * bytes.len()` bytes+/// long, without filling any of the `out` buffer.+///+/// # Example+///+/// ```+/// use datafusion_common::utils::hex::{HexCase, encode_bytes_to_slice};+///+/// let mut out = [0u8; 8];+/// encode_bytes_to_slice(&[0xde, 0xad, 0xbe, 0xef], HexCase::Lower, &mut out)?;+/// assert_eq!(&out, b"deadbeef");+/// # Ok::<(), datafusion_common::DataFusionError>(())+/// ```+#[inline(always)]+pub fn encode_bytes_to_slice(bytes: &[u8], case: HexCase, out: &mut [u8]) -> Result<()> {+ let expected = bytes.len() * 2;+ if out.len() != expected {+ return _internal_err!(+ "hex output buffer is {} bytes, expected {expected}",+ out.len()+ );+ }+ let lookup = case.lookup();+ for (&b, chunk) in bytes.iter().zip(out.chunks_exact_mut(2)) {+ chunk.copy_from_slice(&lookup[b as usize]);+ }+ Ok(())+}++/// Returns the hex encoding of `bytes` as an owned `String`.+///+/// # Example+///+/// ```+/// use datafusion_common::utils::hex::{HexCase, encode_bytes};+///+/// assert_eq!(encode_bytes(&[0xde, 0xad, 0xbe, 0xef], HexCase::Lower), "deadbeef");+/// assert_eq!(encode_bytes(&[0xde, 0xad, 0xbe, 0xef], HexCase::Upper), "DEADBEEF");+/// ```+#[inline]+pub fn encode_bytes(bytes: &[u8], case: HexCase) -> String {+ let mut out = Vec::with_capacity(bytes.len() * 2);+ encode_bytes_into(bytes, case, &mut out);+ // SAFETY: `out` holds only ASCII hex digits, which are valid UTF-8.+ unsafe { String::from_utf8_unchecked(out) }+}++/// Writes `v` as hex into `buf` and returns the written subslice.+///+/// Digits are written right-aligned with leading zeros trimmed, so the result+/// borrows the tail of `buf`. Zero encodes as `"0"`.+///+/// Signed values should be cast with `as u64`, which yields the two's+/// complement representation that both `to_hex` and Spark's `hex` produce for+/// negative input.+///+/// # Example+///+/// The caller owns the buffer and can reuse it across calls; each call+/// returns a fresh subslice of it, borrowed for as long as `buf` is:+///+/// ```+/// use datafusion_common::utils::hex::{HexCase, encode_u64};+///+/// let mut buf = [0u8; 16];+/// assert_eq!(encode_u64(0xAB, HexCase::Lower, &mut buf), b"ab");+/// assert_eq!(encode_u64(0, HexCase::Lower, &mut buf), b"0");+/// ```+#[inline(always)]+pub fn encode_u64(v: u64, case: HexCase, buf: &mut [u8; 16]) -> &[u8] {+ let start = write_digits(v, case, buf);+ &buf[start..]+}++/// Writes the digits of `v` right-aligned in `buf`, returning the index of the+/// first digit.+///+/// Split out from [`encode_u64`] so the mutable borrow of `buf` ends before the+/// returned slice reborrows it.+#[inline(always)]+fn write_digits(v: u64, case: HexCase, buf: &mut [u8; 16]) -> usize {+ if v == 0 {+ buf[15] = b'0';+ return 15;+ }++ // Consume two nibbles (one full byte) per iteration.+ let lookup = case.lookup();+ let mut pos = 16;+ let mut rest = v;+ while rest >= 0x10 {+ pos -= 2;+ let pair = lookup[(rest & 0xFF) as usize];+ buf[pos] = pair[0];+ buf[pos + 1] = pair[1];+ rest >>= 8;+ }+ if rest > 0 {+ // A single high nibble (0x1..=0xF) remains.+ pos -= 1;+ buf[pos] = case.digits()[rest as usize];+ }++ pos+}++#[cfg(test)]+mod tests {+ use super::*;++ fn hex_u64(v: u64, case: HexCase) -> String {+ let mut buf = [0u8; 16];+ String::from_utf8(encode_u64(v, case, &mut buf).to_vec()).unwrap()+ }++ #[test]+ fn encode_u64_zero() {+ assert_eq!(hex_u64(0, HexCase::Lower), "0");+ assert_eq!(hex_u64(0, HexCase::Upper), "0");+ }++ #[test]+ fn encode_u64_single_nibble() {+ for v in 1..=0xFu64 {+ assert_eq!(hex_u64(v, HexCase::Lower), format!("{v:x}"));+ assert_eq!(hex_u64(v, HexCase::Upper), format!("{v:X}"));+ }+ }++ #[test]+ fn encode_u64_digit_count_boundaries() {+ // Straddle each odd/even digit-count boundary: the two-nibbles-per+ // iteration loop plus the trailing single-nibble fixup.+ for v in [+ 0x10u64,+ 0xFF,+ 0x100,+ 0xFFF,+ 0x1000,+ 0xFFFFF,+ 0xFFFF_FFFF,+ 0x1_0000_0000,+ ] {+ assert_eq!(hex_u64(v, HexCase::Lower), format!("{v:x}"));+ assert_eq!(hex_u64(v, HexCase::Upper), format!("{v:X}"));+ }+ }++ #[test]+ fn encode_u64_max() {+ assert_eq!(hex_u64(u64::MAX, HexCase::Lower), "ffffffffffffffff");+ assert_eq!(hex_u64(u64::MAX, HexCase::Upper), "FFFFFFFFFFFFFFFF");+ }++ #[test]+ fn encode_u64_signed_is_twos_complement() {+ // Callers cast signed values with `as u64`; this is the behaviour both+ // `to_hex` and Spark `hex` rely on for negative input.+ assert_eq!(hex_u64(-1i64 as u64, HexCase::Lower), "ffffffffffffffff");+ assert_eq!(hex_u64(i64::MIN as u64, HexCase::Upper), "8000000000000000");+ }++ #[test]+ fn encode_bytes_empty() {+ assert_eq!(encode_bytes(&[], HexCase::Lower), "");+ assert_eq!(encode_bytes(&[], HexCase::Upper), "");+ }++ #[test]+ fn encode_bytes_examples() {+ assert_eq!(encode_bytes(&[0x00], HexCase::Lower), "00");+ assert_eq!(encode_bytes(&[0xAB], HexCase::Lower), "ab");+ assert_eq!(encode_bytes(&[0xAB], HexCase::Upper), "AB");+ assert_eq!(+ encode_bytes(&[0xde, 0xad, 0xbe, 0xef], HexCase::Lower),+ "deadbeef"+ );+ assert_eq!(+ encode_bytes(&[0xde, 0xad, 0xbe, 0xef], HexCase::Upper),+ "DEADBEEF"+ );+ }++ #[test]+ fn encode_bytes_covers_every_byte_value() {+ let bytes: Vec<u8> = (0..=255u8).collect();++ let expected: String = bytes.iter().map(|b| format!("{b:02x}")).collect();+ assert_eq!(encode_bytes(&bytes, HexCase::Lower), expected);++ let expected: String = bytes.iter().map(|b| format!("{b:02X}")).collect();+ assert_eq!(encode_bytes(&bytes, HexCase::Upper), expected);+ }++ #[test]+ fn encode_bytes_into_appends_without_clearing() {+ let mut out = b"prefix-".to_vec();+ encode_bytes_into(&[0x01, 0x02], HexCase::Lower, &mut out);+ assert_eq!(out, b"prefix-0102");+ }++ #[test]+ fn encode_u64_reused_buffer_leaks_no_stale_digits() {+ let mut buf = [0u8; 16];+ assert_eq!(+ encode_u64(u64::MAX, HexCase::Lower, &mut buf),+ b"ffffffffffffffff"+ );+ assert_eq!(encode_u64(0, HexCase::Lower, &mut buf), b"0");+ assert_eq!(encode_u64(0xAB, HexCase::Lower, &mut buf), b"ab");+ }++ #[test]+ fn encode_bytes_to_slice_empty() -> Result<()> {+ let mut out: [u8; 0] = [];+ encode_bytes_to_slice(&[], HexCase::Lower, &mut out)?;+ assert_eq!(out, [] as [u8; 0]);+ Ok(())+ }++ #[test]+ fn encode_bytes_to_slice_examples() -> Result<()> {+ let mut out = [0u8; 8];+ encode_bytes_to_slice(&[0xde, 0xad, 0xbe, 0xef], HexCase::Lower, &mut out)?;+ assert_eq!(&out, b"deadbeef");++ let mut out = [0u8; 8];+ encode_bytes_to_slice(&[0xde, 0xad, 0xbe, 0xef], HexCase::Upper, &mut out)?;+ assert_eq!(&out, b"DEADBEEF");+ Ok(())+ }++ #[test]+ fn encode_bytes_to_slice_agrees_with_encode_bytes() -> Result<()> {+ let bytes: Vec<u8> = (0..=255u8).collect();+ for case in [HexCase::Lower, HexCase::Upper] {+ let mut out = vec![0u8; bytes.len() * 2];+ encode_bytes_to_slice(&bytes, case, &mut out)?;+ assert_eq!(String::from_utf8(out).unwrap(), encode_bytes(&bytes, case));+ }+ Ok(())+ }++ #[test]+ fn encode_bytes_to_slice_rejects_wrong_length() {+ // Too short: the old `debug_assert` let release builds silently drop+ // the remaining input.+ let mut short = [0u8; 6];+ let err =+ encode_bytes_to_slice(&[0xde, 0xad, 0xbe, 0xef], HexCase::Lower, &mut short)+ .unwrap_err();+ assert!(+ err.message()+ .contains("hex output buffer is 6 bytes, expected 8"),+ "unexpected message: {err}"+ );++ // Too long: would have left stale bytes at the tail.+ let mut long = [0u8; 10];+ assert!(+ encode_bytes_to_slice(&[0xde, 0xad, 0xbe, 0xef], HexCase::Lower, &mut long)+ .is_err()+ );+ }+}diff --git a/datafusion/common/src/utils/mod.rs b/datafusion/common/src/utils/mod.rsindex 94bbb91a7f..73772b3193 100644--- a/datafusion/common/src/utils/mod.rs+++ b/datafusion/common/src/utils/mod.rs@@ -19,6 +19,7 @@pub(crate) mod aggregate;pub mod expr;+pub mod hex;pub mod memory;pub mod proxy;pub mod string_utils;diff --git a/datafusion/functions/src/crypto/md5.rs b/datafusion/functions/src/crypto/md5.rsindex 178aebf0fb..b1206d2e42 100644--- a/datafusion/functions/src/crypto/md5.rs+++ b/datafusion/functions/src/crypto/md5.rs@@ -21,6 +21,7 @@ use datafusion_common::{cast::as_binary_array,internal_err,types::{logical_binary, logical_string},+ utils::hex::{HexCase, encode_bytes},utils::take_function_args,};use datafusion_expr::{@@ -98,22 +99,6 @@ impl ScalarUDFImpl for Md5Func {}}-/// Hex encoding lookup table for fast byte-to-hex conversion-const HEX_CHARS_LOWER: &[u8; 16] = b"0123456789abcdef";--/// Fast hex encoding using a lookup table instead of format strings.-/// This is significantly faster than using `write!("{:02x}")` for each byte.-#[inline]-fn hex_encode(data: impl AsRef<[u8]>) -> String {- let bytes = data.as_ref();- let mut s = String::with_capacity(bytes.len() * 2);- for &b in bytes {- s.push(HEX_CHARS_LOWER[(b >> 4) as usize] as char);- s.push(HEX_CHARS_LOWER[(b & 0x0f) as usize] as char);- }- s-}-fn md5(args: &[ColumnarValue]) -> Result<ColumnarValue> {let [data] = take_function_args("md5", args)?;let value = digest_process(data, DigestAlgorithm::Md5)?;@@ -122,13 +107,15 @@ fn md5(args: &[ColumnarValue]) -> Result<ColumnarValue> {Ok(match value {ColumnarValue::Array(array) => {let binary_array = as_binary_array(&array)?;- let string_array: StringViewArray =- binary_array.iter().map(|opt| opt.map(hex_encode)).collect();+ let string_array: StringViewArray = binary_array+ .iter()+ .map(|opt| opt.map(|b| encode_bytes(b, HexCase::Lower)))+ .collect();ColumnarValue::Array(Arc::new(string_array))}- ColumnarValue::Scalar(ScalarValue::Binary(opt)) => {- ColumnarValue::Scalar(ScalarValue::Utf8View(opt.map(hex_encode)))- }+ ColumnarValue::Scalar(ScalarValue::Binary(opt)) => ColumnarValue::Scalar(+ ScalarValue::Utf8View(opt.map(|b| encode_bytes(&b, HexCase::Lower))),+ ),_ => return internal_err!("Impossibly got invalid results from digest"),})}diff --git a/datafusion/functions/src/encoding/inner.rs b/datafusion/functions/src/encoding/inner.rsindex 5d4740d80b..850e312abd 100644--- a/datafusion/functions/src/encoding/inner.rs+++ b/datafusion/functions/src/encoding/inner.rs@@ -33,7 +33,10 @@ use datafusion_common::{DataFusionError, Result, ScalarValue, exec_datafusion_err, exec_err, internal_err,not_impl_err, plan_err,types::{NativeType, logical_string},- utils::take_function_args,+ utils::{+ hex::{HexCase, encode_bytes as encode_hex, encode_bytes_to_slice},+ take_function_args,+ },};use datafusion_expr::{Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,@@ -370,7 +373,7 @@ impl Encoding {match self {Self::Base64 => BASE64_ENGINE.encode(value),Self::Base64Padded => BASE64_ENGINE_PADDED.encode(value),- Self::Hex => hex::encode(value),+ Self::Hex => encode_hex(value, HexCase::Lower),}}@@ -477,11 +480,7 @@ wherefor v in array.iter() {if let Some(v) = v {let out_len = v.len() * 2;- // The slice is sized to exactly `2 * v.len()`, which is the only- // condition under which `encode_to_slice` can fail, so this cannot- // error.- hex::encode_to_slice(v, &mut values[pos..pos + out_len])- .map_err(|e| exec_datafusion_err!("Failed to encode to hex: {e}"))?;+ encode_bytes_to_slice(v, HexCase::Lower, &mut values[pos..pos + out_len])?;pos += out_len;}offsets.push(OutputOffset::usize_as(pos));diff --git a/datafusion/functions/src/string/to_hex.rs b/datafusion/functions/src/string/to_hex.rsindex 497a0a1206..a6bcd17966 100644--- a/datafusion/functions/src/string/to_hex.rs+++ b/datafusion/functions/src/string/to_hex.rs@@ -24,6 +24,7 @@ use arrow::datatypes::{Int64Type, UInt8Type, UInt16Type, UInt32Type, UInt64Type,};use datafusion_common::cast::as_primitive_array;+use datafusion_common::utils::hex::{HexCase, encode_u64};use datafusion_common::{Result, ScalarValue, exec_err, internal_err};use datafusion_expr::{Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,@@ -31,9 +32,6 @@ use datafusion_expr::{};use datafusion_macros::user_doc;-/// Hex lookup table for fast conversion-const HEX_CHARS: &[u8; 16] = b"0123456789abcdef";-/// Converts the number to its equivalent hexadecimal representation./// to_hex(2147483647) = '7fffffff'fn to_hex_array<T: ArrowPrimitiveType>(array: &ArrayRef) -> Result<ArrayRef>@@ -59,8 +57,7 @@ where// Process all values directly (including null slots - we write empty strings for nulls)// The null bitmap will mark which entries are actually nullfor value in integer_array.values() {- let hex_len = value.write_hex_to_buffer(&mut hex_buffer);- values.extend_from_slice(&hex_buffer[16 - hex_len..]);+ values.extend_from_slice(value.write_hex(&mut hex_buffer));offsets.push(values.len() as i32);}@@ -79,101 +76,50 @@ where#[inline]fn to_hex_scalar<T: ToHex>(value: T) -> String {let mut hex_buffer = [0u8; 16];- let hex_len = value.write_hex_to_buffer(&mut hex_buffer);- // SAFETY: hex_buffer is ASCII hex digits- unsafe { std::str::from_utf8_unchecked(&hex_buffer[16 - hex_len..]).to_string() }+ let hex = value.write_hex(&mut hex_buffer);+ // SAFETY: hex holds only ASCII hex digits.+ unsafe { std::str::from_utf8_unchecked(hex).to_string() }}/// Trait for converting integer types to hexadecimal in a buffertrait ToHex: ArrowNativeType {- /// Write hex representation to buffer and return the number of hex digits written.- /// The hex digits are written right-aligned in the buffer (starting from position 16 - len).- fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize;-}--/// Write unsigned value to hex buffer and return the number of digits written.-/// Digits are written right-aligned in the buffer.-#[inline]-fn write_unsigned_hex_to_buffer(value: u64, buffer: &mut [u8; 16]) -> usize {- if value == 0 {- buffer[15] = b'0';- return 1;- }-- // Write hex digits from right to left- let mut pos = 16;- let mut v = value;- while v > 0 {- pos -= 1;- buffer[pos] = HEX_CHARS[(v & 0xf) as usize];- v >>= 4;- }-- 16 - pos-}--/// Write signed value to hex buffer (two's complement for negative) and return digit count-#[inline]-fn write_signed_hex_to_buffer(value: i64, buffer: &mut [u8; 16]) -> usize {- // For negative values, use two's complement representation (same as casting to u64)- write_unsigned_hex_to_buffer(value as u64, buffer)-}--impl ToHex for i8 {- #[inline]- fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize {- write_signed_hex_to_buffer(self as i64, buffer)- }-}--impl ToHex for i16 {- #[inline]- fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize {- write_signed_hex_to_buffer(self as i64, buffer)- }-}--impl ToHex for i32 {- #[inline]- fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize {- write_signed_hex_to_buffer(self as i64, buffer)- }-}--impl ToHex for i64 {- #[inline]- fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize {- write_signed_hex_to_buffer(self, buffer)- }+ /// Writes the hex representation into `buf` and returns the written+ /// subslice. Digits are right-aligned in `buf` with leading zeros trimmed.+ fn write_hex(self, buf: &mut [u8; 16]) -> &[u8];}-impl ToHex for u8 {- #[inline]- fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize {- write_unsigned_hex_to_buffer(self as u64, buffer)- }-}--impl ToHex for u16 {- #[inline]- fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize {- write_unsigned_hex_to_buffer(self as u64, buffer)- }+/// Signed values use their two's complement representation, matching a cast to+/// the corresponding unsigned type.+macro_rules! impl_to_hex_signed {+ ($ty:ty) => {+ impl ToHex for $ty {+ #[inline]+ fn write_hex(self, buf: &mut [u8; 16]) -> &[u8] {+ encode_u64(self as i64 as u64, HexCase::Lower, buf)+ }+ }+ };}-impl ToHex for u32 {- #[inline]- fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize {- write_unsigned_hex_to_buffer(self as u64, buffer)- }+macro_rules! impl_to_hex_unsigned {+ ($ty:ty) => {+ impl ToHex for $ty {+ #[inline]+ fn write_hex(self, buf: &mut [u8; 16]) -> &[u8] {+ encode_u64(self as u64, HexCase::Lower, buf)+ }+ }+ };}-impl ToHex for u64 {- #[inline]- fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize {- write_unsigned_hex_to_buffer(self, buffer)- }-}+impl_to_hex_signed!(i8);+impl_to_hex_signed!(i16);+impl_to_hex_signed!(i32);+impl_to_hex_signed!(i64);+impl_to_hex_unsigned!(u8);+impl_to_hex_unsigned!(u16);+impl_to_hex_unsigned!(u32);+impl_to_hex_unsigned!(u64);#[user_doc(doc_section(label = "String Functions"),diff --git a/datafusion/spark/src/function/hash/sha1.rs b/datafusion/spark/src/function/hash/sha1.rsindex dd9009eb82..05a224f33f 100644--- a/datafusion/spark/src/function/hash/sha1.rs+++ b/datafusion/spark/src/function/hash/sha1.rs@@ -24,6 +24,7 @@ use datafusion_common::cast::{as_large_binary_array,};use datafusion_common::types::{NativeType, logical_string};+use datafusion_common::utils::hex::{HexCase, encode_bytes};use datafusion_common::utils::take_function_args;use datafusion_common::{Result, internal_err};use datafusion_expr::{@@ -89,18 +90,9 @@ impl ScalarUDFImpl for SparkSha1 {}}-/// Hex encoding lookup table for fast byte-to-hex conversion-const HEX_CHARS_LOWER: &[u8; 16] = b"0123456789abcdef";-#[inline]fn spark_sha1_digest(value: &[u8]) -> String {- let result = Sha1::digest(value);- let mut s = String::with_capacity(result.len() * 2);- for &b in result.as_slice() {- s.push(HEX_CHARS_LOWER[(b >> 4) as usize] as char);- s.push(HEX_CHARS_LOWER[(b & 0x0f) as usize] as char);- }- s+ encode_bytes(&Sha1::digest(value), HexCase::Lower)}fn spark_sha1_impl<'a>(input: impl Iterator<Item = Option<&'a [u8]>>) -> ArrayRef {diff --git a/datafusion/spark/src/function/hash/sha2.rs b/datafusion/spark/src/function/hash/sha2.rsindex 38fa0cc643..541df29576 100644--- a/datafusion/spark/src/function/hash/sha2.rs+++ b/datafusion/spark/src/function/hash/sha2.rs@@ -20,6 +20,7 @@ use arrow::datatypes::{DataType, Int32Type};use datafusion_common::types::{NativeType, logical_binary, logical_int32, logical_string,};+use datafusion_common::utils::hex::{HexCase, encode_bytes};use datafusion_common::utils::take_function_args;use datafusion_common::{Result, ScalarValue, internal_err};use datafusion_expr::{@@ -112,22 +113,22 @@ impl ScalarUDFImpl for SparkSha2 {224 => {let mut digest = sha2::Sha224::default();digest.update(bytes);- Some(hex_encode(digest.finalize()))+ Some(encode_bytes(&digest.finalize(), HexCase::Lower))}0 | 256 => {let mut digest = sha2::Sha256::default();digest.update(bytes);- Some(hex_encode(digest.finalize()))+ Some(encode_bytes(&digest.finalize(), HexCase::Lower))}384 => {let mut digest = sha2::Sha384::default();digest.update(bytes);- Some(hex_encode(digest.finalize()))+ Some(encode_bytes(&digest.finalize(), HexCase::Lower))}512 => {let mut digest = sha2::Sha512::default();digest.update(bytes);- Some(hex_encode(digest.finalize()))+ Some(encode_bytes(&digest.finalize(), HexCase::Lower))}_ => None,};@@ -222,22 +223,22 @@ where(Some(value), Some(224)) => {let mut digest = sha2::Sha224::default();digest.update(value);- Some(hex_encode(digest.finalize()))+ Some(encode_bytes(&digest.finalize(), HexCase::Lower))}(Some(value), Some(0 | 256)) => {let mut digest = sha2::Sha256::default();digest.update(value);- Some(hex_encode(digest.finalize()))+ Some(encode_bytes(&digest.finalize(), HexCase::Lower))}(Some(value), Some(384)) => {let mut digest = sha2::Sha384::default();digest.update(value);- Some(hex_encode(digest.finalize()))+ Some(encode_bytes(&digest.finalize(), HexCase::Lower))}(Some(value), Some(512)) => {let mut digest = sha2::Sha512::default();digest.update(value);- Some(hex_encode(digest.finalize()))+ Some(encode_bytes(&digest.finalize(), HexCase::Lower))}// Unknown bit-lengths go to null, same as in Spark_ => None,@@ -245,19 +246,3 @@ where.collect::<StringArray>();Arc::new(array)}--const HEX_CHARS: [u8; 16] = *b"0123456789abcdef";--#[inline]-fn hex_encode<T: AsRef<[u8]>>(data: T) -> String {- let bytes = data.as_ref();- let mut out = Vec::with_capacity(bytes.len() * 2);- for &b in bytes {- let hi = b >> 4;- let lo = b & 0x0F;- out.push(HEX_CHARS[hi as usize]);- out.push(HEX_CHARS[lo as usize]);- }- // SAFETY: out contains only ASCII- unsafe { String::from_utf8_unchecked(out) }-}diff --git a/datafusion/spark/src/function/math/hex.rs b/datafusion/spark/src/function/math/hex.rsindex a283bd8fa7..1f505fda21 100644--- a/datafusion/spark/src/function/math/hex.rs+++ b/datafusion/spark/src/function/math/hex.rs@@ -28,6 +28,7 @@ use arrow::{use datafusion_common::cast::as_large_binary_array;use datafusion_common::cast::as_string_view_array;use datafusion_common::types::{NativeType, logical_int64, logical_string};+use datafusion_common::utils::hex::{HexCase, encode_bytes_into, encode_u64};use datafusion_common::utils::take_function_args;use datafusion_common::{DataFusionError,@@ -110,54 +111,6 @@ impl ScalarUDFImpl for SparkHex {}}-/// Hex encoding lookup tables for fast byte-to-hex conversion.-///-/// Each entry maps a full byte to its two-character hex encoding so the-/// hot loop becomes one load + one two-byte extend per input byte instead-/// of two nibble lookups and two pushes.-const HEX_CHARS_UPPER_NIBBLES: &[u8; 16] = b"0123456789ABCDEF";-const HEX_CHARS_LOWER_NIBBLES: &[u8; 16] = b"0123456789abcdef";--const HEX_LOOKUP_UPPER: [[u8; 2]; 256] = build_hex_lookup(HEX_CHARS_UPPER_NIBBLES);-const HEX_LOOKUP_LOWER: [[u8; 2]; 256] = build_hex_lookup(HEX_CHARS_LOWER_NIBBLES);--const fn build_hex_lookup(nibbles: &[u8; 16]) -> [[u8; 2]; 256] {- let mut table = [[0u8; 2]; 256];- let mut i = 0;- while i < 256 {- table[i][0] = nibbles[(i >> 4) & 0xF];- table[i][1] = nibbles[i & 0xF];- i += 1;- }- table-}--#[inline]-fn hex_int64(num: i64, buffer: &mut [u8; 16]) -> &[u8] {- if num == 0 {- return b"0";- }-- // Walk the value two nibbles (one full byte) at a time. The buffer is- // filled from the right so the high-order nibbles end up first; the- // returned slice trims leading zeros automatically.- let mut n = num as u64;- let mut i = 16;- while n >= 0x10 {- i -= 2;- let pair = HEX_LOOKUP_UPPER[(n & 0xFF) as usize];- buffer[i] = pair[0];- buffer[i + 1] = pair[1];- n >>= 8;- }- if n > 0 {- // Single remaining high nibble (value 0x1..=0xF).- i -= 1;- buffer[i] = HEX_CHARS_UPPER_NIBBLES[n as usize];- }- &buffer[i..]-}-/// Generic hex encoding for byte array typesfn hex_encode_bytes<'a, I, T>(iter: I,@@ -168,10 +121,10 @@ whereI: Iterator<Item = Option<T>>,T: AsRef<[u8]> + 'a,{- let lookup = if lowercase {- &HEX_LOOKUP_LOWER+ let case = if lowercase {+ HexCase::Lower} else {- &HEX_LOOKUP_UPPER+ HexCase::Upper};// Write hex digits directly into one growing value buffer, tracking offsets@@ -195,9 +148,7 @@ where"failed to reserve {additional} bytes for hex output: {e}")})?;- for &byte in bytes {- values.extend_from_slice(&lookup[byte as usize]);- }+ encode_bytes_into(bytes, case, &mut values);nulls.append_non_null();} else {nulls.append_null();@@ -233,7 +184,7 @@ fn hex_encode_int64(for v in iter {if let Some(num) = v {let mut temp = [0u8; 16];- let slice = hex_int64(num, &mut temp);+ let slice = encode_u64(num as u64, HexCase::Upper, &mut temp);// SAFETY: slice contains only ASCII hex digests, which are valid UTF-8unsafe {builder.append_value(from_utf8_unchecked(slice));@@ -381,11 +332,10 @@ pub fn compute_hex(#[cfg(test)]mod test {- use std::str::from_utf8_unchecked;use std::sync::Arc;use arrow::array::{- BinaryArray, DictionaryArray, Int32Array, Int64Array, StringArray,+ Array, BinaryArray, DictionaryArray, Int32Array, Int64Array, StringArray,};use arrow::{array::{@@ -486,7 +436,7 @@ mod test {#[test]fn test_hex_int64() {- let test_cases = vec![+ let cases = vec![(0_i64, "0"),(1, "1"),(15, "F"),@@ -499,37 +449,28 @@ mod test {(-1, "FFFFFFFFFFFFFFFF"),];- for (num, expected) in test_cases {- let mut cache = [0u8; 16];- let slice = super::hex_int64(num, &mut cache);-- unsafe {- let result = from_utf8_unchecked(slice);- assert_eq!(expected, result, "hex_int64({num}) mismatch");- }+ let arr =+ super::hex_encode_int64(cases.iter().map(|(n, _)| Some(*n)), cases.len())+ .unwrap();+ let arr = as_string_array(&arr);+ for (i, (num, expected)) in cases.iter().enumerate() {+ assert_eq!(*expected, arr.value(i), "hex({num})");}}#[test]- fn test_hex_lookup_table_covers_all_bytes() {- // Cross-check the precomputed table against an independent encoder- // for every possible byte value and both casings.- for byte in 0u8..=255 {- let upper = format!("{byte:02X}");- let lower = format!("{byte:02x}");- let upper_pair = super::HEX_LOOKUP_UPPER[byte as usize];- let lower_pair = super::HEX_LOOKUP_LOWER[byte as usize];- assert_eq!(- upper.as_bytes(),- &upper_pair,- "upper encoding mismatch for byte 0x{byte:02X}"- );- assert_eq!(- lower.as_bytes(),- &lower_pair,- "lower encoding mismatch for byte 0x{byte:02X}"- );- }+ fn test_hex_encode_bytes_lowercase() {+ // Every in-repo caller of `hex_encode_bytes` goes through `spark_hex`,+ // which always passes `lowercase = false`. The `lowercase = true` path+ // is reachable only via `spark_sha2_hex`, which has no in-workspace+ // caller, so it otherwise has no coverage. Drive it directly here.+ let input = StringArray::from(vec![Some("hi"), Some("bye"), None, Some("rust")]);+ let result = super::hex_encode_bytes(input.iter(), true, input.len()).unwrap();+ let result = as_string_array(&result);++ let expected =+ StringArray::from(vec![Some("6869"), Some("627965"), None, Some("72757374")]);+ assert_eq!(result, &expected);}#[test]