Skip to content

Apache DataFusion SQL Query Engine

refactor: share hex encoding across datafusion-common, functions, and spark (#23766)

Andy Grove8393fd32b715
diff --git a/datafusion/common/src/utils/hex.rs b/datafusion/common/src/utils/hex.rs
new file mode 100644
index 0000000000..6d0811350e
--- /dev/null
+++ b/datafusion/common/src/utils/hex.rs
@@ -0,0 +1,357 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Hex encoding of bytes and integers.
+//!
+//! [`encode_bytes`] and [`encode_bytes_into`] encode a byte slice into an
+//! owned `String` or an appended `Vec<u8>`, respectively; [`encode_bytes_to_slice`]
+//! writes into a caller-provided, pre-sized buffer. [`encode_u64`] encodes an
+//! integer, trimming leading zeros. All four take a [`HexCase`] to choose
+//! between lowercase and uppercase digits.
+
+use crate::Result;
+use crate::error::_internal_err;
+
+/// Case of the emitted hex digits.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum HexCase {
+ /// Digits `0123456789abcdef`.
+ Lower,
+ /// Digits `0123456789ABCDEF`.
+ Upper,
+}
+
+const LOWER_DIGITS: &[u8; 16] = b"0123456789abcdef";
+const UPPER_DIGITS: &[u8; 16] = b"0123456789ABCDEF";
+
+/// Maps a full byte to its two hex digits, so encoding advances a whole byte
+/// per iteration instead of a nibble.
+const LOOKUP_LOWER: [[u8; 2]; 256] = build_lookup(LOWER_DIGITS);
+const LOOKUP_UPPER: [[u8; 2]; 256] = build_lookup(UPPER_DIGITS);
+
+const fn build_lookup(digits: &[u8; 16]) -> [[u8; 2]; 256] {
+ let mut table = [[0u8; 2]; 256];
+ let mut i = 0;
+ while i < 256 {
+ table[i][0] = digits[i >> 4];
+ table[i][1] = digits[i & 0xF];
+ i += 1;
+ }
+ table
+}
+
+impl HexCase {
+ #[inline]
+ const fn lookup(self) -> &'static [[u8; 2]; 256] {
+ match self {
+ HexCase::Lower => &LOOKUP_LOWER,
+ HexCase::Upper => &LOOKUP_UPPER,
+ }
+ }
+
+ #[inline]
+ const fn digits(self) -> &'static [u8; 16] {
+ match self {
+ HexCase::Lower => LOWER_DIGITS,
+ HexCase::Upper => UPPER_DIGITS,
+ }
+ }
+}
+
+/// Appends the hex encoding of `bytes` to `out`.
+///
+/// Allocates only through `out`'s own growth. Callers that must bound or guard
+/// that growth should reserve capacity in `out` before calling.
+#[inline(always)]
+pub fn encode_bytes_into(bytes: &[u8], case: HexCase, out: &mut Vec<u8>) {
+ let lookup = case.lookup();
+ for &byte in bytes {
+ out.extend_from_slice(&lookup[byte as usize]);
+ }
+}
+
+/// Writes the hex encoding of `bytes` into `out`.
+///
+/// This is for callers that already own a pre-sized buffer (for example a
+/// slice of a larger, pre-allocated output array) and want to write directly
+/// into it rather than appending to a `Vec`.
+///
+/// Returns an internal error if `out` is not exactly `2 * bytes.len()` bytes
+/// long, without filling any of the `out` buffer.
+///
+/// # Example
+///
+/// ```
+/// use datafusion_common::utils::hex::{HexCase, encode_bytes_to_slice};
+///
+/// let mut out = [0u8; 8];
+/// encode_bytes_to_slice(&[0xde, 0xad, 0xbe, 0xef], HexCase::Lower, &mut out)?;
+/// assert_eq!(&out, b"deadbeef");
+/// # Ok::<(), datafusion_common::DataFusionError>(())
+/// ```
+#[inline(always)]
+pub fn encode_bytes_to_slice(bytes: &[u8], case: HexCase, out: &mut [u8]) -> Result<()> {
+ let expected = bytes.len() * 2;
+ if out.len() != expected {
+ return _internal_err!(
+ "hex output buffer is {} bytes, expected {expected}",
+ out.len()
+ );
+ }
+ let lookup = case.lookup();
+ for (&b, chunk) in bytes.iter().zip(out.chunks_exact_mut(2)) {
+ chunk.copy_from_slice(&lookup[b as usize]);
+ }
+ Ok(())
+}
+
+/// Returns the hex encoding of `bytes` as an owned `String`.
+///
+/// # Example
+///
+/// ```
+/// use datafusion_common::utils::hex::{HexCase, encode_bytes};
+///
+/// assert_eq!(encode_bytes(&[0xde, 0xad, 0xbe, 0xef], HexCase::Lower), "deadbeef");
+/// assert_eq!(encode_bytes(&[0xde, 0xad, 0xbe, 0xef], HexCase::Upper), "DEADBEEF");
+/// ```
+#[inline]
+pub fn encode_bytes(bytes: &[u8], case: HexCase) -> String {
+ let mut out = Vec::with_capacity(bytes.len() * 2);
+ encode_bytes_into(bytes, case, &mut out);
+ // SAFETY: `out` holds only ASCII hex digits, which are valid UTF-8.
+ unsafe { String::from_utf8_unchecked(out) }
+}
+
+/// Writes `v` as hex into `buf` and returns the written subslice.
+///
+/// Digits are written right-aligned with leading zeros trimmed, so the result
+/// borrows the tail of `buf`. Zero encodes as `"0"`.
+///
+/// Signed values should be cast with `as u64`, which yields the two's
+/// complement representation that both `to_hex` and Spark's `hex` produce for
+/// negative input.
+///
+/// # Example
+///
+/// The caller owns the buffer and can reuse it across calls; each call
+/// returns a fresh subslice of it, borrowed for as long as `buf` is:
+///
+/// ```
+/// use datafusion_common::utils::hex::{HexCase, encode_u64};
+///
+/// let mut buf = [0u8; 16];
+/// assert_eq!(encode_u64(0xAB, HexCase::Lower, &mut buf), b"ab");
+/// assert_eq!(encode_u64(0, HexCase::Lower, &mut buf), b"0");
+/// ```
+#[inline(always)]
+pub fn encode_u64(v: u64, case: HexCase, buf: &mut [u8; 16]) -> &[u8] {
+ let start = write_digits(v, case, buf);
+ &buf[start..]
+}
+
+/// Writes the digits of `v` right-aligned in `buf`, returning the index of the
+/// first digit.
+///
+/// Split out from [`encode_u64`] so the mutable borrow of `buf` ends before the
+/// returned slice reborrows it.
+#[inline(always)]
+fn write_digits(v: u64, case: HexCase, buf: &mut [u8; 16]) -> usize {
+ if v == 0 {
+ buf[15] = b'0';
+ return 15;
+ }
+
+ // Consume two nibbles (one full byte) per iteration.
+ let lookup = case.lookup();
+ let mut pos = 16;
+ let mut rest = v;
+ while rest >= 0x10 {
+ pos -= 2;
+ let pair = lookup[(rest & 0xFF) as usize];
+ buf[pos] = pair[0];
+ buf[pos + 1] = pair[1];
+ rest >>= 8;
+ }
+ if rest > 0 {
+ // A single high nibble (0x1..=0xF) remains.
+ pos -= 1;
+ buf[pos] = case.digits()[rest as usize];
+ }
+
+ pos
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn hex_u64(v: u64, case: HexCase) -> String {
+ let mut buf = [0u8; 16];
+ String::from_utf8(encode_u64(v, case, &mut buf).to_vec()).unwrap()
+ }
+
+ #[test]
+ fn encode_u64_zero() {
+ assert_eq!(hex_u64(0, HexCase::Lower), "0");
+ assert_eq!(hex_u64(0, HexCase::Upper), "0");
+ }
+
+ #[test]
+ fn encode_u64_single_nibble() {
+ for v in 1..=0xFu64 {
+ assert_eq!(hex_u64(v, HexCase::Lower), format!("{v:x}"));
+ assert_eq!(hex_u64(v, HexCase::Upper), format!("{v:X}"));
+ }
+ }
+
+ #[test]
+ fn encode_u64_digit_count_boundaries() {
+ // Straddle each odd/even digit-count boundary: the two-nibbles-per
+ // iteration loop plus the trailing single-nibble fixup.
+ for v in [
+ 0x10u64,
+ 0xFF,
+ 0x100,
+ 0xFFF,
+ 0x1000,
+ 0xFFFFF,
+ 0xFFFF_FFFF,
+ 0x1_0000_0000,
+ ] {
+ assert_eq!(hex_u64(v, HexCase::Lower), format!("{v:x}"));
+ assert_eq!(hex_u64(v, HexCase::Upper), format!("{v:X}"));
+ }
+ }
+
+ #[test]
+ fn encode_u64_max() {
+ assert_eq!(hex_u64(u64::MAX, HexCase::Lower), "ffffffffffffffff");
+ assert_eq!(hex_u64(u64::MAX, HexCase::Upper), "FFFFFFFFFFFFFFFF");
+ }
+
+ #[test]
+ fn encode_u64_signed_is_twos_complement() {
+ // Callers cast signed values with `as u64`; this is the behaviour both
+ // `to_hex` and Spark `hex` rely on for negative input.
+ assert_eq!(hex_u64(-1i64 as u64, HexCase::Lower), "ffffffffffffffff");
+ assert_eq!(hex_u64(i64::MIN as u64, HexCase::Upper), "8000000000000000");
+ }
+
+ #[test]
+ fn encode_bytes_empty() {
+ assert_eq!(encode_bytes(&[], HexCase::Lower), "");
+ assert_eq!(encode_bytes(&[], HexCase::Upper), "");
+ }
+
+ #[test]
+ fn encode_bytes_examples() {
+ assert_eq!(encode_bytes(&[0x00], HexCase::Lower), "00");
+ assert_eq!(encode_bytes(&[0xAB], HexCase::Lower), "ab");
+ assert_eq!(encode_bytes(&[0xAB], HexCase::Upper), "AB");
+ assert_eq!(
+ encode_bytes(&[0xde, 0xad, 0xbe, 0xef], HexCase::Lower),
+ "deadbeef"
+ );
+ assert_eq!(
+ encode_bytes(&[0xde, 0xad, 0xbe, 0xef], HexCase::Upper),
+ "DEADBEEF"
+ );
+ }
+
+ #[test]
+ fn encode_bytes_covers_every_byte_value() {
+ let bytes: Vec<u8> = (0..=255u8).collect();
+
+ let expected: String = bytes.iter().map(|b| format!("{b:02x}")).collect();
+ assert_eq!(encode_bytes(&bytes, HexCase::Lower), expected);
+
+ let expected: String = bytes.iter().map(|b| format!("{b:02X}")).collect();
+ assert_eq!(encode_bytes(&bytes, HexCase::Upper), expected);
+ }
+
+ #[test]
+ fn encode_bytes_into_appends_without_clearing() {
+ let mut out = b"prefix-".to_vec();
+ encode_bytes_into(&[0x01, 0x02], HexCase::Lower, &mut out);
+ assert_eq!(out, b"prefix-0102");
+ }
+
+ #[test]
+ fn encode_u64_reused_buffer_leaks_no_stale_digits() {
+ let mut buf = [0u8; 16];
+ assert_eq!(
+ encode_u64(u64::MAX, HexCase::Lower, &mut buf),
+ b"ffffffffffffffff"
+ );
+ assert_eq!(encode_u64(0, HexCase::Lower, &mut buf), b"0");
+ assert_eq!(encode_u64(0xAB, HexCase::Lower, &mut buf), b"ab");
+ }
+
+ #[test]
+ fn encode_bytes_to_slice_empty() -> Result<()> {
+ let mut out: [u8; 0] = [];
+ encode_bytes_to_slice(&[], HexCase::Lower, &mut out)?;
+ assert_eq!(out, [] as [u8; 0]);
+ Ok(())
+ }
+
+ #[test]
+ fn encode_bytes_to_slice_examples() -> Result<()> {
+ let mut out = [0u8; 8];
+ encode_bytes_to_slice(&[0xde, 0xad, 0xbe, 0xef], HexCase::Lower, &mut out)?;
+ assert_eq!(&out, b"deadbeef");
+
+ let mut out = [0u8; 8];
+ encode_bytes_to_slice(&[0xde, 0xad, 0xbe, 0xef], HexCase::Upper, &mut out)?;
+ assert_eq!(&out, b"DEADBEEF");
+ Ok(())
+ }
+
+ #[test]
+ fn encode_bytes_to_slice_agrees_with_encode_bytes() -> Result<()> {
+ let bytes: Vec<u8> = (0..=255u8).collect();
+ for case in [HexCase::Lower, HexCase::Upper] {
+ let mut out = vec![0u8; bytes.len() * 2];
+ encode_bytes_to_slice(&bytes, case, &mut out)?;
+ assert_eq!(String::from_utf8(out).unwrap(), encode_bytes(&bytes, case));
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn encode_bytes_to_slice_rejects_wrong_length() {
+ // Too short: the old `debug_assert` let release builds silently drop
+ // the remaining input.
+ let mut short = [0u8; 6];
+ let err =
+ encode_bytes_to_slice(&[0xde, 0xad, 0xbe, 0xef], HexCase::Lower, &mut short)
+ .unwrap_err();
+ assert!(
+ err.message()
+ .contains("hex output buffer is 6 bytes, expected 8"),
+ "unexpected message: {err}"
+ );
+
+ // Too long: would have left stale bytes at the tail.
+ let mut long = [0u8; 10];
+ assert!(
+ encode_bytes_to_slice(&[0xde, 0xad, 0xbe, 0xef], HexCase::Lower, &mut long)
+ .is_err()
+ );
+ }
+}
diff --git a/datafusion/common/src/utils/mod.rs b/datafusion/common/src/utils/mod.rs
index 94bbb91a7f..73772b3193 100644
--- a/datafusion/common/src/utils/mod.rs
+++ b/datafusion/common/src/utils/mod.rs
@@ -19,6 +19,7 @@
pub(crate) mod aggregate;
pub mod expr;
+pub mod hex;
pub mod memory;
pub mod proxy;
pub mod string_utils;
diff --git a/datafusion/functions/src/crypto/md5.rs b/datafusion/functions/src/crypto/md5.rs
index 178aebf0fb..b1206d2e42 100644
--- a/datafusion/functions/src/crypto/md5.rs
+++ b/datafusion/functions/src/crypto/md5.rs
@@ -21,6 +21,7 @@ use datafusion_common::{
cast::as_binary_array,
internal_err,
types::{logical_binary, logical_string},
+ utils::hex::{HexCase, encode_bytes},
utils::take_function_args,
};
use datafusion_expr::{
@@ -98,22 +99,6 @@ impl ScalarUDFImpl for Md5Func {
}
}
-/// Hex encoding lookup table for fast byte-to-hex conversion
-const HEX_CHARS_LOWER: &[u8; 16] = b"0123456789abcdef";
-
-/// Fast hex encoding using a lookup table instead of format strings.
-/// This is significantly faster than using `write!("{:02x}")` for each byte.
-#[inline]
-fn hex_encode(data: impl AsRef<[u8]>) -> String {
- let bytes = data.as_ref();
- let mut s = String::with_capacity(bytes.len() * 2);
- for &b in bytes {
- s.push(HEX_CHARS_LOWER[(b >> 4) as usize] as char);
- s.push(HEX_CHARS_LOWER[(b & 0x0f) as usize] as char);
- }
- s
-}
-
fn md5(args: &[ColumnarValue]) -> Result<ColumnarValue> {
let [data] = take_function_args("md5", args)?;
let value = digest_process(data, DigestAlgorithm::Md5)?;
@@ -122,13 +107,15 @@ fn md5(args: &[ColumnarValue]) -> Result<ColumnarValue> {
Ok(match value {
ColumnarValue::Array(array) => {
let binary_array = as_binary_array(&array)?;
- let string_array: StringViewArray =
- binary_array.iter().map(|opt| opt.map(hex_encode)).collect();
+ let string_array: StringViewArray = binary_array
+ .iter()
+ .map(|opt| opt.map(|b| encode_bytes(b, HexCase::Lower)))
+ .collect();
ColumnarValue::Array(Arc::new(string_array))
}
- ColumnarValue::Scalar(ScalarValue::Binary(opt)) => {
- ColumnarValue::Scalar(ScalarValue::Utf8View(opt.map(hex_encode)))
- }
+ ColumnarValue::Scalar(ScalarValue::Binary(opt)) => ColumnarValue::Scalar(
+ ScalarValue::Utf8View(opt.map(|b| encode_bytes(&b, HexCase::Lower))),
+ ),
_ => return internal_err!("Impossibly got invalid results from digest"),
})
}
diff --git a/datafusion/functions/src/encoding/inner.rs b/datafusion/functions/src/encoding/inner.rs
index 5d4740d80b..850e312abd 100644
--- a/datafusion/functions/src/encoding/inner.rs
+++ b/datafusion/functions/src/encoding/inner.rs
@@ -33,7 +33,10 @@ use datafusion_common::{
DataFusionError, Result, ScalarValue, exec_datafusion_err, exec_err, internal_err,
not_impl_err, plan_err,
types::{NativeType, logical_string},
- utils::take_function_args,
+ utils::{
+ hex::{HexCase, encode_bytes as encode_hex, encode_bytes_to_slice},
+ take_function_args,
+ },
};
use datafusion_expr::{
Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
@@ -370,7 +373,7 @@ impl Encoding {
match self {
Self::Base64 => BASE64_ENGINE.encode(value),
Self::Base64Padded => BASE64_ENGINE_PADDED.encode(value),
- Self::Hex => hex::encode(value),
+ Self::Hex => encode_hex(value, HexCase::Lower),
}
}
@@ -477,11 +480,7 @@ where
for v in array.iter() {
if let Some(v) = v {
let out_len = v.len() * 2;
- // The slice is sized to exactly `2 * v.len()`, which is the only
- // condition under which `encode_to_slice` can fail, so this cannot
- // error.
- hex::encode_to_slice(v, &mut values[pos..pos + out_len])
- .map_err(|e| exec_datafusion_err!("Failed to encode to hex: {e}"))?;
+ encode_bytes_to_slice(v, HexCase::Lower, &mut values[pos..pos + out_len])?;
pos += out_len;
}
offsets.push(OutputOffset::usize_as(pos));
diff --git a/datafusion/functions/src/string/to_hex.rs b/datafusion/functions/src/string/to_hex.rs
index 497a0a1206..a6bcd17966 100644
--- a/datafusion/functions/src/string/to_hex.rs
+++ b/datafusion/functions/src/string/to_hex.rs
@@ -24,6 +24,7 @@ use arrow::datatypes::{
Int64Type, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
};
use datafusion_common::cast::as_primitive_array;
+use datafusion_common::utils::hex::{HexCase, encode_u64};
use datafusion_common::{Result, ScalarValue, exec_err, internal_err};
use datafusion_expr::{
Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
@@ -31,9 +32,6 @@ use datafusion_expr::{
};
use datafusion_macros::user_doc;
-/// Hex lookup table for fast conversion
-const HEX_CHARS: &[u8; 16] = b"0123456789abcdef";
-
/// Converts the number to its equivalent hexadecimal representation.
/// to_hex(2147483647) = '7fffffff'
fn to_hex_array<T: ArrowPrimitiveType>(array: &ArrayRef) -> Result<ArrayRef>
@@ -59,8 +57,7 @@ where
// Process all values directly (including null slots - we write empty strings for nulls)
// The null bitmap will mark which entries are actually null
for value in integer_array.values() {
- let hex_len = value.write_hex_to_buffer(&mut hex_buffer);
- values.extend_from_slice(&hex_buffer[16 - hex_len..]);
+ values.extend_from_slice(value.write_hex(&mut hex_buffer));
offsets.push(values.len() as i32);
}
@@ -79,101 +76,50 @@ where
#[inline]
fn to_hex_scalar<T: ToHex>(value: T) -> String {
let mut hex_buffer = [0u8; 16];
- let hex_len = value.write_hex_to_buffer(&mut hex_buffer);
- // SAFETY: hex_buffer is ASCII hex digits
- unsafe { std::str::from_utf8_unchecked(&hex_buffer[16 - hex_len..]).to_string() }
+ let hex = value.write_hex(&mut hex_buffer);
+ // SAFETY: hex holds only ASCII hex digits.
+ unsafe { std::str::from_utf8_unchecked(hex).to_string() }
}
/// Trait for converting integer types to hexadecimal in a buffer
trait ToHex: ArrowNativeType {
- /// Write hex representation to buffer and return the number of hex digits written.
- /// The hex digits are written right-aligned in the buffer (starting from position 16 - len).
- fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize;
-}
-
-/// Write unsigned value to hex buffer and return the number of digits written.
-/// Digits are written right-aligned in the buffer.
-#[inline]
-fn write_unsigned_hex_to_buffer(value: u64, buffer: &mut [u8; 16]) -> usize {
- if value == 0 {
- buffer[15] = b'0';
- return 1;
- }
-
- // Write hex digits from right to left
- let mut pos = 16;
- let mut v = value;
- while v > 0 {
- pos -= 1;
- buffer[pos] = HEX_CHARS[(v & 0xf) as usize];
- v >>= 4;
- }
-
- 16 - pos
-}
-
-/// Write signed value to hex buffer (two's complement for negative) and return digit count
-#[inline]
-fn write_signed_hex_to_buffer(value: i64, buffer: &mut [u8; 16]) -> usize {
- // For negative values, use two's complement representation (same as casting to u64)
- write_unsigned_hex_to_buffer(value as u64, buffer)
-}
-
-impl ToHex for i8 {
- #[inline]
- fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize {
- write_signed_hex_to_buffer(self as i64, buffer)
- }
-}
-
-impl ToHex for i16 {
- #[inline]
- fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize {
- write_signed_hex_to_buffer(self as i64, buffer)
- }
-}
-
-impl ToHex for i32 {
- #[inline]
- fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize {
- write_signed_hex_to_buffer(self as i64, buffer)
- }
-}
-
-impl ToHex for i64 {
- #[inline]
- fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize {
- write_signed_hex_to_buffer(self, buffer)
- }
+ /// Writes the hex representation into `buf` and returns the written
+ /// subslice. Digits are right-aligned in `buf` with leading zeros trimmed.
+ fn write_hex(self, buf: &mut [u8; 16]) -> &[u8];
}
-impl ToHex for u8 {
- #[inline]
- fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize {
- write_unsigned_hex_to_buffer(self as u64, buffer)
- }
-}
-
-impl ToHex for u16 {
- #[inline]
- fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize {
- write_unsigned_hex_to_buffer(self as u64, buffer)
- }
+/// Signed values use their two's complement representation, matching a cast to
+/// the corresponding unsigned type.
+macro_rules! impl_to_hex_signed {
+ ($ty:ty) => {
+ impl ToHex for $ty {
+ #[inline]
+ fn write_hex(self, buf: &mut [u8; 16]) -> &[u8] {
+ encode_u64(self as i64 as u64, HexCase::Lower, buf)
+ }
+ }
+ };
}
-impl ToHex for u32 {
- #[inline]
- fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize {
- write_unsigned_hex_to_buffer(self as u64, buffer)
- }
+macro_rules! impl_to_hex_unsigned {
+ ($ty:ty) => {
+ impl ToHex for $ty {
+ #[inline]
+ fn write_hex(self, buf: &mut [u8; 16]) -> &[u8] {
+ encode_u64(self as u64, HexCase::Lower, buf)
+ }
+ }
+ };
}
-impl ToHex for u64 {
- #[inline]
- fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize {
- write_unsigned_hex_to_buffer(self, buffer)
- }
-}
+impl_to_hex_signed!(i8);
+impl_to_hex_signed!(i16);
+impl_to_hex_signed!(i32);
+impl_to_hex_signed!(i64);
+impl_to_hex_unsigned!(u8);
+impl_to_hex_unsigned!(u16);
+impl_to_hex_unsigned!(u32);
+impl_to_hex_unsigned!(u64);
#[user_doc(
doc_section(label = "String Functions"),
diff --git a/datafusion/spark/src/function/hash/sha1.rs b/datafusion/spark/src/function/hash/sha1.rs
index dd9009eb82..05a224f33f 100644
--- a/datafusion/spark/src/function/hash/sha1.rs
+++ b/datafusion/spark/src/function/hash/sha1.rs
@@ -24,6 +24,7 @@ use datafusion_common::cast::{
as_large_binary_array,
};
use datafusion_common::types::{NativeType, logical_string};
+use datafusion_common::utils::hex::{HexCase, encode_bytes};
use datafusion_common::utils::take_function_args;
use datafusion_common::{Result, internal_err};
use datafusion_expr::{
@@ -89,18 +90,9 @@ impl ScalarUDFImpl for SparkSha1 {
}
}
-/// Hex encoding lookup table for fast byte-to-hex conversion
-const HEX_CHARS_LOWER: &[u8; 16] = b"0123456789abcdef";
-
#[inline]
fn spark_sha1_digest(value: &[u8]) -> String {
- let result = Sha1::digest(value);
- let mut s = String::with_capacity(result.len() * 2);
- for &b in result.as_slice() {
- s.push(HEX_CHARS_LOWER[(b >> 4) as usize] as char);
- s.push(HEX_CHARS_LOWER[(b & 0x0f) as usize] as char);
- }
- s
+ encode_bytes(&Sha1::digest(value), HexCase::Lower)
}
fn spark_sha1_impl<'a>(input: impl Iterator<Item = Option<&'a [u8]>>) -> ArrayRef {
diff --git a/datafusion/spark/src/function/hash/sha2.rs b/datafusion/spark/src/function/hash/sha2.rs
index 38fa0cc643..541df29576 100644
--- a/datafusion/spark/src/function/hash/sha2.rs
+++ b/datafusion/spark/src/function/hash/sha2.rs
@@ -20,6 +20,7 @@ use arrow::datatypes::{DataType, Int32Type};
use datafusion_common::types::{
NativeType, logical_binary, logical_int32, logical_string,
};
+use datafusion_common::utils::hex::{HexCase, encode_bytes};
use datafusion_common::utils::take_function_args;
use datafusion_common::{Result, ScalarValue, internal_err};
use datafusion_expr::{
@@ -112,22 +113,22 @@ impl ScalarUDFImpl for SparkSha2 {
224 => {
let mut digest = sha2::Sha224::default();
digest.update(bytes);
- Some(hex_encode(digest.finalize()))
+ Some(encode_bytes(&digest.finalize(), HexCase::Lower))
}
0 | 256 => {
let mut digest = sha2::Sha256::default();
digest.update(bytes);
- Some(hex_encode(digest.finalize()))
+ Some(encode_bytes(&digest.finalize(), HexCase::Lower))
}
384 => {
let mut digest = sha2::Sha384::default();
digest.update(bytes);
- Some(hex_encode(digest.finalize()))
+ Some(encode_bytes(&digest.finalize(), HexCase::Lower))
}
512 => {
let mut digest = sha2::Sha512::default();
digest.update(bytes);
- Some(hex_encode(digest.finalize()))
+ Some(encode_bytes(&digest.finalize(), HexCase::Lower))
}
_ => None,
};
@@ -222,22 +223,22 @@ where
(Some(value), Some(224)) => {
let mut digest = sha2::Sha224::default();
digest.update(value);
- Some(hex_encode(digest.finalize()))
+ Some(encode_bytes(&digest.finalize(), HexCase::Lower))
}
(Some(value), Some(0 | 256)) => {
let mut digest = sha2::Sha256::default();
digest.update(value);
- Some(hex_encode(digest.finalize()))
+ Some(encode_bytes(&digest.finalize(), HexCase::Lower))
}
(Some(value), Some(384)) => {
let mut digest = sha2::Sha384::default();
digest.update(value);
- Some(hex_encode(digest.finalize()))
+ Some(encode_bytes(&digest.finalize(), HexCase::Lower))
}
(Some(value), Some(512)) => {
let mut digest = sha2::Sha512::default();
digest.update(value);
- Some(hex_encode(digest.finalize()))
+ Some(encode_bytes(&digest.finalize(), HexCase::Lower))
}
// Unknown bit-lengths go to null, same as in Spark
_ => None,
@@ -245,19 +246,3 @@ where
.collect::<StringArray>();
Arc::new(array)
}
-
-const HEX_CHARS: [u8; 16] = *b"0123456789abcdef";
-
-#[inline]
-fn hex_encode<T: AsRef<[u8]>>(data: T) -> String {
- let bytes = data.as_ref();
- let mut out = Vec::with_capacity(bytes.len() * 2);
- for &b in bytes {
- let hi = b >> 4;
- let lo = b & 0x0F;
- out.push(HEX_CHARS[hi as usize]);
- out.push(HEX_CHARS[lo as usize]);
- }
- // SAFETY: out contains only ASCII
- unsafe { String::from_utf8_unchecked(out) }
-}
diff --git a/datafusion/spark/src/function/math/hex.rs b/datafusion/spark/src/function/math/hex.rs
index a283bd8fa7..1f505fda21 100644
--- a/datafusion/spark/src/function/math/hex.rs
+++ b/datafusion/spark/src/function/math/hex.rs
@@ -28,6 +28,7 @@ use arrow::{
use datafusion_common::cast::as_large_binary_array;
use datafusion_common::cast::as_string_view_array;
use datafusion_common::types::{NativeType, logical_int64, logical_string};
+use datafusion_common::utils::hex::{HexCase, encode_bytes_into, encode_u64};
use datafusion_common::utils::take_function_args;
use datafusion_common::{
DataFusionError,
@@ -110,54 +111,6 @@ impl ScalarUDFImpl for SparkHex {
}
}
-/// Hex encoding lookup tables for fast byte-to-hex conversion.
-///
-/// Each entry maps a full byte to its two-character hex encoding so the
-/// hot loop becomes one load + one two-byte extend per input byte instead
-/// of two nibble lookups and two pushes.
-const HEX_CHARS_UPPER_NIBBLES: &[u8; 16] = b"0123456789ABCDEF";
-const HEX_CHARS_LOWER_NIBBLES: &[u8; 16] = b"0123456789abcdef";
-
-const HEX_LOOKUP_UPPER: [[u8; 2]; 256] = build_hex_lookup(HEX_CHARS_UPPER_NIBBLES);
-const HEX_LOOKUP_LOWER: [[u8; 2]; 256] = build_hex_lookup(HEX_CHARS_LOWER_NIBBLES);
-
-const fn build_hex_lookup(nibbles: &[u8; 16]) -> [[u8; 2]; 256] {
- let mut table = [[0u8; 2]; 256];
- let mut i = 0;
- while i < 256 {
- table[i][0] = nibbles[(i >> 4) & 0xF];
- table[i][1] = nibbles[i & 0xF];
- i += 1;
- }
- table
-}
-
-#[inline]
-fn hex_int64(num: i64, buffer: &mut [u8; 16]) -> &[u8] {
- if num == 0 {
- return b"0";
- }
-
- // Walk the value two nibbles (one full byte) at a time. The buffer is
- // filled from the right so the high-order nibbles end up first; the
- // returned slice trims leading zeros automatically.
- let mut n = num as u64;
- let mut i = 16;
- while n >= 0x10 {
- i -= 2;
- let pair = HEX_LOOKUP_UPPER[(n & 0xFF) as usize];
- buffer[i] = pair[0];
- buffer[i + 1] = pair[1];
- n >>= 8;
- }
- if n > 0 {
- // Single remaining high nibble (value 0x1..=0xF).
- i -= 1;
- buffer[i] = HEX_CHARS_UPPER_NIBBLES[n as usize];
- }
- &buffer[i..]
-}
-
/// Generic hex encoding for byte array types
fn hex_encode_bytes<'a, I, T>(
iter: I,
@@ -168,10 +121,10 @@ where
I: Iterator<Item = Option<T>>,
T: AsRef<[u8]> + 'a,
{
- let lookup = if lowercase {
- &HEX_LOOKUP_LOWER
+ let case = if lowercase {
+ HexCase::Lower
} else {
- &HEX_LOOKUP_UPPER
+ HexCase::Upper
};
// Write hex digits directly into one growing value buffer, tracking offsets
@@ -195,9 +148,7 @@ where
"failed to reserve {additional} bytes for hex output: {e}"
)
})?;
- for &byte in bytes {
- values.extend_from_slice(&lookup[byte as usize]);
- }
+ encode_bytes_into(bytes, case, &mut values);
nulls.append_non_null();
} else {
nulls.append_null();
@@ -233,7 +184,7 @@ fn hex_encode_int64(
for v in iter {
if let Some(num) = v {
let mut temp = [0u8; 16];
- let slice = hex_int64(num, &mut temp);
+ let slice = encode_u64(num as u64, HexCase::Upper, &mut temp);
// SAFETY: slice contains only ASCII hex digests, which are valid UTF-8
unsafe {
builder.append_value(from_utf8_unchecked(slice));
@@ -381,11 +332,10 @@ pub fn compute_hex(
#[cfg(test)]
mod test {
- use std::str::from_utf8_unchecked;
use std::sync::Arc;
use arrow::array::{
- BinaryArray, DictionaryArray, Int32Array, Int64Array, StringArray,
+ Array, BinaryArray, DictionaryArray, Int32Array, Int64Array, StringArray,
};
use arrow::{
array::{
@@ -486,7 +436,7 @@ mod test {
#[test]
fn test_hex_int64() {
- let test_cases = vec![
+ let cases = vec![
(0_i64, "0"),
(1, "1"),
(15, "F"),
@@ -499,37 +449,28 @@ mod test {
(-1, "FFFFFFFFFFFFFFFF"),
];
- for (num, expected) in test_cases {
- let mut cache = [0u8; 16];
- let slice = super::hex_int64(num, &mut cache);
-
- unsafe {
- let result = from_utf8_unchecked(slice);
- assert_eq!(expected, result, "hex_int64({num}) mismatch");
- }
+ let arr =
+ super::hex_encode_int64(cases.iter().map(|(n, _)| Some(*n)), cases.len())
+ .unwrap();
+ let arr = as_string_array(&arr);
+ for (i, (num, expected)) in cases.iter().enumerate() {
+ assert_eq!(*expected, arr.value(i), "hex({num})");
}
}
#[test]
- fn test_hex_lookup_table_covers_all_bytes() {
- // Cross-check the precomputed table against an independent encoder
- // for every possible byte value and both casings.
- for byte in 0u8..=255 {
- let upper = format!("{byte:02X}");
- let lower = format!("{byte:02x}");
- let upper_pair = super::HEX_LOOKUP_UPPER[byte as usize];
- let lower_pair = super::HEX_LOOKUP_LOWER[byte as usize];
- assert_eq!(
- upper.as_bytes(),
- &upper_pair,
- "upper encoding mismatch for byte 0x{byte:02X}"
- );
- assert_eq!(
- lower.as_bytes(),
- &lower_pair,
- "lower encoding mismatch for byte 0x{byte:02X}"
- );
- }
+ fn test_hex_encode_bytes_lowercase() {
+ // Every in-repo caller of `hex_encode_bytes` goes through `spark_hex`,
+ // which always passes `lowercase = false`. The `lowercase = true` path
+ // is reachable only via `spark_sha2_hex`, which has no in-workspace
+ // caller, so it otherwise has no coverage. Drive it directly here.
+ let input = StringArray::from(vec![Some("hi"), Some("bye"), None, Some("rust")]);
+ let result = super::hex_encode_bytes(input.iter(), true, input.len()).unwrap();
+ let result = as_string_array(&result);
+
+ let expected =
+ StringArray::from(vec![Some("6869"), Some("627965"), None, Some("72757374")]);
+ assert_eq!(result, &expected);
}
#[test]