Apache DataFusion SQL Query Engine
minor: remove unused crypto functions & narrow public API (#20045)
Jeffrey Vo613f87d61bd7
diff --git a/datafusion/functions/src/crypto/basic.rs b/datafusion/functions/src/crypto/basic.rsindex bda16684c8..abb86b8246 100644--- a/datafusion/functions/src/crypto/basic.rs+++ b/datafusion/functions/src/crypto/basic.rs@@ -17,19 +17,13 @@//! "crypto" DataFusion functions-use arrow::array::{- Array, ArrayRef, AsArray, BinaryArray, BinaryArrayType, StringViewArray,-};+use arrow::array::{Array, ArrayRef, AsArray, BinaryArray, BinaryArrayType};use arrow::datatypes::DataType;use blake2::{Blake2b512, Blake2s256, Digest};use blake3::Hasher as Blake3;-use datafusion_common::cast::as_binary_array;use arrow::compute::StringArrayType;-use datafusion_common::{- DataFusionError, Result, ScalarValue, exec_err, internal_err, plan_err,- utils::take_function_args,-};+use datafusion_common::{DataFusionError, Result, ScalarValue, exec_err, plan_err};use datafusion_expr::ColumnarValue;use md5::Md5;use sha2::{Sha224, Sha256, Sha384, Sha512};@@ -37,53 +31,8 @@ use std::fmt;use std::str::FromStr;use std::sync::Arc;-macro_rules! define_digest_function {- ($NAME: ident, $METHOD: ident, $DOC: expr) => {- #[doc = $DOC]- pub fn $NAME(args: &[ColumnarValue]) -> Result<ColumnarValue> {- let [data] = take_function_args(&DigestAlgorithm::$METHOD.to_string(), args)?;- digest_process(data, DigestAlgorithm::$METHOD)- }- };-}-define_digest_function!(- sha224,- Sha224,- "computes sha224 hash digest of the given input"-);-define_digest_function!(- sha256,- Sha256,- "computes sha256 hash digest of the given input"-);-define_digest_function!(- sha384,- Sha384,- "computes sha384 hash digest of the given input"-);-define_digest_function!(- sha512,- Sha512,- "computes sha512 hash digest of the given input"-);-define_digest_function!(- blake2b,- Blake2b,- "computes blake2b hash digest of the given input"-);-define_digest_function!(- blake2s,- Blake2s,- "computes blake2s hash digest of the given input"-);-define_digest_function!(- blake3,- Blake3,- "computes blake3 hash digest of the given input"-);-#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]-pub enum DigestAlgorithm {+pub(crate) enum DigestAlgorithm {Md5,Sha224,Sha256,@@ -135,44 +84,6 @@ impl fmt::Display for DigestAlgorithm {}}-/// computes md5 hash digest of the given input-pub fn md5(args: &[ColumnarValue]) -> Result<ColumnarValue> {- let [data] = take_function_args("md5", args)?;- let value = digest_process(data, DigestAlgorithm::Md5)?;-- // md5 requires special handling because of its unique utf8view return type- Ok(match value {- ColumnarValue::Array(array) => {- let binary_array = as_binary_array(&array)?;- let string_array: StringViewArray = binary_array- .iter()- .map(|opt| opt.map(hex_encode::<_>))- .collect();- ColumnarValue::Array(Arc::new(string_array))- }- ColumnarValue::Scalar(ScalarValue::Binary(opt)) => {- ColumnarValue::Scalar(ScalarValue::Utf8View(opt.map(hex_encode::<_>)))- }- _ => return internal_err!("Impossibly got invalid results from digest"),- })-}--/// Hex encoding lookup table for fast byte-to-hex conversion-const HEX_CHARS_LOWER: &[u8; 16] = b"0123456789abcdef";--/// Fast hex encoding using a lookup table instead of format strings.-/// This is significantly faster than using `write!("{:02x}")` for each byte.-#[inline]-fn hex_encode<T: AsRef<[u8]>>(data: T) -> String {- let bytes = data.as_ref();- let mut s = String::with_capacity(bytes.len() * 2);- for &b in bytes {- s.push(HEX_CHARS_LOWER[(b >> 4) as usize] as char);- s.push(HEX_CHARS_LOWER[(b & 0x0f) as usize] as char);- }- s-}-macro_rules! digest_to_array {($METHOD:ident, $INPUT:expr) => {{let binary_array: BinaryArray = $INPUT@@ -269,7 +180,7 @@ impl DigestAlgorithm {}}-pub fn digest_process(+pub(crate) fn digest_process(value: &ColumnarValue,digest_algorithm: DigestAlgorithm,) -> Result<ColumnarValue> {diff --git a/datafusion/functions/src/crypto/md5.rs b/datafusion/functions/src/crypto/md5.rsindex 728e0d4a33..355e3e287a 100644--- a/datafusion/functions/src/crypto/md5.rs+++ b/datafusion/functions/src/crypto/md5.rs@@ -15,11 +15,13 @@// specific language governing permissions and limitations// under the License.-use crate::crypto::basic::md5;-use arrow::datatypes::DataType;+use arrow::{array::StringViewArray, datatypes::DataType};use datafusion_common::{- Result,+ Result, ScalarValue,+ cast::as_binary_array,+ internal_err,types::{logical_binary, logical_string},+ utils::take_function_args,};use datafusion_expr::{ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,@@ -27,7 +29,9 @@ use datafusion_expr::{};use datafusion_expr_common::signature::{Coercion, TypeSignatureClass};use datafusion_macros::user_doc;-use std::any::Any;+use std::{any::Any, sync::Arc};++use crate::crypto::basic::{DigestAlgorithm, digest_process};#[user_doc(doc_section(label = "Hashing Functions"),@@ -97,3 +101,38 @@ impl ScalarUDFImpl for Md5Func {self.doc()}}++/// Hex encoding lookup table for fast byte-to-hex conversion+const HEX_CHARS_LOWER: &[u8; 16] = b"0123456789abcdef";++/// Fast hex encoding using a lookup table instead of format strings.+/// This is significantly faster than using `write!("{:02x}")` for each byte.+#[inline]+fn hex_encode(data: impl AsRef<[u8]>) -> String {+ let bytes = data.as_ref();+ let mut s = String::with_capacity(bytes.len() * 2);+ for &b in bytes {+ s.push(HEX_CHARS_LOWER[(b >> 4) as usize] as char);+ s.push(HEX_CHARS_LOWER[(b & 0x0f) as usize] as char);+ }+ s+}++fn md5(args: &[ColumnarValue]) -> Result<ColumnarValue> {+ let [data] = take_function_args("md5", args)?;+ let value = digest_process(data, DigestAlgorithm::Md5)?;++ // md5 requires special handling because of its unique utf8view return type+ Ok(match value {+ ColumnarValue::Array(array) => {+ let binary_array = as_binary_array(&array)?;+ let string_array: StringViewArray =+ binary_array.iter().map(|opt| opt.map(hex_encode)).collect();+ ColumnarValue::Array(Arc::new(string_array))+ }+ ColumnarValue::Scalar(ScalarValue::Binary(opt)) => {+ ColumnarValue::Scalar(ScalarValue::Utf8View(opt.map(hex_encode)))+ }+ _ => return internal_err!("Impossibly got invalid results from digest"),+ })+}