Apache DataFusion SQL Query Engine
Simplify Spark `sha2` implementation (#19475)
Jeffrey Vo568f19fa3911
diff --git a/Cargo.lock b/Cargo.lockindex 2cf439134d..1f28687f4f 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2582,6 +2582,7 @@ dependencies = ["percent-encoding","rand 0.9.2","sha1",+ "sha2","url",]diff --git a/Cargo.toml b/Cargo.tomlindex 03c3f8d4b3..102749a55a 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -181,6 +181,7 @@ recursive = "0.1.1"regex = "1.12"rstest = "0.26.1"serde_json = "1"+sha2 = "^0.10.9"sqlparser = { version = "0.60.0", default-features = false, features = ["std", "visitor"] }strum = "0.27.2"strum_macros = "0.27.2"diff --git a/datafusion/functions/Cargo.toml b/datafusion/functions/Cargo.tomlindex e6ba7b63ce..529f6354ef 100644--- a/datafusion/functions/Cargo.toml+++ b/datafusion/functions/Cargo.toml@@ -85,7 +85,7 @@ md-5 = { version = "^0.10.0", optional = true }num-traits = { workspace = true }rand = { workspace = true }regex = { workspace = true, optional = true }-sha2 = { version = "^0.10.9", optional = true }+sha2 = { workspace = true, optional = true }unicode-segmentation = { version = "^1.7.1", optional = true }uuid = { workspace = true, features = ["v4"], optional = true }diff --git a/datafusion/spark/Cargo.toml b/datafusion/spark/Cargo.tomlindex afab872d1b..466c17b789 100644--- a/datafusion/spark/Cargo.toml+++ b/datafusion/spark/Cargo.toml@@ -60,6 +60,7 @@ log = { workspace = true }percent-encoding = "2.3.2"rand = { workspace = true }sha1 = "0.10"+sha2 = { workspace = true }url = { workspace = true }[dev-dependencies]diff --git a/datafusion/spark/src/function/hash/sha2.rs b/datafusion/spark/src/function/hash/sha2.rsindex 1f17275062..a7ce5d7eb0 100644--- a/datafusion/spark/src/function/hash/sha2.rs+++ b/datafusion/spark/src/function/hash/sha2.rs@@ -15,26 +15,30 @@// specific language governing permissions and limitations// under the License.-extern crate datafusion_functions;--use crate::function::error_utils::{- invalid_arg_count_exec_err, unsupported_data_type_exec_err,-};-use crate::function::math::hex::spark_sha2_hex;-use arrow::array::{ArrayRef, AsArray, StringArray};+use arrow::array::{ArrayRef, AsArray, BinaryArrayType, Int32Array, StringArray};use arrow::datatypes::{DataType, Int32Type};-use datafusion_common::{Result, ScalarValue, exec_err, internal_datafusion_err};-use datafusion_expr::Signature;-use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Volatility};-pub use datafusion_functions::crypto::basic::{sha224, sha256, sha384, sha512};+use datafusion_common::types::{+ NativeType, logical_binary, logical_int32, logical_string,+};+use datafusion_common::utils::take_function_args;+use datafusion_common::{Result, internal_err};+use datafusion_expr::{+ Coercion, ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature,+ TypeSignatureClass, Volatility,+};+use datafusion_functions::utils::make_scalar_function;+use sha2::{self, Digest};use std::any::Any;+use std::fmt::Write;use std::sync::Arc;+/// Differs from DataFusion version in allowing array input for bit lengths, and+/// also hex encoding the output.+////// <https://spark.apache.org/docs/latest/api/sql/index.html#sha2>#[derive(Debug, PartialEq, Eq, Hash)]pub struct SparkSha2 {signature: Signature,- aliases: Vec<String>,}impl Default for SparkSha2 {@@ -46,8 +50,21 @@ impl Default for SparkSha2 {impl SparkSha2 {pub fn new() -> Self {Self {- signature: Signature::user_defined(Volatility::Immutable),- aliases: vec![],+ signature: Signature::coercible(+ vec![+ Coercion::new_implicit(+ TypeSignatureClass::Native(logical_binary()),+ vec![TypeSignatureClass::Native(logical_string())],+ NativeType::Binary,+ ),+ Coercion::new_implicit(+ TypeSignatureClass::Native(logical_int32()),+ vec![TypeSignatureClass::Integer],+ NativeType::Int32,+ ),+ ],+ Volatility::Immutable,+ ),}}}@@ -65,163 +82,73 @@ impl ScalarUDFImpl for SparkSha2 {&self.signature}- fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {- if arg_types[1].is_null() {- return Ok(DataType::Null);- }- Ok(match arg_types[0] {- DataType::Utf8View- | DataType::LargeUtf8- | DataType::Utf8- | DataType::Binary- | DataType::BinaryView- | DataType::LargeBinary => DataType::Utf8,- DataType::Null => DataType::Null,- _ => {- return exec_err!(- "{} function can only accept strings or binary arrays.",- self.name()- );- }- })+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {+ Ok(DataType::Utf8)}fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {- let args: [ColumnarValue; 2] = args.args.try_into().map_err(|_| {- internal_datafusion_err!("Expected 2 arguments for function sha2")- })?;-- sha2(args)+ make_scalar_function(sha2_impl, vec![])(&args.args)}+}- fn aliases(&self) -> &[String] {- &self.aliases- }+fn sha2_impl(args: &[ArrayRef]) -> Result<ArrayRef> {+ let [values, bit_lengths] = take_function_args("sha2", args)?;- fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {- if arg_types.len() != 2 {- return Err(invalid_arg_count_exec_err(- self.name(),- (2, 2),- arg_types.len(),- ));+ let bit_lengths = bit_lengths.as_primitive::<Int32Type>();+ let output = match values.data_type() {+ DataType::Binary => sha2_binary_impl(&values.as_binary::<i32>(), bit_lengths),+ DataType::LargeBinary => {+ sha2_binary_impl(&values.as_binary::<i64>(), bit_lengths)}- let expr_type = match &arg_types[0] {- DataType::Utf8View- | DataType::LargeUtf8- | DataType::Utf8- | DataType::Binary- | DataType::BinaryView- | DataType::LargeBinary- | DataType::Null => Ok(arg_types[0].clone()),- _ => Err(unsupported_data_type_exec_err(- self.name(),- "String, Binary",- &arg_types[0],- )),- }?;- let bit_length_type = if arg_types[1].is_numeric() {- Ok(DataType::Int32)- } else if arg_types[1].is_null() {- Ok(DataType::Null)- } else {- Err(unsupported_data_type_exec_err(- self.name(),- "Numeric Type",- &arg_types[1],- ))- }?;-- Ok(vec![expr_type, bit_length_type])- }+ DataType::BinaryView => sha2_binary_impl(&values.as_binary_view(), bit_lengths),+ dt => return internal_err!("Unsupported datatype for sha2: {dt}"),+ };+ Ok(output)}-pub fn sha2(args: [ColumnarValue; 2]) -> Result<ColumnarValue> {- match args {- [- ColumnarValue::Scalar(ScalarValue::Utf8(expr_arg)),- ColumnarValue::Scalar(ScalarValue::Int32(Some(bit_length_arg))),- ] => compute_sha2(- bit_length_arg,- &[ColumnarValue::from(ScalarValue::Utf8(expr_arg))],- ),- [- ColumnarValue::Array(expr_arg),- ColumnarValue::Scalar(ScalarValue::Int32(Some(bit_length_arg))),- ] => compute_sha2(bit_length_arg, &[ColumnarValue::from(expr_arg)]),- [- ColumnarValue::Scalar(ScalarValue::Utf8(expr_arg)),- ColumnarValue::Array(bit_length_arg),- ] => {- let arr: StringArray = bit_length_arg- .as_primitive::<Int32Type>()- .iter()- .map(|bit_length| {- match sha2([- ColumnarValue::Scalar(ScalarValue::Utf8(expr_arg.clone())),- ColumnarValue::Scalar(ScalarValue::Int32(bit_length)),- ])- .unwrap()- {- ColumnarValue::Scalar(ScalarValue::Utf8(str)) => str,- ColumnarValue::Array(arr) => arr- .as_string::<i32>()- .iter()- .map(|str| str.unwrap().to_string())- .next(), // first element- _ => unreachable!(),- }- })- .collect();- Ok(ColumnarValue::Array(Arc::new(arr) as ArrayRef))- }- [- ColumnarValue::Array(expr_arg),- ColumnarValue::Array(bit_length_arg),- ] => {- let expr_iter = expr_arg.as_string::<i32>().iter();- let bit_length_iter = bit_length_arg.as_primitive::<Int32Type>().iter();- let arr: StringArray = expr_iter- .zip(bit_length_iter)- .map(|(expr, bit_length)| {- match sha2([- ColumnarValue::Scalar(ScalarValue::Utf8(Some(- expr.unwrap().to_string(),- ))),- ColumnarValue::Scalar(ScalarValue::Int32(bit_length)),- ])- .unwrap()- {- ColumnarValue::Scalar(ScalarValue::Utf8(str)) => str,- ColumnarValue::Array(arr) => arr- .as_string::<i32>()- .iter()- .map(|str| str.unwrap().to_string())- .next(), // first element- _ => unreachable!(),- }- })- .collect();- Ok(ColumnarValue::Array(Arc::new(arr) as ArrayRef))- }- _ => exec_err!("Unsupported argument types for sha2 function"),- }+fn sha2_binary_impl<'a, BinaryArrType>(+ values: &BinaryArrType,+ bit_lengths: &Int32Array,+) -> ArrayRef+where+ BinaryArrType: BinaryArrayType<'a>,+{+ let array = values+ .iter()+ .zip(bit_lengths.iter())+ .map(|(value, bit_length)| match (value, bit_length) {+ (Some(value), Some(224)) => {+ let mut digest = sha2::Sha224::default();+ digest.update(value);+ Some(hex_encode(digest.finalize()))+ }+ (Some(value), Some(0 | 256)) => {+ let mut digest = sha2::Sha256::default();+ digest.update(value);+ Some(hex_encode(digest.finalize()))+ }+ (Some(value), Some(384)) => {+ let mut digest = sha2::Sha384::default();+ digest.update(value);+ Some(hex_encode(digest.finalize()))+ }+ (Some(value), Some(512)) => {+ let mut digest = sha2::Sha512::default();+ digest.update(value);+ Some(hex_encode(digest.finalize()))+ }+ // Unknown bit-lengths go to null, same as in Spark+ _ => None,+ })+ .collect::<StringArray>();+ Arc::new(array)}-fn compute_sha2(- bit_length_arg: i32,- expr_arg: &[ColumnarValue],-) -> Result<ColumnarValue> {- match bit_length_arg {- 0 | 256 => sha256(expr_arg),- 224 => sha224(expr_arg),- 384 => sha384(expr_arg),- 512 => sha512(expr_arg),- _ => {- // Return null for unsupported bit lengths instead of error, because spark sha2 does not- // error out for this.- return Ok(ColumnarValue::Scalar(ScalarValue::Utf8(None)));- }+fn hex_encode<T: AsRef<[u8]>>(data: T) -> String {+ let mut s = String::with_capacity(data.as_ref().len() * 2);+ for b in data.as_ref() {+ // Writing to a string never errors, so we can unwrap here.+ write!(&mut s, "{b:02x}").unwrap();}- .map(|hashed| spark_sha2_hex(&[hashed]).unwrap())+ s}diff --git a/datafusion/sqllogictest/test_files/spark/hash/sha2.slt b/datafusion/sqllogictest/test_files/spark/hash/sha2.sltindex 7690a38773..07f70947fe 100644--- a/datafusion/sqllogictest/test_files/spark/hash/sha2.slt+++ b/datafusion/sqllogictest/test_files/spark/hash/sha2.slt@@ -75,3 +75,58 @@ SELECT sha2(expr, bit_length) FROM VALUES ('foo',0::INT), ('bar',224::INT), ('ba967004d25de4abc1bd6a7c9a216254a5ac0733e8ad96dc9f1ea0fad9619da7c32d654ec8ad8ba2f9b5728fed6633bd918c6be9ed448a34883a13a13f4ead4aefa036b67dcda59020c01e57ea075ea8a4792d428f2c6fd0c09d1c49994d6c22789336e062188df29572ed07e7f9779c52NULL++# All string types+query T+SELECT sha2(arrow_cast('foo', 'Utf8'), bit_length) FROM VALUES (224::INT), (256::INT) AS t(bit_length);+----+0808f64e60d58979fcb676c96ec938270dea42445aeefcd3a4e6f8db+2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae++query T+SELECT sha2(arrow_cast('foo', 'LargeUtf8'), bit_length) FROM VALUES (224::INT), (256::INT) AS t(bit_length);+----+0808f64e60d58979fcb676c96ec938270dea42445aeefcd3a4e6f8db+2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae++query T+SELECT sha2(arrow_cast('foo', 'Utf8View'), bit_length) FROM VALUES (224::INT), (256::INT) AS t(bit_length);+----+0808f64e60d58979fcb676c96ec938270dea42445aeefcd3a4e6f8db+2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae++# All binary types+query T+SELECT sha2(arrow_cast('foo', 'Binary'), bit_length) FROM VALUES (224::INT), (256::INT) AS t(bit_length);+----+0808f64e60d58979fcb676c96ec938270dea42445aeefcd3a4e6f8db+2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae++query T+SELECT sha2(arrow_cast('foo', 'LargeBinary'), bit_length) FROM VALUES (224::INT), (256::INT) AS t(bit_length);+----+0808f64e60d58979fcb676c96ec938270dea42445aeefcd3a4e6f8db+2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae++query T+SELECT sha2(arrow_cast('foo', 'BinaryView'), bit_length) FROM VALUES (224::INT), (256::INT) AS t(bit_length);+----+0808f64e60d58979fcb676c96ec938270dea42445aeefcd3a4e6f8db+2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae+++# Null cases+query T+select sha2(null, 0);+----+NULL++query T+select sha2('a', null);+----+NULL++query T+select sha2('a', null::int);+----+NULL