Apache DataFusion SQL Query Engine
Refactor Spark crc32/sha1 signatures (#18662)
Jeffrey Voc39334f65efb
diff --git a/datafusion/spark/src/function/hash/crc32.rs b/datafusion/spark/src/function/hash/crc32.rsindex 76e31d12c6..8280e24b8e 100644--- a/datafusion/spark/src/function/hash/crc32.rs+++ b/datafusion/spark/src/function/hash/crc32.rs@@ -22,11 +22,15 @@ use arrow::array::{ArrayRef, Int64Array};use arrow::datatypes::DataType;use crc32fast::Hasher;use datafusion_common::cast::{- as_binary_array, as_binary_view_array, as_large_binary_array,+ as_binary_array, as_binary_view_array, as_fixed_size_binary_array,+ as_large_binary_array,};-use datafusion_common::{exec_err, internal_err, Result};+use datafusion_common::types::{logical_string, NativeType};+use datafusion_common::utils::take_function_args;+use datafusion_common::{internal_err, Result};use datafusion_expr::{- ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,+ Coercion, ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature,+ TypeSignatureClass, Volatility,};use datafusion_functions::utils::make_scalar_function;@@ -45,7 +49,14 @@ impl Default for SparkCrc32 {impl SparkCrc32 {pub fn new() -> Self {Self {- signature: Signature::user_defined(Volatility::Immutable),+ signature: Signature::coercible(+ vec![Coercion::new_implicit(+ TypeSignatureClass::Binary,+ vec![TypeSignatureClass::Native(logical_string())],+ NativeType::Binary,+ )],+ Volatility::Immutable,+ ),}}}@@ -70,24 +81,6 @@ impl ScalarUDFImpl for SparkCrc32 {fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {make_scalar_function(spark_crc32, vec![])(&args.args)}-- fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {- if arg_types.len() != 1 {- return exec_err!(- "`crc32` function requires 1 argument, got {}",- arg_types.len()- );- }- match arg_types[0] {- DataType::Binary | DataType::LargeBinary | DataType::BinaryView => {- Ok(vec![arg_types[0].clone()])- }- DataType::Utf8 | DataType::Utf8View => Ok(vec![DataType::Binary]),- DataType::LargeUtf8 => Ok(vec![DataType::LargeBinary]),- DataType::Null => Ok(vec![DataType::Binary]),- _ => exec_err!("`crc32` function does not support type {}", arg_types[0]),- }- }}fn spark_crc32_digest(value: &[u8]) -> i64 {@@ -104,14 +97,10 @@ fn spark_crc32_impl<'a>(input: impl Iterator<Item = Option<&'a [u8]>>) -> ArrayR}fn spark_crc32(args: &[ArrayRef]) -> Result<ArrayRef> {- let [input] = args else {- return internal_err!(- "Spark `crc32` function requires 1 argument, got {}",- args.len()- );- };+ let [input] = take_function_args("crc32", args)?;match input.data_type() {+ DataType::Null => Ok(Arc::new(Int64Array::new_null(input.len()))),DataType::Binary => {let input = as_binary_array(input)?;Ok(spark_crc32_impl(input.iter()))@@ -124,11 +113,12 @@ fn spark_crc32(args: &[ArrayRef]) -> Result<ArrayRef> {let input = as_binary_view_array(input)?;Ok(spark_crc32_impl(input.iter()))}- _ => {- exec_err!(- "Spark `crc32` function: argument must be binary or large binary, got {:?}",- input.data_type()- )+ DataType::FixedSizeBinary(_) => {+ let input = as_fixed_size_binary_array(input)?;+ Ok(spark_crc32_impl(input.iter()))+ }+ dt => {+ internal_err!("Unsupported data type for crc32: {dt}")}}}diff --git a/datafusion/spark/src/function/hash/sha1.rs b/datafusion/spark/src/function/hash/sha1.rsindex 5b2a2653ed..9e3d96b803 100644--- a/datafusion/spark/src/function/hash/sha1.rs+++ b/datafusion/spark/src/function/hash/sha1.rs@@ -22,11 +22,15 @@ use std::sync::Arc;use arrow::array::{ArrayRef, StringArray};use arrow::datatypes::DataType;use datafusion_common::cast::{- as_binary_array, as_binary_view_array, as_large_binary_array,+ as_binary_array, as_binary_view_array, as_fixed_size_binary_array,+ as_large_binary_array,};-use datafusion_common::{exec_err, internal_err, Result};+use datafusion_common::types::{logical_string, NativeType};+use datafusion_common::utils::take_function_args;+use datafusion_common::{internal_err, Result};use datafusion_expr::{- ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,+ Coercion, ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature,+ TypeSignatureClass, Volatility,};use datafusion_functions::utils::make_scalar_function;use sha1::{Digest, Sha1};@@ -47,7 +51,14 @@ impl Default for SparkSha1 {impl SparkSha1 {pub fn new() -> Self {Self {- signature: Signature::user_defined(Volatility::Immutable),+ signature: Signature::coercible(+ vec![Coercion::new_implicit(+ TypeSignatureClass::Binary,+ vec![TypeSignatureClass::Native(logical_string())],+ NativeType::Binary,+ )],+ Volatility::Immutable,+ ),aliases: vec!["sha".to_string()],}}@@ -77,24 +88,6 @@ impl ScalarUDFImpl for SparkSha1 {fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {make_scalar_function(spark_sha1, vec![])(&args.args)}-- fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {- if arg_types.len() != 1 {- return exec_err!(- "`sha1` function requires 1 argument, got {}",- arg_types.len()- );- }- match arg_types[0] {- DataType::Binary | DataType::LargeBinary | DataType::BinaryView => {- Ok(vec![arg_types[0].clone()])- }- DataType::Utf8 | DataType::Utf8View => Ok(vec![DataType::Binary]),- DataType::LargeUtf8 => Ok(vec![DataType::LargeBinary]),- DataType::Null => Ok(vec![DataType::Binary]),- _ => exec_err!("`sha1` function does not support type {}", arg_types[0]),- }- }}fn spark_sha1_digest(value: &[u8]) -> String {@@ -102,7 +95,6 @@ fn spark_sha1_digest(value: &[u8]) -> String {let mut s = String::with_capacity(result.len() * 2);#[allow(deprecated)]for b in result.as_slice() {- #[allow(clippy::unwrap_used)]write!(&mut s, "{b:02x}").unwrap();}s@@ -116,14 +108,10 @@ fn spark_sha1_impl<'a>(input: impl Iterator<Item = Option<&'a [u8]>>) -> ArrayRe}fn spark_sha1(args: &[ArrayRef]) -> Result<ArrayRef> {- let [input] = args else {- return internal_err!(- "Spark `sha1` function requires 1 argument, got {}",- args.len()- );- };+ let [input] = take_function_args("sha1", args)?;match input.data_type() {+ DataType::Null => Ok(Arc::new(StringArray::new_null(input.len()))),DataType::Binary => {let input = as_binary_array(input)?;Ok(spark_sha1_impl(input.iter()))@@ -136,11 +124,12 @@ fn spark_sha1(args: &[ArrayRef]) -> Result<ArrayRef> {let input = as_binary_view_array(input)?;Ok(spark_sha1_impl(input.iter()))}- _ => {- exec_err!(- "Spark `sha1` function: argument must be binary or large binary, got {:?}",- input.data_type()- )+ DataType::FixedSizeBinary(_) => {+ let input = as_fixed_size_binary_array(input)?;+ Ok(spark_sha1_impl(input.iter()))+ }+ dt => {+ internal_err!("Unsupported data type for sha1: {dt}")}}}diff --git a/datafusion/sqllogictest/test_files/spark/hash/crc32.slt b/datafusion/sqllogictest/test_files/spark/hash/crc32.sltindex 87b69d8d40..6fbeb11fb9 100644--- a/datafusion/sqllogictest/test_files/spark/hash/crc32.slt+++ b/datafusion/sqllogictest/test_files/spark/hash/crc32.slt@@ -30,11 +30,6 @@ SELECT crc32('Spark');----1557323817-query I-SELECT crc32(NULL);------NULL-query ISELECT crc32('');----@@ -45,32 +40,48 @@ SELECT crc32(arrow_cast('', 'Binary'));----0-# Test with LargeUtf8 (using CAST to ensure type)+# Test with different types+query I+SELECT crc32(NULL);+----+NULL+query ISELECT crc32(arrow_cast('Spark', 'LargeUtf8'));----1557323817-# Test with Utf8View (using CAST to ensure type)query ISELECT crc32(arrow_cast('Spark', 'Utf8View'));----1557323817-# Test with different binary types+query I+SELECT crc32(arrow_cast('Spark', 'Utf8'));+----+1557323817+query ISELECT crc32(arrow_cast('Spark', 'Binary'));----1557323817-# Test with LargeBinary+query I+SELECT crc32(arrow_cast(arrow_cast('Spark', 'Binary'), 'FixedSizeBinary(5)'));+----+1557323817+query ISELECT crc32(arrow_cast('Spark', 'LargeBinary'));----1557323817-# Test with BinaryViewquery ISELECT crc32(arrow_cast('Spark', 'BinaryView'));----1557323817++# Upstream arrow-rs issue: https://github.com/apache/arrow-rs/issues/8841+# This should succeed after we receive the fix+query error Arrow error: Compute error: Internal Error: Cannot cast BinaryView to BinaryArray of expected type+select crc32(arrow_cast(null, 'Dictionary(Int32, Utf8)'))diff --git a/datafusion/sqllogictest/test_files/spark/hash/sha1.slt b/datafusion/sqllogictest/test_files/spark/hash/sha1.sltindex 1ce7346160..5185c45d09 100644--- a/datafusion/sqllogictest/test_files/spark/hash/sha1.slt+++ b/datafusion/sqllogictest/test_files/spark/hash/sha1.slt@@ -31,40 +31,51 @@ SELECT sha1('Spark');85f5955f4b27a9a4c2aab6ffe5d7189fc298b92cquery T-SELECT sha1(NULL);+SELECT sha('Spark');-----NULL+85f5955f4b27a9a4c2aab6ffe5d7189fc298b92cquery TSELECT sha1('');----da39a3ee5e6b4b0d3255bfef95601890afd80709-# Test with LargeUtf8 (using CAST to ensure type)+# Test with different types+query T+SELECT sha1(NULL);+----+NULL+query TSELECT sha1(arrow_cast('Spark', 'LargeUtf8'));----85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c-# Test with Utf8View (using CAST to ensure type)query TSELECT sha1(arrow_cast('Spark', 'Utf8View'));----85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c-# Test with Binary+query T+SELECT sha1(arrow_cast('Spark', 'Utf8'));+----+85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c+query TSELECT sha1(arrow_cast('Spark', 'Binary'));----85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c-# Test with LargeBinary+query T+SELECT sha1(arrow_cast(arrow_cast('Spark', 'Binary'), 'FixedSizeBinary(5)'));+----+85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c+query TSELECT sha1(arrow_cast('Spark', 'LargeBinary'));----85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c-# Test with BinaryViewquery TSELECT sha1(arrow_cast('Spark', 'BinaryView'));----