Apache DataFusion SQL Query Engine
Refactor Spark crc32 & sha1 to remove unnecessary scalar argument check (#19466)
Jeffrey Vo67b526a6202a
diff --git a/datafusion/spark/src/function/hash/crc32.rs b/datafusion/spark/src/function/hash/crc32.rsindex a48d70c975..f079d82f9b 100644--- a/datafusion/spark/src/function/hash/crc32.rs+++ b/datafusion/spark/src/function/hash/crc32.rs@@ -79,11 +79,7 @@ impl ScalarUDFImpl for SparkCrc32 {}fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {- let nullable = args.arg_fields.iter().any(|f| f.is_nullable())- || args- .scalar_arguments- .iter()- .any(|scalar| scalar.is_some_and(|s| s.is_null()));+ let nullable = args.arg_fields.iter().any(|f| f.is_nullable());Ok(Arc::new(Field::new(self.name(), DataType::Int64, nullable)))}@@ -135,7 +131,6 @@ fn spark_crc32(args: &[ArrayRef]) -> Result<ArrayRef> {#[cfg(test)]mod tests {use super::*;- use datafusion_common::ScalarValue;#[test]fn test_crc32_nullability() -> Result<()> {@@ -159,15 +154,6 @@ mod tests {assert!(result.is_nullable());assert_eq!(result.data_type(), &DataType::Int64);- // null scalar value - user input literal NULL- let scalar_null = ScalarValue::Binary(None);- let result = crc32_func.return_field_from_args(ReturnFieldArgs {- arg_fields: &[field_not_null],- scalar_arguments: &[Some(&scalar_null)],- })?;- assert!(result.is_nullable());- assert_eq!(result.data_type(), &DataType::Int64);-Ok(())}}diff --git a/datafusion/spark/src/function/hash/sha1.rs b/datafusion/spark/src/function/hash/sha1.rsindex 2fd108a996..3250c8ca84 100644--- a/datafusion/spark/src/function/hash/sha1.rs+++ b/datafusion/spark/src/function/hash/sha1.rs@@ -86,12 +86,7 @@ impl ScalarUDFImpl for SparkSha1 {}fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {- let nullable = args.arg_fields.iter().any(|f| f.is_nullable())- || args- .scalar_arguments- .iter()- .any(|scalar| scalar.is_some_and(|s| s.is_null()));-+ let nullable = args.arg_fields.iter().any(|f| f.is_nullable());Ok(Arc::new(Field::new(self.name(), DataType::Utf8, nullable)))}@@ -146,7 +141,6 @@ fn spark_sha1(args: &[ArrayRef]) -> Result<ArrayRef> {#[cfg(test)]mod tests {use super::*;- use datafusion_common::ScalarValue;#[test]fn test_sha1_nullability() -> Result<()> {@@ -170,15 +164,6 @@ mod tests {assert!(out.is_nullable());assert_eq!(out.data_type(), &DataType::Utf8);- // Null scalar argument also makes output nullable- let null_scalar = ScalarValue::Binary(None);- let out = func.return_field_from_args(ReturnFieldArgs {- arg_fields: &[Arc::clone(&non_nullable)],- scalar_arguments: &[Some(&null_scalar)],- })?;- assert!(out.is_nullable());- assert_eq!(out.data_type(), &DataType::Utf8);-Ok(())}}