Skip to content

Apache DataFusion SQL Query Engine

Refactor Spark crc32 & sha1 to remove unnecessary scalar argument check (#19466)

Jeffrey Vo67b526a6202a
diff --git a/datafusion/spark/src/function/hash/crc32.rs b/datafusion/spark/src/function/hash/crc32.rs
index a48d70c975..f079d82f9b 100644
--- a/datafusion/spark/src/function/hash/crc32.rs
+++ b/datafusion/spark/src/function/hash/crc32.rs
@@ -79,11 +79,7 @@ impl ScalarUDFImpl for SparkCrc32 {
}
fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {
- let nullable = args.arg_fields.iter().any(|f| f.is_nullable())
- || args
- .scalar_arguments
- .iter()
- .any(|scalar| scalar.is_some_and(|s| s.is_null()));
+ let nullable = args.arg_fields.iter().any(|f| f.is_nullable());
Ok(Arc::new(Field::new(self.name(), DataType::Int64, nullable)))
}
@@ -135,7 +131,6 @@ fn spark_crc32(args: &[ArrayRef]) -> Result<ArrayRef> {
#[cfg(test)]
mod tests {
use super::*;
- use datafusion_common::ScalarValue;
#[test]
fn test_crc32_nullability() -> Result<()> {
@@ -159,15 +154,6 @@ mod tests {
assert!(result.is_nullable());
assert_eq!(result.data_type(), &DataType::Int64);
- // null scalar value - user input literal NULL
- let scalar_null = ScalarValue::Binary(None);
- let result = crc32_func.return_field_from_args(ReturnFieldArgs {
- arg_fields: &[field_not_null],
- scalar_arguments: &[Some(&scalar_null)],
- })?;
- assert!(result.is_nullable());
- assert_eq!(result.data_type(), &DataType::Int64);
-
Ok(())
}
}
diff --git a/datafusion/spark/src/function/hash/sha1.rs b/datafusion/spark/src/function/hash/sha1.rs
index 2fd108a996..3250c8ca84 100644
--- a/datafusion/spark/src/function/hash/sha1.rs
+++ b/datafusion/spark/src/function/hash/sha1.rs
@@ -86,12 +86,7 @@ impl ScalarUDFImpl for SparkSha1 {
}
fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {
- let nullable = args.arg_fields.iter().any(|f| f.is_nullable())
- || args
- .scalar_arguments
- .iter()
- .any(|scalar| scalar.is_some_and(|s| s.is_null()));
-
+ let nullable = args.arg_fields.iter().any(|f| f.is_nullable());
Ok(Arc::new(Field::new(self.name(), DataType::Utf8, nullable)))
}
@@ -146,7 +141,6 @@ fn spark_sha1(args: &[ArrayRef]) -> Result<ArrayRef> {
#[cfg(test)]
mod tests {
use super::*;
- use datafusion_common::ScalarValue;
#[test]
fn test_sha1_nullability() -> Result<()> {
@@ -170,15 +164,6 @@ mod tests {
assert!(out.is_nullable());
assert_eq!(out.data_type(), &DataType::Utf8);
- // Null scalar argument also makes output nullable
- let null_scalar = ScalarValue::Binary(None);
- let out = func.return_field_from_args(ReturnFieldArgs {
- arg_fields: &[Arc::clone(&non_nullable)],
- scalar_arguments: &[Some(&null_scalar)],
- })?;
- assert!(out.is_nullable());
- assert_eq!(out.data_type(), &DataType::Utf8);
-
Ok(())
}
}