Apache DataFusion SQL Query Engine
fix : correct nullability propagation for spark.bitwise_not (#19224)
shifluxxc06f27e94d927
diff --git a/datafusion/spark/src/function/bitwise/bitwise_not.rs b/datafusion/spark/src/function/bitwise/bitwise_not.rsindex 2f3fe22783..5710fa4fdd 100644--- a/datafusion/spark/src/function/bitwise/bitwise_not.rs+++ b/datafusion/spark/src/function/bitwise/bitwise_not.rs@@ -15,12 +15,14 @@// specific language governing permissions and limitations// under the License.+use arrow::array::*;use arrow::compute::kernels::bitwise;-use arrow::datatypes::{Int16Type, Int32Type, Int64Type, Int8Type};-use arrow::{array::*, datatypes::DataType};-use datafusion_common::{plan_err, Result};+use arrow::datatypes::{+ DataType, Field, FieldRef, Int16Type, Int32Type, Int64Type, Int8Type,+};+use datafusion_common::{internal_err, plan_err, Result};use datafusion_expr::{ColumnarValue, TypeSignature, Volatility};-use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl, Signature};+use datafusion_expr::{ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature};use datafusion_functions::utils::make_scalar_function;use std::{any::Any, sync::Arc};@@ -64,8 +66,32 @@ impl ScalarUDFImpl for SparkBitwiseNot {&self.signature}- fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {- Ok(arg_types[0].clone())+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {+ internal_err!(+ "SparkBitwiseNot: return_type() is not used; return_field_from_args() is implemented"+ )+ }++ fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {+ if args.arg_fields.len() != 1 {+ return plan_err!("bitwise_not expects exactly 1 argument");+ }++ let input_field = &args.arg_fields[0];++ let out_dt = input_field.data_type().clone();+ let mut out_nullable = input_field.is_nullable();++ let scalar_null_present = args+ .scalar_arguments+ .iter()+ .any(|opt_s| opt_s.is_some_and(|sv| sv.is_null()));++ if scalar_null_present {+ out_nullable = true;+ }++ Ok(Arc::new(Field::new(self.name(), out_dt, out_nullable)))}fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {@@ -107,3 +133,95 @@ pub fn spark_bitwise_not(args: &[ArrayRef]) -> Result<ArrayRef> {}}}++#[cfg(test)]+mod tests {+ use super::*;+ use arrow::datatypes::{DataType, Field};+ use std::sync::Arc;++ use datafusion_expr::ReturnFieldArgs;++ #[test]+ fn test_bitwise_not_nullability() {+ let bitwise_not = SparkBitwiseNot::new();++ // --- non-nullable Int32 input ---+ let non_nullable_i32 = Arc::new(Field::new("c", DataType::Int32, false));+ let out_non_null = bitwise_not+ .return_field_from_args(ReturnFieldArgs {+ arg_fields: &[Arc::clone(&non_nullable_i32)],+ // single-argument function -> one scalar_argument slot (None)+ scalar_arguments: &[None],+ })+ .unwrap();++ // result should be non-nullable and the same DataType as input+ assert!(!out_non_null.is_nullable());+ assert_eq!(out_non_null.data_type(), &DataType::Int32);++ // --- nullable Int32 input ---+ let nullable_i32 = Arc::new(Field::new("c", DataType::Int32, true));+ let out_nullable = bitwise_not+ .return_field_from_args(ReturnFieldArgs {+ arg_fields: &[Arc::clone(&nullable_i32)],+ scalar_arguments: &[None],+ })+ .unwrap();++ // result should be nullable and the same DataType as input+ assert!(out_nullable.is_nullable());+ assert_eq!(out_nullable.data_type(), &DataType::Int32);++ // --- also test another integer type (Int64) for completeness ---+ let non_nullable_i64 = Arc::new(Field::new("c", DataType::Int64, false));+ let out_i64 = bitwise_not+ .return_field_from_args(ReturnFieldArgs {+ arg_fields: &[Arc::clone(&non_nullable_i64)],+ scalar_arguments: &[None],+ })+ .unwrap();++ assert!(!out_i64.is_nullable());+ assert_eq!(out_i64.data_type(), &DataType::Int64);++ let nullable_i64 = Arc::new(Field::new("c", DataType::Int64, true));+ let out_i64_null = bitwise_not+ .return_field_from_args(ReturnFieldArgs {+ arg_fields: &[Arc::clone(&nullable_i64)],+ scalar_arguments: &[None],+ })+ .unwrap();++ assert!(out_i64_null.is_nullable());+ assert_eq!(out_i64_null.data_type(), &DataType::Int64);+ }++ #[test]+ fn test_bitwise_not_nullability_with_null_scalar() -> Result<()> {+ use arrow::datatypes::{DataType, Field};+ use datafusion_common::ScalarValue;+ use std::sync::Arc;++ let func = SparkBitwiseNot::new();++ let non_nullable: FieldRef = Arc::new(Field::new("col", DataType::Int32, false));++ let out = func.return_field_from_args(ReturnFieldArgs {+ arg_fields: &[Arc::clone(&non_nullable)],+ scalar_arguments: &[None],+ })?;+ assert!(!out.is_nullable());+ assert_eq!(out.data_type(), &DataType::Int32);++ let null_scalar = ScalarValue::Int32(None);+ let out_with_null_scalar = func.return_field_from_args(ReturnFieldArgs {+ arg_fields: &[Arc::clone(&non_nullable)],+ scalar_arguments: &[Some(&null_scalar)],+ })?;+ assert!(out_with_null_scalar.is_nullable());+ assert_eq!(out_with_null_scalar.data_type(), &DataType::Int32);++ Ok(())+ }+}