Apache DataFusion SQL Query Engine
fix: derive nullability for spark `bit_get` (#19220)
Kumar Ujjawal9163feed6b62
diff --git a/datafusion/spark/src/function/bitwise/bit_get.rs b/datafusion/spark/src/function/bitwise/bit_get.rsindex bc8d8cdbd1..f8980a697b 100644--- a/datafusion/spark/src/function/bitwise/bit_get.rs+++ b/datafusion/spark/src/function/bitwise/bit_get.rs@@ -24,13 +24,13 @@ use arrow::array::{Int8Array, PrimitiveArray,};use arrow::compute::try_binary;-use arrow::datatypes::{ArrowNativeType, DataType, Int32Type, Int8Type};+use arrow::datatypes::{ArrowNativeType, DataType, Field, FieldRef, Int32Type, Int8Type};use datafusion_common::types::{logical_int32, NativeType};use datafusion_common::utils::take_function_args;use datafusion_common::{internal_err, Result};use datafusion_expr::{- Coercion, ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature,- TypeSignatureClass, Volatility,+ Coercion, ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl,+ Signature, TypeSignatureClass, Volatility,};use datafusion_functions::utils::make_scalar_function;@@ -83,7 +83,13 @@ impl ScalarUDFImpl for SparkBitGet {}fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {- Ok(DataType::Int8)+ internal_err!("return_field_from_args should be used instead")+ }++ fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {+ // Spark derives nullability for BinaryExpression from its children+ let nullable = args.arg_fields.iter().any(|f| f.is_nullable());+ Ok(Arc::new(Field::new(self.name(), DataType::Int8, nullable)))}fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {@@ -118,3 +124,43 @@ fn spark_bit_get(args: &[ArrayRef]) -> Result<ArrayRef> {)?;Ok(Arc::new(ret))}++#[cfg(test)]+mod tests {+ use super::*;+ use arrow::datatypes::Field;++ #[test]+ fn test_bit_get_nullability_non_nullable_inputs() {+ let func = SparkBitGet::new();+ let value_field = Arc::new(Field::new("value", DataType::Int32, false));+ let pos_field = Arc::new(Field::new("pos", DataType::Int32, false));++ let out_field = func+ .return_field_from_args(ReturnFieldArgs {+ arg_fields: &[value_field, pos_field],+ scalar_arguments: &[None, None],+ })+ .unwrap();++ assert_eq!(out_field.data_type(), &DataType::Int8);+ assert!(!out_field.is_nullable());+ }++ #[test]+ fn test_bit_get_nullability_nullable_inputs() {+ let func = SparkBitGet::new();+ let value_field = Arc::new(Field::new("value", DataType::Int32, true));+ let pos_field = Arc::new(Field::new("pos", DataType::Int32, false));++ let out_field = func+ .return_field_from_args(ReturnFieldArgs {+ arg_fields: &[value_field, pos_field],+ scalar_arguments: &[None, None],+ })+ .unwrap();++ assert_eq!(out_field.data_type(), &DataType::Int8);+ assert!(out_field.is_nullable());+ }+}