Skip to content

Apache DataFusion SQL Query Engine

fix: derive custom nullability for spark `bit_shift` (#19222)

Kumar Ujjawalea9f35412371
diff --git a/datafusion/spark/src/function/bitwise/bit_shift.rs b/datafusion/spark/src/function/bitwise/bit_shift.rs
index 65df048580..ff7f7662ec 100644
--- a/datafusion/spark/src/function/bitwise/bit_shift.rs
+++ b/datafusion/spark/src/function/bitwise/bit_shift.rs
@@ -21,7 +21,8 @@ use std::sync::Arc;
use arrow::array::{ArrayRef, ArrowPrimitiveType, AsArray, Int32Array, PrimitiveArray};
use arrow::compute;
use arrow::datatypes::{
- ArrowNativeType, DataType, Int32Type, Int64Type, UInt32Type, UInt64Type,
+ ArrowNativeType, DataType, Field, FieldRef, Int32Type, Int64Type, UInt32Type,
+ UInt64Type,
};
use datafusion_common::types::{
logical_int16, logical_int32, logical_int64, logical_int8, logical_uint16,
@@ -30,8 +31,8 @@ use datafusion_common::types::{
use datafusion_common::utils::take_function_args;
use datafusion_common::{internal_err, Result};
use datafusion_expr::{
- Coercion, ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature,
- TypeSignatureClass, Volatility,
+ Coercion, ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl,
+ Signature, TypeSignature, TypeSignatureClass, Volatility,
};
use datafusion_functions::utils::make_scalar_function;
@@ -275,8 +276,14 @@ impl ScalarUDFImpl for SparkBitShift {
&self.signature
}
- fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
- Ok(arg_types[0].clone())
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ internal_err!("return_field_from_args should be used instead")
+ }
+
+ fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {
+ let nullable = args.arg_fields.iter().any(|f| f.is_nullable());
+ let data_type = args.arg_fields[0].data_type().clone();
+ Ok(Arc::new(Field::new(self.name(), data_type, nullable)))
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
@@ -286,3 +293,58 @@ impl ScalarUDFImpl for SparkBitShift {
make_scalar_function(inner, vec![])(&args.args)
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use arrow::datatypes::Field;
+ use datafusion_expr::ReturnFieldArgs;
+
+ #[test]
+ fn test_bit_shift_nullability() -> Result<()> {
+ let func = SparkBitShift::left();
+
+ let non_nullable_value: FieldRef =
+ Arc::new(Field::new("value", DataType::Int64, false));
+ let non_nullable_shift: FieldRef =
+ Arc::new(Field::new("shift", DataType::Int32, false));
+
+ let out = func.return_field_from_args(ReturnFieldArgs {
+ arg_fields: &[
+ Arc::clone(&non_nullable_value),
+ Arc::clone(&non_nullable_shift),
+ ],
+ scalar_arguments: &[None, None],
+ })?;
+
+ assert_eq!(out.data_type(), non_nullable_value.data_type());
+ assert!(
+ !out.is_nullable(),
+ "shift result should be non-nullable when both inputs are non-nullable"
+ );
+
+ let nullable_value: FieldRef =
+ Arc::new(Field::new("value", DataType::Int64, true));
+ let out_nullable_value = func.return_field_from_args(ReturnFieldArgs {
+ arg_fields: &[Arc::clone(&nullable_value), Arc::clone(&non_nullable_shift)],
+ scalar_arguments: &[None, None],
+ })?;
+ assert!(
+ out_nullable_value.is_nullable(),
+ "shift result should be nullable when value is nullable"
+ );
+
+ let nullable_shift: FieldRef =
+ Arc::new(Field::new("shift", DataType::Int32, true));
+ let out_nullable_shift = func.return_field_from_args(ReturnFieldArgs {
+ arg_fields: &[non_nullable_value, nullable_shift],
+ scalar_arguments: &[None, None],
+ })?;
+ assert!(
+ out_nullable_shift.is_nullable(),
+ "shift result should be nullable when shift is nullable"
+ );
+
+ Ok(())
+ }
+}