Skip to content

Apache DataFusion SQL Query Engine

Refactor Spark bitshift signature (#18649)

Jeffrey Vo35b5363777a9
diff --git a/datafusion/functions/src/macros.rs b/datafusion/functions/src/macros.rs
index 9e195f2d52..077690cd3f 100644
--- a/datafusion/functions/src/macros.rs
+++ b/datafusion/functions/src/macros.rs
@@ -77,7 +77,8 @@ macro_rules! export_functions {
}
/// Creates a singleton `ScalarUDF` of the `$UDF` function and a function
-/// named `$NAME` which returns that singleton.
+/// named `$NAME` which returns that singleton. Optionally use a custom constructor
+/// `$CTOR` which defaults to `$UDF::new()` if not specified.
///
/// This is used to ensure creating the list of `ScalarUDF` only happens once.
#[macro_export]
@@ -97,6 +98,21 @@ macro_rules! make_udf_function {
std::sync::Arc::clone(&INSTANCE)
}
};
+ ($UDF:ty, $NAME:ident, $CTOR:path) => {
+ #[allow(rustdoc::redundant_explicit_links)]
+ #[doc = concat!("Return a [`ScalarUDF`](datafusion_expr::ScalarUDF) implementation of ", stringify!($NAME))]
+ pub fn $NAME() -> std::sync::Arc<datafusion_expr::ScalarUDF> {
+ // Singleton instance of the function
+ static INSTANCE: std::sync::LazyLock<
+ std::sync::Arc<datafusion_expr::ScalarUDF>,
+ > = std::sync::LazyLock::new(|| {
+ std::sync::Arc::new(datafusion_expr::ScalarUDF::new_from_impl(
+ $CTOR(),
+ ))
+ });
+ std::sync::Arc::clone(&INSTANCE)
+ }
+ };
}
/// Creates a singleton `ScalarUDF` of the `$UDF` function and a function
diff --git a/datafusion/spark/src/function/bitwise/bit_shift.rs b/datafusion/spark/src/function/bitwise/bit_shift.rs
index 68911b0492..65df048580 100644
--- a/datafusion/spark/src/function/bitwise/bit_shift.rs
+++ b/datafusion/spark/src/function/bitwise/bit_shift.rs
@@ -18,36 +18,33 @@
use std::any::Any;
use std::sync::Arc;
-use arrow::array::{ArrayRef, ArrowPrimitiveType, AsArray, PrimitiveArray};
+use arrow::array::{ArrayRef, ArrowPrimitiveType, AsArray, Int32Array, PrimitiveArray};
use arrow::compute;
use arrow::datatypes::{
ArrowNativeType, DataType, Int32Type, Int64Type, UInt32Type, UInt64Type,
};
-use datafusion_common::{plan_err, Result};
+use datafusion_common::types::{
+ logical_int16, logical_int32, logical_int64, logical_int8, logical_uint16,
+ logical_uint32, logical_uint64, logical_uint8, NativeType,
+};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{internal_err, Result};
use datafusion_expr::{
- ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
+ Coercion, ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature,
+ TypeSignatureClass, Volatility,
};
use datafusion_functions::utils::make_scalar_function;
-use crate::function::error_utils::{
- invalid_arg_count_exec_err, unsupported_data_type_exec_err,
-};
-
-/// Performs a bitwise left shift on each element of the `value` array by the corresponding amount in the `shift` array.
-/// The shift amount is normalized to the bit width of the type, matching Spark/Java semantics for negative and large shifts.
-///
-/// # Arguments
-/// * `value` - The array of values to shift.
-/// * `shift` - The array of shift amounts (must be Int32).
-///
-/// # Returns
-/// A new array with the shifted values.
-fn shift_left<T: ArrowPrimitiveType>(
+/// Bitwise left shift on elements in `value` by corresponding `shift` amount.
+/// The shift amount is normalized to the bit width of the type, matching Spark/Java
+/// semantics for negative and large shifts.
+fn shift_left<T>(
value: &PrimitiveArray<T>,
- shift: &PrimitiveArray<Int32Type>,
+ shift: &Int32Array,
) -> Result<PrimitiveArray<T>>
where
- T::Native: ArrowNativeType + std::ops::Shl<i32, Output = T::Native>,
+ T: ArrowPrimitiveType,
+ T::Native: std::ops::Shl<i32, Output = T::Native>,
{
let bit_num = (T::Native::get_byte_width() * 8) as i32;
let result = compute::binary::<_, Int32Type, _, _>(
@@ -61,21 +58,16 @@ where
Ok(result)
}
-/// Performs a bitwise right shift on each element of the `value` array by the corresponding amount in the `shift` array.
-/// The shift amount is normalized to the bit width of the type, matching Spark/Java semantics for negative and large shifts.
-///
-/// # Arguments
-/// * `value` - The array of values to shift.
-/// * `shift` - The array of shift amounts (must be Int32).
-///
-/// # Returns
-/// A new array with the shifted values.
-fn shift_right<T: ArrowPrimitiveType>(
+/// Bitwise right shift on elements in `value` by corresponding `shift` amount.
+/// The shift amount is normalized to the bit width of the type, matching Spark/Java
+/// semantics for negative and large shifts.
+fn shift_right<T>(
value: &PrimitiveArray<T>,
- shift: &PrimitiveArray<Int32Type>,
+ shift: &Int32Array,
) -> Result<PrimitiveArray<T>>
where
- T::Native: ArrowNativeType + std::ops::Shr<i32, Output = T::Native>,
+ T: ArrowPrimitiveType,
+ T::Native: std::ops::Shr<i32, Output = T::Native>,
{
let bit_num = (T::Native::get_byte_width() * 8) as i32;
let result = compute::binary::<_, Int32Type, _, _>(
@@ -93,49 +85,44 @@ where
/// This is used to mimic Java's `>>>` operator, which does not exist in Rust.
/// For unsigned types, this is just the normal right shift.
/// For signed types, this casts to the unsigned type, shifts, then casts back.
-trait UShr<Rhs> {
- fn ushr(self, rhs: Rhs) -> Self;
+trait UShr {
+ fn ushr(self, rhs: i32) -> Self;
}
-impl UShr<i32> for u32 {
+impl UShr for u32 {
fn ushr(self, rhs: i32) -> Self {
self >> rhs
}
}
-impl UShr<i32> for u64 {
+impl UShr for u64 {
fn ushr(self, rhs: i32) -> Self {
self >> rhs
}
}
-impl UShr<i32> for i32 {
+impl UShr for i32 {
fn ushr(self, rhs: i32) -> Self {
((self as u32) >> rhs) as i32
}
}
-impl UShr<i32> for i64 {
+impl UShr for i64 {
fn ushr(self, rhs: i32) -> Self {
((self as u64) >> rhs) as i64
}
}
-/// Performs a bitwise unsigned right shift on each element of the `value` array by the corresponding amount in the `shift` array.
-/// The shift amount is normalized to the bit width of the type, matching Spark/Java semantics for negative and large shifts.
-///
-/// # Arguments
-/// * `value` - The array of values to shift.
-/// * `shift` - The array of shift amounts (must be Int32).
-///
-/// # Returns
-/// A new array with the shifted values.
-fn shift_right_unsigned<T: ArrowPrimitiveType>(
+/// Bitwise unsigned right shift on elements in `value` by corresponding `shift`
+/// amount. The shift amount is normalized to the bit width of the type, matching
+/// Spark/Java semantics for negative and large shifts.
+fn shift_right_unsigned<T>(
value: &PrimitiveArray<T>,
- shift: &PrimitiveArray<Int32Type>,
+ shift: &Int32Array,
) -> Result<PrimitiveArray<T>>
where
- T::Native: ArrowNativeType + UShr<i32>,
+ T: ArrowPrimitiveType,
+ T::Native: UShr,
{
let bit_num = (T::Native::get_byte_width() * 8) as i32;
let result = compute::binary::<_, Int32Type, _, _>(
@@ -149,589 +136,153 @@ where
Ok(result)
}
-trait BitShiftUDF: ScalarUDFImpl {
- fn shift<T: ArrowPrimitiveType>(
- &self,
- value: &PrimitiveArray<T>,
- shift: &PrimitiveArray<Int32Type>,
- ) -> Result<PrimitiveArray<T>>
- where
- T::Native: ArrowNativeType
- + std::ops::Shl<i32, Output = T::Native>
- + std::ops::Shr<i32, Output = T::Native>
- + UShr<i32>;
-
- fn spark_shift(&self, arrays: &[ArrayRef]) -> Result<ArrayRef> {
- let value_array = arrays[0].as_ref();
- let shift_array = arrays[1].as_ref();
-
- // Ensure shift array is Int32
- let shift_array = if shift_array.data_type() != &DataType::Int32 {
- return plan_err!("{} shift amount must be Int32", self.name());
- } else {
- shift_array.as_primitive::<Int32Type>()
- };
-
- match value_array.data_type() {
- DataType::Int32 => {
- let value_array = value_array.as_primitive::<Int32Type>();
- Ok(Arc::new(self.shift(value_array, shift_array)?))
- }
- DataType::Int64 => {
- let value_array = value_array.as_primitive::<Int64Type>();
- Ok(Arc::new(self.shift(value_array, shift_array)?))
- }
- DataType::UInt32 => {
- let value_array = value_array.as_primitive::<UInt32Type>();
- Ok(Arc::new(self.shift(value_array, shift_array)?))
- }
- DataType::UInt64 => {
- let value_array = value_array.as_primitive::<UInt64Type>();
- Ok(Arc::new(self.shift(value_array, shift_array)?))
- }
- _ => {
- plan_err!(
- "{} function does not support data type: {}",
- self.name(),
- value_array.data_type()
- )
- }
- }
- }
-}
-
-fn bit_shift_coerce_types(arg_types: &[DataType], func: &str) -> Result<Vec<DataType>> {
- if arg_types.len() != 2 {
- return Err(invalid_arg_count_exec_err(func, (2, 2), arg_types.len()));
- }
- if !arg_types[0].is_integer() && !arg_types[0].is_null() {
- return Err(unsupported_data_type_exec_err(
- func,
- "Integer Type",
- &arg_types[0],
- ));
- }
- if !arg_types[1].is_integer() && !arg_types[1].is_null() {
- return Err(unsupported_data_type_exec_err(
- func,
- "Integer Type",
- &arg_types[1],
- ));
- }
-
- // Coerce smaller integer types to Int32
- let coerced_first = match &arg_types[0] {
- DataType::Int8 | DataType::Int16 | DataType::Null => DataType::Int32,
- DataType::UInt8 | DataType::UInt16 => DataType::UInt32,
- _ => arg_types[0].clone(),
- };
-
- Ok(vec![coerced_first, DataType::Int32])
-}
-
-#[derive(Debug, Hash, Eq, PartialEq)]
-pub struct SparkShiftLeft {
- signature: Signature,
-}
-
-impl Default for SparkShiftLeft {
- fn default() -> Self {
- Self::new()
- }
-}
-
-impl SparkShiftLeft {
- pub fn new() -> Self {
- Self {
- signature: Signature::user_defined(Volatility::Immutable),
- }
- }
-}
+fn shift_inner(
+ arrays: &[ArrayRef],
+ name: &str,
+ bit_shift_type: BitShiftType,
+) -> Result<ArrayRef> {
+ let [value_array, shift_array] = take_function_args(name, arrays)?;
+ let shift_array = shift_array.as_primitive::<Int32Type>();
-impl BitShiftUDF for SparkShiftLeft {
- fn shift<T: ArrowPrimitiveType>(
- &self,
+ fn shift<T>(
value: &PrimitiveArray<T>,
- shift: &PrimitiveArray<Int32Type>,
+ shift: &Int32Array,
+ bit_shift_type: BitShiftType,
) -> Result<PrimitiveArray<T>>
where
- T::Native: ArrowNativeType
- + std::ops::Shl<i32, Output = T::Native>
+ T: ArrowPrimitiveType,
+ T::Native: std::ops::Shl<i32, Output = T::Native>
+ std::ops::Shr<i32, Output = T::Native>
- + UShr<i32>,
+ + UShr,
{
- shift_left(value, shift)
- }
-}
-
-impl ScalarUDFImpl for SparkShiftLeft {
- fn as_any(&self) -> &dyn Any {
- self
- }
-
- fn name(&self) -> &str {
- "shiftleft"
- }
-
- fn signature(&self) -> &Signature {
- &self.signature
- }
-
- fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
- bit_shift_coerce_types(arg_types, "shiftleft")
- }
-
- fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
- if arg_types.len() != 2 {
- return plan_err!("shiftleft expects exactly 2 arguments");
+ match bit_shift_type {
+ BitShiftType::Left => shift_left(value, shift),
+ BitShiftType::Right => shift_right(value, shift),
+ BitShiftType::RightUnsigned => shift_right_unsigned(value, shift),
}
- // Return type is the same as the first argument (the value to shift)
- Ok(arg_types[0].clone())
}
- fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
- if args.args.len() != 2 {
- return plan_err!("shiftleft expects exactly 2 arguments");
+ match value_array.data_type() {
+ DataType::Int32 => {
+ let value_array = value_array.as_primitive::<Int32Type>();
+ Ok(Arc::new(shift(value_array, shift_array, bit_shift_type)?))
+ }
+ DataType::Int64 => {
+ let value_array = value_array.as_primitive::<Int64Type>();
+ Ok(Arc::new(shift(value_array, shift_array, bit_shift_type)?))
+ }
+ DataType::UInt32 => {
+ let value_array = value_array.as_primitive::<UInt32Type>();
+ Ok(Arc::new(shift(value_array, shift_array, bit_shift_type)?))
+ }
+ DataType::UInt64 => {
+ let value_array = value_array.as_primitive::<UInt64Type>();
+ Ok(Arc::new(shift(value_array, shift_array, bit_shift_type)?))
+ }
+ dt => {
+ internal_err!("{name} function does not support data type: {dt}")
}
- let inner = |arr: &[ArrayRef]| -> Result<ArrayRef> { self.spark_shift(arr) };
- make_scalar_function(inner, vec![])(&args.args)
}
}
-#[derive(Debug, Hash, Eq, PartialEq)]
-pub struct SparkShiftRightUnsigned {
- signature: Signature,
+#[derive(Debug, Hash, Copy, Clone, Eq, PartialEq)]
+enum BitShiftType {
+ Left,
+ Right,
+ RightUnsigned,
}
-impl Default for SparkShiftRightUnsigned {
- fn default() -> Self {
- Self::new()
- }
+#[derive(Debug, Hash, Eq, PartialEq)]
+pub struct SparkBitShift {
+ signature: Signature,
+ name: &'static str,
+ bit_shift_type: BitShiftType,
}
-impl SparkShiftRightUnsigned {
- pub fn new() -> Self {
+impl SparkBitShift {
+ fn new(name: &'static str, bit_shift_type: BitShiftType) -> Self {
+ let shift_amount = Coercion::new_implicit(
+ TypeSignatureClass::Native(logical_int32()),
+ vec![TypeSignatureClass::Integer],
+ NativeType::Int32,
+ );
Self {
- signature: Signature::user_defined(Volatility::Immutable),
+ signature: Signature::one_of(
+ vec![
+ // Upcast small ints to 32bit
+ TypeSignature::Coercible(vec![
+ Coercion::new_implicit(
+ TypeSignatureClass::Native(logical_int32()),
+ vec![
+ TypeSignatureClass::Native(logical_int8()),
+ TypeSignatureClass::Native(logical_int16()),
+ ],
+ NativeType::Int32,
+ ),
+ shift_amount.clone(),
+ ]),
+ TypeSignature::Coercible(vec![
+ Coercion::new_implicit(
+ TypeSignatureClass::Native(logical_uint32()),
+ vec![
+ TypeSignatureClass::Native(logical_uint8()),
+ TypeSignatureClass::Native(logical_uint16()),
+ ],
+ NativeType::UInt32,
+ ),
+ shift_amount.clone(),
+ ]),
+ // Otherwise accept direct 64 bit integers
+ TypeSignature::Coercible(vec![
+ Coercion::new_exact(TypeSignatureClass::Native(logical_int64())),
+ shift_amount.clone(),
+ ]),
+ TypeSignature::Coercible(vec![
+ Coercion::new_exact(TypeSignatureClass::Native(logical_uint64())),
+ shift_amount.clone(),
+ ]),
+ ],
+ Volatility::Immutable,
+ ),
+ name,
+ bit_shift_type,
}
}
-}
-
-impl BitShiftUDF for SparkShiftRightUnsigned {
- fn shift<T: ArrowPrimitiveType>(
- &self,
- value: &PrimitiveArray<T>,
- shift: &PrimitiveArray<Int32Type>,
- ) -> Result<PrimitiveArray<T>>
- where
- T::Native: ArrowNativeType
- + std::ops::Shl<i32, Output = T::Native>
- + std::ops::Shr<i32, Output = T::Native>
- + UShr<i32>,
- {
- shift_right_unsigned(value, shift)
- }
-}
-
-impl ScalarUDFImpl for SparkShiftRightUnsigned {
- fn as_any(&self) -> &dyn Any {
- self
- }
- fn name(&self) -> &str {
- "shiftrightunsigned"
+ pub fn left() -> Self {
+ Self::new("shiftleft", BitShiftType::Left)
}
- fn signature(&self) -> &Signature {
- &self.signature
+ pub fn right() -> Self {
+ Self::new("shiftright", BitShiftType::Right)
}
- fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
- bit_shift_coerce_types(arg_types, "shiftrightunsigned")
- }
-
- fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
- if arg_types.len() != 2 {
- return plan_err!("shiftrightunsigned expects exactly 2 arguments");
- }
- // Return type is the same as the first argument (the value to shift)
- Ok(arg_types[0].clone())
- }
-
- fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
- if args.args.len() != 2 {
- return plan_err!("shiftrightunsigned expects exactly 2 arguments");
- }
- let inner = |arr: &[ArrayRef]| -> Result<ArrayRef> { self.spark_shift(arr) };
- make_scalar_function(inner, vec![])(&args.args)
- }
-}
-
-#[derive(Debug, Hash, Eq, PartialEq)]
-pub struct SparkShiftRight {
- signature: Signature,
-}
-
-impl Default for SparkShiftRight {
- fn default() -> Self {
- Self::new()
- }
-}
-
-impl SparkShiftRight {
- pub fn new() -> Self {
- Self {
- signature: Signature::user_defined(Volatility::Immutable),
- }
+ pub fn right_unsigned() -> Self {
+ Self::new("shiftrightunsigned", BitShiftType::RightUnsigned)
}
}
-impl BitShiftUDF for SparkShiftRight {
- fn shift<T: ArrowPrimitiveType>(
- &self,
- value: &PrimitiveArray<T>,
- shift: &PrimitiveArray<Int32Type>,
- ) -> Result<PrimitiveArray<T>>
- where
- T::Native: ArrowNativeType
- + std::ops::Shl<i32, Output = T::Native>
- + std::ops::Shr<i32, Output = T::Native>
- + UShr<i32>,
- {
- shift_right(value, shift)
- }
-}
-
-impl ScalarUDFImpl for SparkShiftRight {
+impl ScalarUDFImpl for SparkBitShift {
fn as_any(&self) -> &dyn Any {
self
}
fn name(&self) -> &str {
- "shiftright"
+ self.name
}
fn signature(&self) -> &Signature {
&self.signature
}
- fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
- bit_shift_coerce_types(arg_types, "shiftright")
- }
-
fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
- if arg_types.len() != 2 {
- return plan_err!("shiftright expects exactly 2 arguments");
- }
- // Return type is the same as the first argument (the value to shift)
Ok(arg_types[0].clone())
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
- if args.args.len() != 2 {
- return plan_err!("shiftright expects exactly 2 arguments");
- }
- let inner = |arr: &[ArrayRef]| -> Result<ArrayRef> { self.spark_shift(arr) };
+ let inner = |arr: &[ArrayRef]| -> Result<ArrayRef> {
+ shift_inner(arr, self.name(), self.bit_shift_type)
+ };
make_scalar_function(inner, vec![])(&args.args)
}
}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use arrow::array::{Array, Int32Array, Int64Array, UInt32Array, UInt64Array};
-
- #[test]
- fn test_shift_right_unsigned_int32() {
- let value_array = Arc::new(Int32Array::from(vec![4, 8, 16, 32]));
- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));
- let result = SparkShiftRightUnsigned::new()
- .spark_shift(&[value_array, shift_array])
- .unwrap();
- let arr = result.as_primitive::<Int32Type>();
- assert_eq!(arr.value(0), 2); // 4 >>> 1 = 2
- assert_eq!(arr.value(1), 2); // 8 >>> 2 = 2
- assert_eq!(arr.value(2), 2); // 16 >>> 3 = 2
- assert_eq!(arr.value(3), 2); // 32 >>> 4 = 2
- }
-
- #[test]
- fn test_shift_right_unsigned_int64() {
- let value_array = Arc::new(Int64Array::from(vec![4i64, 8, 16]));
- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3]));
- let result = SparkShiftRightUnsigned::new()
- .spark_shift(&[value_array, shift_array])
- .unwrap();
- let arr = result.as_primitive::<Int64Type>();
- assert_eq!(arr.value(0), 2); // 4 >>> 1 = 2
- assert_eq!(arr.value(1), 2); // 8 >>> 2 = 2
- assert_eq!(arr.value(2), 2); // 16 >>> 3 = 2
- }
-
- #[test]
- fn test_shift_right_unsigned_uint32() {
- let value_array = Arc::new(UInt32Array::from(vec![4u32, 8, 16]));
- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3]));
- let result = SparkShiftRightUnsigned::new()
- .spark_shift(&[value_array, shift_array])
- .unwrap();
- let arr = result.as_primitive::<UInt32Type>();
- assert_eq!(arr.value(0), 2); // 4 >>> 1 = 2
- assert_eq!(arr.value(1), 2); // 8 >>> 2 = 2
- assert_eq!(arr.value(2), 2); // 16 >>> 3 = 2
- }
-
- #[test]
- fn test_shift_right_unsigned_uint64() {
- let value_array = Arc::new(UInt64Array::from(vec![4u64, 8, 16]));
- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3]));
- let result = SparkShiftRightUnsigned::new()
- .spark_shift(&[value_array, shift_array])
- .unwrap();
- let arr = result.as_primitive::<UInt64Type>();
- assert_eq!(arr.value(0), 2); // 4 >>> 1 = 2
- assert_eq!(arr.value(1), 2); // 8 >>> 2 = 2
- assert_eq!(arr.value(2), 2); // 16 >>> 3 = 2
- }
-
- #[test]
- fn test_shift_right_unsigned_nulls() {
- let value_array = Arc::new(Int32Array::from(vec![Some(4), None, Some(8)]));
- let shift_array = Arc::new(Int32Array::from(vec![Some(1), Some(2), None]));
- let result = SparkShiftRightUnsigned::new()
- .spark_shift(&[value_array, shift_array])
- .unwrap();
- let arr = result.as_primitive::<Int32Type>();
- assert_eq!(arr.value(0), 2); // 4 >>> 1 = 2
- assert!(arr.is_null(1)); // null >>> 2 = null
- assert!(arr.is_null(2)); // 8 >>> null = null
- }
-
- #[test]
- fn test_shift_right_unsigned_negative_shift() {
- let value_array = Arc::new(Int32Array::from(vec![4, 8, 16]));
- let shift_array = Arc::new(Int32Array::from(vec![-1, -2, -3]));
- let result = SparkShiftRightUnsigned::new()
- .spark_shift(&[value_array, shift_array])
- .unwrap();
- let arr = result.as_primitive::<Int32Type>();
- assert_eq!(arr.value(0), 0); // 4 >>> -1 = 0
- assert_eq!(arr.value(1), 0); // 8 >>> -2 = 0
- assert_eq!(arr.value(2), 0); // 16 >>> -3 = 0
- }
-
- #[test]
- fn test_shift_right_unsigned_negative_values() {
- let value_array = Arc::new(Int32Array::from(vec![-4, -8, -16]));
- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3]));
- let result = SparkShiftRightUnsigned::new()
- .spark_shift(&[value_array, shift_array])
- .unwrap();
- let arr = result.as_primitive::<Int32Type>();
- // For unsigned right shift, negative values are treated as large positive values
- // -4 as u32 = 4294967292, -4 >>> 1 = 2147483646
- assert_eq!(arr.value(0), 2147483646);
- // -8 as u32 = 4294967288, -8 >>> 2 = 1073741822
- assert_eq!(arr.value(1), 1073741822);
- // -16 as u32 = 4294967280, -16 >>> 3 = 536870910
- assert_eq!(arr.value(2), 536870910);
- }
-
- #[test]
- fn test_shift_right_int32() {
- let value_array = Arc::new(Int32Array::from(vec![4, 8, 16, 32]));
- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));
- let result = SparkShiftRight::new()
- .spark_shift(&[value_array, shift_array])
- .unwrap();
- let arr = result.as_primitive::<Int32Type>();
- assert_eq!(arr.value(0), 2); // 4 >> 1 = 2
- assert_eq!(arr.value(1), 2); // 8 >> 2 = 2
- assert_eq!(arr.value(2), 2); // 16 >> 3 = 2
- assert_eq!(arr.value(3), 2); // 32 >> 4 = 2
- }
-
- #[test]
- fn test_shift_right_int64() {
- let value_array = Arc::new(Int64Array::from(vec![4i64, 8, 16]));
- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3]));
- let result = SparkShiftRight::new()
- .spark_shift(&[value_array, shift_array])
- .unwrap();
- let arr = result.as_primitive::<Int64Type>();
- assert_eq!(arr.value(0), 2); // 4 >> 1 = 2
- assert_eq!(arr.value(1), 2); // 8 >> 2 = 2
- assert_eq!(arr.value(2), 2); // 16 >> 3 = 2
- }
-
- #[test]
- fn test_shift_right_uint32() {
- let value_array = Arc::new(UInt32Array::from(vec![4u32, 8, 16]));
- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3]));
- let result = SparkShiftRight::new()
- .spark_shift(&[value_array, shift_array])
- .unwrap();
- let arr = result.as_primitive::<UInt32Type>();
- assert_eq!(arr.value(0), 2); // 4 >> 1 = 2
- assert_eq!(arr.value(1), 2); // 8 >> 2 = 2
- assert_eq!(arr.value(2), 2); // 16 >> 3 = 2
- }
-
- #[test]
- fn test_shift_right_uint64() {
- let value_array = Arc::new(UInt64Array::from(vec![4u64, 8, 16]));
- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3]));
- let result = SparkShiftRight::new()
- .spark_shift(&[value_array, shift_array])
- .unwrap();
- let arr = result.as_primitive::<UInt64Type>();
- assert_eq!(arr.value(0), 2); // 4 >> 1 = 2
- assert_eq!(arr.value(1), 2); // 8 >> 2 = 2
- assert_eq!(arr.value(2), 2); // 16 >> 3 = 2
- }
-
- #[test]
- fn test_shift_right_nulls() {
- let value_array = Arc::new(Int32Array::from(vec![Some(4), None, Some(8)]));
- let shift_array = Arc::new(Int32Array::from(vec![Some(1), Some(2), None]));
- let result = SparkShiftRight::new()
- .spark_shift(&[value_array, shift_array])
- .unwrap();
- let arr = result.as_primitive::<Int32Type>();
- assert_eq!(arr.value(0), 2); // 4 >> 1 = 2
- assert!(arr.is_null(1)); // null >> 2 = null
- assert!(arr.is_null(2)); // 8 >> null = null
- }
-
- #[test]
- fn test_shift_right_large_shift() {
- let value_array = Arc::new(Int32Array::from(vec![1, 2, 3]));
- let shift_array = Arc::new(Int32Array::from(vec![32, 33, 64]));
- let result = SparkShiftRight::new()
- .spark_shift(&[value_array, shift_array])
- .unwrap();
- let arr = result.as_primitive::<Int32Type>();
- assert_eq!(arr.value(0), 1); // 1 >> 32 = 1
- assert_eq!(arr.value(1), 1); // 2 >> 33 = 1
- assert_eq!(arr.value(2), 3); // 3 >> 64 = 3
- }
-
- #[test]
- fn test_shift_right_negative_shift() {
- let value_array = Arc::new(Int32Array::from(vec![4, 8, 16]));
- let shift_array = Arc::new(Int32Array::from(vec![-1, -2, -3]));
- let result = SparkShiftRight::new()
- .spark_shift(&[value_array, shift_array])
- .unwrap();
- let arr = result.as_primitive::<Int32Type>();
- assert_eq!(arr.value(0), 0); // 4 >> -1 = 0
- assert_eq!(arr.value(1), 0); // 8 >> -2 = 0
- assert_eq!(arr.value(2), 0); // 16 >> -3 = 0
- }
-
- #[test]
- fn test_shift_right_negative_values() {
- let value_array = Arc::new(Int32Array::from(vec![-4, -8, -16]));
- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3]));
- let result = SparkShiftRight::new()
- .spark_shift(&[value_array, shift_array])
- .unwrap();
- let arr = result.as_primitive::<Int32Type>();
- // For signed integers, right shift preserves the sign bit
- assert_eq!(arr.value(0), -2); // -4 >> 1 = -2
- assert_eq!(arr.value(1), -2); // -8 >> 2 = -2
- assert_eq!(arr.value(2), -2); // -16 >> 3 = -2
- }
-
- #[test]
- fn test_shift_left_int32() {
- let value_array = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));
- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));
- let result = SparkShiftLeft::new()
- .spark_shift(&[value_array, shift_array])
- .unwrap();
- let arr = result.as_primitive::<Int32Type>();
- assert_eq!(arr.value(0), 2); // 1 << 1 = 2
- assert_eq!(arr.value(1), 8); // 2 << 2 = 8
- assert_eq!(arr.value(2), 24); // 3 << 3 = 24
- assert_eq!(arr.value(3), 64); // 4 << 4 = 64
- }
-
- #[test]
- fn test_shift_left_int64() {
- let value_array = Arc::new(Int64Array::from(vec![1i64, 2, 3]));
- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3]));
- let result = SparkShiftLeft::new()
- .spark_shift(&[value_array, shift_array])
- .unwrap();
- let arr = result.as_primitive::<Int64Type>();
- assert_eq!(arr.value(0), 2); // 1 << 1 = 2
- assert_eq!(arr.value(1), 8); // 2 << 2 = 8
- assert_eq!(arr.value(2), 24); // 3 << 3 = 24
- }
-
- #[test]
- fn test_shift_left_uint32() {
- let value_array = Arc::new(UInt32Array::from(vec![1u32, 2, 3]));
- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3]));
- let result = SparkShiftLeft::new()
- .spark_shift(&[value_array, shift_array])
- .unwrap();
- let arr = result.as_primitive::<UInt32Type>();
- assert_eq!(arr.value(0), 2); // 1 << 1 = 2
- assert_eq!(arr.value(1), 8); // 2 << 2 = 8
- assert_eq!(arr.value(2), 24); // 3 << 3 = 24
- }
-
- #[test]
- fn test_shift_left_uint64() {
- let value_array = Arc::new(UInt64Array::from(vec![1u64, 2, 3]));
- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3]));
- let result = SparkShiftLeft::new()
- .spark_shift(&[value_array, shift_array])
- .unwrap();
- let arr = result.as_primitive::<UInt64Type>();
- assert_eq!(arr.value(0), 2); // 1 << 1 = 2
- assert_eq!(arr.value(1), 8); // 2 << 2 = 8
- assert_eq!(arr.value(2), 24); // 3 << 3 = 24
- }
-
- #[test]
- fn test_shift_left_nulls() {
- let value_array = Arc::new(Int32Array::from(vec![Some(2), None, Some(3)]));
- let shift_array = Arc::new(Int32Array::from(vec![Some(1), Some(2), None]));
- let result = SparkShiftLeft::new()
- .spark_shift(&[value_array, shift_array])
- .unwrap();
- let arr = result.as_primitive::<Int32Type>();
- assert_eq!(arr.value(0), 4); // 2 << 1 = 4
- assert!(arr.is_null(1)); // null << 2 = null
- assert!(arr.is_null(2)); // 3 << null = null
- }
-
- #[test]
- fn test_shift_left_large_shift() {
- let value_array = Arc::new(Int32Array::from(vec![1, 2, 3]));
- let shift_array = Arc::new(Int32Array::from(vec![32, 33, 64]));
- let result = SparkShiftLeft::new()
- .spark_shift(&[value_array, shift_array])
- .unwrap();
- let arr = result.as_primitive::<Int32Type>();
- assert_eq!(arr.value(0), 1); // 1 << 32 = 0 (overflow)
- assert_eq!(arr.value(1), 4); // 2 << 33 = 0 (overflow)
- assert_eq!(arr.value(2), 3); // 3 << 64 = 0 (overflow)
- }
-
- #[test]
- fn test_shift_left_negative_shift() {
- let value_array = Arc::new(Int32Array::from(vec![4, 8, 16]));
- let shift_array = Arc::new(Int32Array::from(vec![-1, -2, -3]));
- let result = SparkShiftLeft::new()
- .spark_shift(&[value_array, shift_array])
- .unwrap();
- let arr = result.as_primitive::<Int32Type>();
- assert_eq!(arr.value(0), 0); // 4 << -1 = 0
- assert_eq!(arr.value(1), 0); // 8 << -2 = 0
- assert_eq!(arr.value(2), 0); // 16 << -3 = 0
- }
-}
diff --git a/datafusion/spark/src/function/bitwise/mod.rs b/datafusion/spark/src/function/bitwise/mod.rs
index d729a3ddd0..769ecf5c2f 100644
--- a/datafusion/spark/src/function/bitwise/mod.rs
+++ b/datafusion/spark/src/function/bitwise/mod.rs
@@ -24,9 +24,21 @@ use datafusion_expr::ScalarUDF;
use datafusion_functions::make_udf_function;
use std::sync::Arc;
-make_udf_function!(bit_shift::SparkShiftLeft, shiftleft);
-make_udf_function!(bit_shift::SparkShiftRight, shiftright);
-make_udf_function!(bit_shift::SparkShiftRightUnsigned, shiftrightunsigned);
+make_udf_function!(
+ bit_shift::SparkBitShift,
+ shiftleft,
+ bit_shift::SparkBitShift::left
+);
+make_udf_function!(
+ bit_shift::SparkBitShift,
+ shiftright,
+ bit_shift::SparkBitShift::right
+);
+make_udf_function!(
+ bit_shift::SparkBitShift,
+ shiftrightunsigned,
+ bit_shift::SparkBitShift::right_unsigned
+);
make_udf_function!(bit_get::SparkBitGet, bit_get);
make_udf_function!(bit_count::SparkBitCount, bit_count);
make_udf_function!(bitwise_not::SparkBitwiseNot, bitwise_not);
diff --git a/datafusion/sqllogictest/test_files/spark/bitwise/shiftright.slt b/datafusion/sqllogictest/test_files/spark/bitwise/shiftright.slt
index 3587bcc7ca..3dd43509b1 100644
--- a/datafusion/sqllogictest/test_files/spark/bitwise/shiftright.slt
+++ b/datafusion/sqllogictest/test_files/spark/bitwise/shiftright.slt
@@ -145,3 +145,118 @@ query I
select shiftright(3::int,-32);
----
3
+
+# i32 + nulls
+query IT
+SELECT
+ shiftright(arrow_cast(value, 'Int32'), shift),
+ arrow_typeof(shiftright(arrow_cast(value, 'Int32'), shift))
+FROM VALUES
+(4, 1),
+(8, 2),
+(16, 3),
+(32, 4),
+(null, 2),
+(8, null)
+t(value, shift)
+----
+2 Int32
+2 Int32
+2 Int32
+2 Int32
+NULL Int32
+NULL Int32
+
+# big shifts
+query IT
+SELECT
+ shiftright(arrow_cast(value, 'Int32'), shift),
+ arrow_typeof(shiftright(arrow_cast(value, 'Int32'), shift))
+FROM VALUES
+(1, 32),
+(2, 33),
+(3, 64)
+t(value, shift)
+----
+1 Int32
+1 Int32
+3 Int32
+
+# negative shift
+query IT
+SELECT
+ shiftright(arrow_cast(value, 'Int32'), shift),
+ arrow_typeof(shiftright(arrow_cast(value, 'Int32'), shift))
+FROM VALUES
+(4, -1),
+(8, -2),
+(16, -3)
+t(value, shift)
+----
+0 Int32
+0 Int32
+0 Int32
+
+# For signed integers, right shift preserves sign bit
+query IT
+SELECT
+ shiftright(arrow_cast(value, 'Int32'), shift),
+ arrow_typeof(shiftright(arrow_cast(value, 'Int32'), shift))
+FROM VALUES
+(-4, 1),
+(-8, 2),
+(-16, 3)
+t(value, shift)
+----
+-2 Int32
+-2 Int32
+-2 Int32
+
+# i64 value
+query IT
+SELECT
+ shiftright(arrow_cast(value, 'Int64'), shift),
+ arrow_typeof(shiftright(arrow_cast(value, 'Int64'), shift))
+FROM VALUES (4, 1), (8, 2), (16, 3) t(value, shift)
+----
+2 Int64
+2 Int64
+2 Int64
+
+# u32 value
+query IT
+SELECT
+ shiftright(arrow_cast(value, 'UInt32'), shift),
+ arrow_typeof(shiftright(arrow_cast(value, 'UInt32'), shift))
+FROM VALUES (4, 1), (8, 2), (16, 3) t(value, shift)
+----
+2 UInt32
+2 UInt32
+2 UInt32
+
+# u64 value
+query IT
+SELECT
+ shiftright(arrow_cast(value, 'UInt64'), shift),
+ arrow_typeof(shiftright(arrow_cast(value, 'UInt64'), shift))
+FROM VALUES (4, 1), (8, 2), (16, 3) t(value, shift)
+----
+2 UInt64
+2 UInt64
+2 UInt64
+
+# pure null handling
+query IT
+SELECT shiftright(null, 1), arrow_typeof(shiftright(null, 1));
+----
+NULL Int32
+
+query IT
+SELECT shiftright(null, null), arrow_typeof(shiftright(null, null));
+----
+NULL Int32
+
+query IT
+SELECT shiftright(1::bigint, null), arrow_typeof(shiftright(1::bigint, null));
+----
+NULL Int64
diff --git a/datafusion/sqllogictest/test_files/spark/bitwise/shiftrightunsigned.slt b/datafusion/sqllogictest/test_files/spark/bitwise/shiftrightunsigned.slt
index b0d4cfaec7..b9ef530b36 100644
--- a/datafusion/sqllogictest/test_files/spark/bitwise/shiftrightunsigned.slt
+++ b/datafusion/sqllogictest/test_files/spark/bitwise/shiftrightunsigned.slt
@@ -146,3 +146,106 @@ query I
select shiftrightunsigned(3::int,-32);
----
3
+
+# i32 + nulls
+query IT
+SELECT
+ shiftrightunsigned(arrow_cast(value, 'Int32'), shift),
+ arrow_typeof(shiftrightunsigned(arrow_cast(value, 'Int32'), shift))
+FROM VALUES
+(4, 1),
+(8, 2),
+(16, 3),
+(32, 4),
+(null, 2),
+(8, null)
+t(value, shift)
+----
+2 Int32
+2 Int32
+2 Int32
+2 Int32
+NULL Int32
+NULL Int32
+
+# negative shift
+query IT
+SELECT
+ shiftrightunsigned(arrow_cast(value, 'Int32'), shift),
+ arrow_typeof(shiftrightunsigned(arrow_cast(value, 'Int32'), shift))
+FROM VALUES
+(4, -1),
+(8, -2),
+(16, -3)
+t(value, shift)
+----
+0 Int32
+0 Int32
+0 Int32
+
+# Negative values are treated as large positive values
+# -4 as u32 = 4294967292, 4294967292 >>> 1 = 2147483646
+# -8 as u32 = 4294967288, 4294967288 >>> 2 = 1073741822
+# -16 as u32 = 4294967280, 4294967280 >>> 3 = 536870910
+query IT
+SELECT
+ shiftrightunsigned(arrow_cast(value, 'Int32'), shift),
+ arrow_typeof(shiftrightunsigned(arrow_cast(value, 'Int32'), shift))
+FROM VALUES
+(-4, 1),
+(-8, 2),
+(-16, 3)
+t(value, shift)
+----
+2147483646 Int32
+1073741822 Int32
+536870910 Int32
+
+# i64 value
+query IT
+SELECT
+ shiftrightunsigned(arrow_cast(value, 'Int64'), shift),
+ arrow_typeof(shiftrightunsigned(arrow_cast(value, 'Int64'), shift))
+FROM VALUES (4, 1), (8, 2), (16, 3) t(value, shift)
+----
+2 Int64
+2 Int64
+2 Int64
+
+# u32 value
+query IT
+SELECT
+ shiftrightunsigned(arrow_cast(value, 'UInt32'), shift),
+ arrow_typeof(shiftrightunsigned(arrow_cast(value, 'UInt32'), shift))
+FROM VALUES (4, 1), (8, 2), (16, 3) t(value, shift)
+----
+2 UInt32
+2 UInt32
+2 UInt32
+
+# u64 value
+query IT
+SELECT
+ shiftrightunsigned(arrow_cast(value, 'UInt64'), shift),
+ arrow_typeof(shiftrightunsigned(arrow_cast(value, 'UInt64'), shift))
+FROM VALUES (4, 1), (8, 2), (16, 3) t(value, shift)
+----
+2 UInt64
+2 UInt64
+2 UInt64
+
+# pure null handling
+query IT
+SELECT shiftrightunsigned(null, 1), arrow_typeof(shiftrightunsigned(null, 1));
+----
+NULL Int32
+
+query IT
+SELECT shiftrightunsigned(null, null), arrow_typeof(shiftrightunsigned(null, null));
+----
+NULL Int32
+
+query IT
+SELECT shiftrightunsigned(1::bigint, null), arrow_typeof(shiftrightunsigned(1::bigint, null));
+----
+NULL Int64
diff --git a/datafusion/sqllogictest/test_files/spark/math/shiftleft.slt b/datafusion/sqllogictest/test_files/spark/math/shiftleft.slt
index 3676e4c181..c8ddeb6740 100644
--- a/datafusion/sqllogictest/test_files/spark/math/shiftleft.slt
+++ b/datafusion/sqllogictest/test_files/spark/math/shiftleft.slt
@@ -144,3 +144,103 @@ query I
select shiftleft(3::int,-32);
----
3
+
+# i32 + nulls
+query IT
+SELECT
+ shiftleft(arrow_cast(value, 'Int32'), shift),
+ arrow_typeof(shiftleft(arrow_cast(value, 'Int32'), shift))
+FROM VALUES
+(1, 1),
+(2, 2),
+(3, 3),
+(4, 4),
+(null, 2),
+(8, null)
+t(value, shift)
+----
+2 Int32
+8 Int32
+24 Int32
+64 Int32
+NULL Int32
+NULL Int32
+
+# big shifts
+query IT
+SELECT
+ shiftleft(arrow_cast(value, 'Int32'), shift),
+ arrow_typeof(shiftleft(arrow_cast(value, 'Int32'), shift))
+FROM VALUES
+(1, 32),
+(2, 33),
+(3, 64)
+t(value, shift)
+----
+1 Int32
+4 Int32
+3 Int32
+
+# negative shift
+query IT
+SELECT
+ shiftleft(arrow_cast(value, 'Int32'), shift),
+ arrow_typeof(shiftleft(arrow_cast(value, 'Int32'), shift))
+FROM VALUES
+(4, -1),
+(8, -2),
+(16, -3)
+t(value, shift)
+----
+0 Int32
+0 Int32
+0 Int32
+
+# i64 value
+query IT
+SELECT
+ shiftleft(arrow_cast(value, 'Int64'), shift),
+ arrow_typeof(shiftleft(arrow_cast(value, 'Int64'), shift))
+FROM VALUES (1, 1), (2, 2), (3, 3) t(value, shift)
+----
+2 Int64
+8 Int64
+24 Int64
+
+# u32 value
+query IT
+SELECT
+ shiftleft(arrow_cast(value, 'UInt32'), shift),
+ arrow_typeof(shiftleft(arrow_cast(value, 'UInt32'), shift))
+FROM VALUES (1, 1), (2, 2), (3, 3) t(value, shift)
+----
+2 UInt32
+8 UInt32
+24 UInt32
+
+# u64 value
+query IT
+SELECT
+ shiftleft(arrow_cast(value, 'UInt64'), shift),
+ arrow_typeof(shiftleft(arrow_cast(value, 'UInt64'), shift))
+FROM VALUES (1, 1), (2, 2), (3, 3) t(value, shift)
+----
+2 UInt64
+8 UInt64
+24 UInt64
+
+# pure null handling
+query IT
+SELECT shiftleft(null, 1), arrow_typeof(shiftleft(null, 1));
+----
+NULL Int32
+
+query IT
+SELECT shiftleft(null, null), arrow_typeof(shiftleft(null, null));
+----
+NULL Int32
+
+query IT
+SELECT shiftleft(1::bigint, null), arrow_typeof(shiftleft(1::bigint, null));
+----
+NULL Int64