Apache DataFusion SQL Query Engine
Refactor Spark bitshift signature (#18649)
Jeffrey Vo35b5363777a9
diff --git a/datafusion/functions/src/macros.rs b/datafusion/functions/src/macros.rsindex 9e195f2d52..077690cd3f 100644--- a/datafusion/functions/src/macros.rs+++ b/datafusion/functions/src/macros.rs@@ -77,7 +77,8 @@ macro_rules! export_functions {}/// Creates a singleton `ScalarUDF` of the `$UDF` function and a function-/// named `$NAME` which returns that singleton.+/// named `$NAME` which returns that singleton. Optionally use a custom constructor+/// `$CTOR` which defaults to `$UDF::new()` if not specified.////// This is used to ensure creating the list of `ScalarUDF` only happens once.#[macro_export]@@ -97,6 +98,21 @@ macro_rules! make_udf_function {std::sync::Arc::clone(&INSTANCE)}};+ ($UDF:ty, $NAME:ident, $CTOR:path) => {+ #[allow(rustdoc::redundant_explicit_links)]+ #[doc = concat!("Return a [`ScalarUDF`](datafusion_expr::ScalarUDF) implementation of ", stringify!($NAME))]+ pub fn $NAME() -> std::sync::Arc<datafusion_expr::ScalarUDF> {+ // Singleton instance of the function+ static INSTANCE: std::sync::LazyLock<+ std::sync::Arc<datafusion_expr::ScalarUDF>,+ > = std::sync::LazyLock::new(|| {+ std::sync::Arc::new(datafusion_expr::ScalarUDF::new_from_impl(+ $CTOR(),+ ))+ });+ std::sync::Arc::clone(&INSTANCE)+ }+ };}/// Creates a singleton `ScalarUDF` of the `$UDF` function and a functiondiff --git a/datafusion/spark/src/function/bitwise/bit_shift.rs b/datafusion/spark/src/function/bitwise/bit_shift.rsindex 68911b0492..65df048580 100644--- a/datafusion/spark/src/function/bitwise/bit_shift.rs+++ b/datafusion/spark/src/function/bitwise/bit_shift.rs@@ -18,36 +18,33 @@use std::any::Any;use std::sync::Arc;-use arrow::array::{ArrayRef, ArrowPrimitiveType, AsArray, PrimitiveArray};+use arrow::array::{ArrayRef, ArrowPrimitiveType, AsArray, Int32Array, PrimitiveArray};use arrow::compute;use arrow::datatypes::{ArrowNativeType, DataType, Int32Type, Int64Type, UInt32Type, UInt64Type,};-use datafusion_common::{plan_err, Result};+use datafusion_common::types::{+ logical_int16, logical_int32, logical_int64, logical_int8, logical_uint16,+ logical_uint32, logical_uint64, logical_uint8, NativeType,+};+use datafusion_common::utils::take_function_args;+use datafusion_common::{internal_err, Result};use datafusion_expr::{- ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,+ Coercion, ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature,+ TypeSignatureClass, Volatility,};use datafusion_functions::utils::make_scalar_function;-use crate::function::error_utils::{- invalid_arg_count_exec_err, unsupported_data_type_exec_err,-};--/// Performs a bitwise left shift on each element of the `value` array by the corresponding amount in the `shift` array.-/// The shift amount is normalized to the bit width of the type, matching Spark/Java semantics for negative and large shifts.-///-/// # Arguments-/// * `value` - The array of values to shift.-/// * `shift` - The array of shift amounts (must be Int32).-///-/// # Returns-/// A new array with the shifted values.-fn shift_left<T: ArrowPrimitiveType>(+/// Bitwise left shift on elements in `value` by corresponding `shift` amount.+/// The shift amount is normalized to the bit width of the type, matching Spark/Java+/// semantics for negative and large shifts.+fn shift_left<T>(value: &PrimitiveArray<T>,- shift: &PrimitiveArray<Int32Type>,+ shift: &Int32Array,) -> Result<PrimitiveArray<T>>where- T::Native: ArrowNativeType + std::ops::Shl<i32, Output = T::Native>,+ T: ArrowPrimitiveType,+ T::Native: std::ops::Shl<i32, Output = T::Native>,{let bit_num = (T::Native::get_byte_width() * 8) as i32;let result = compute::binary::<_, Int32Type, _, _>(@@ -61,21 +58,16 @@ whereOk(result)}-/// Performs a bitwise right shift on each element of the `value` array by the corresponding amount in the `shift` array.-/// The shift amount is normalized to the bit width of the type, matching Spark/Java semantics for negative and large shifts.-///-/// # Arguments-/// * `value` - The array of values to shift.-/// * `shift` - The array of shift amounts (must be Int32).-///-/// # Returns-/// A new array with the shifted values.-fn shift_right<T: ArrowPrimitiveType>(+/// Bitwise right shift on elements in `value` by corresponding `shift` amount.+/// The shift amount is normalized to the bit width of the type, matching Spark/Java+/// semantics for negative and large shifts.+fn shift_right<T>(value: &PrimitiveArray<T>,- shift: &PrimitiveArray<Int32Type>,+ shift: &Int32Array,) -> Result<PrimitiveArray<T>>where- T::Native: ArrowNativeType + std::ops::Shr<i32, Output = T::Native>,+ T: ArrowPrimitiveType,+ T::Native: std::ops::Shr<i32, Output = T::Native>,{let bit_num = (T::Native::get_byte_width() * 8) as i32;let result = compute::binary::<_, Int32Type, _, _>(@@ -93,49 +85,44 @@ where/// This is used to mimic Java's `>>>` operator, which does not exist in Rust./// For unsigned types, this is just the normal right shift./// For signed types, this casts to the unsigned type, shifts, then casts back.-trait UShr<Rhs> {- fn ushr(self, rhs: Rhs) -> Self;+trait UShr {+ fn ushr(self, rhs: i32) -> Self;}-impl UShr<i32> for u32 {+impl UShr for u32 {fn ushr(self, rhs: i32) -> Self {self >> rhs}}-impl UShr<i32> for u64 {+impl UShr for u64 {fn ushr(self, rhs: i32) -> Self {self >> rhs}}-impl UShr<i32> for i32 {+impl UShr for i32 {fn ushr(self, rhs: i32) -> Self {((self as u32) >> rhs) as i32}}-impl UShr<i32> for i64 {+impl UShr for i64 {fn ushr(self, rhs: i32) -> Self {((self as u64) >> rhs) as i64}}-/// Performs a bitwise unsigned right shift on each element of the `value` array by the corresponding amount in the `shift` array.-/// The shift amount is normalized to the bit width of the type, matching Spark/Java semantics for negative and large shifts.-///-/// # Arguments-/// * `value` - The array of values to shift.-/// * `shift` - The array of shift amounts (must be Int32).-///-/// # Returns-/// A new array with the shifted values.-fn shift_right_unsigned<T: ArrowPrimitiveType>(+/// Bitwise unsigned right shift on elements in `value` by corresponding `shift`+/// amount. The shift amount is normalized to the bit width of the type, matching+/// Spark/Java semantics for negative and large shifts.+fn shift_right_unsigned<T>(value: &PrimitiveArray<T>,- shift: &PrimitiveArray<Int32Type>,+ shift: &Int32Array,) -> Result<PrimitiveArray<T>>where- T::Native: ArrowNativeType + UShr<i32>,+ T: ArrowPrimitiveType,+ T::Native: UShr,{let bit_num = (T::Native::get_byte_width() * 8) as i32;let result = compute::binary::<_, Int32Type, _, _>(@@ -149,589 +136,153 @@ whereOk(result)}-trait BitShiftUDF: ScalarUDFImpl {- fn shift<T: ArrowPrimitiveType>(- &self,- value: &PrimitiveArray<T>,- shift: &PrimitiveArray<Int32Type>,- ) -> Result<PrimitiveArray<T>>- where- T::Native: ArrowNativeType- + std::ops::Shl<i32, Output = T::Native>- + std::ops::Shr<i32, Output = T::Native>- + UShr<i32>;-- fn spark_shift(&self, arrays: &[ArrayRef]) -> Result<ArrayRef> {- let value_array = arrays[0].as_ref();- let shift_array = arrays[1].as_ref();-- // Ensure shift array is Int32- let shift_array = if shift_array.data_type() != &DataType::Int32 {- return plan_err!("{} shift amount must be Int32", self.name());- } else {- shift_array.as_primitive::<Int32Type>()- };-- match value_array.data_type() {- DataType::Int32 => {- let value_array = value_array.as_primitive::<Int32Type>();- Ok(Arc::new(self.shift(value_array, shift_array)?))- }- DataType::Int64 => {- let value_array = value_array.as_primitive::<Int64Type>();- Ok(Arc::new(self.shift(value_array, shift_array)?))- }- DataType::UInt32 => {- let value_array = value_array.as_primitive::<UInt32Type>();- Ok(Arc::new(self.shift(value_array, shift_array)?))- }- DataType::UInt64 => {- let value_array = value_array.as_primitive::<UInt64Type>();- Ok(Arc::new(self.shift(value_array, shift_array)?))- }- _ => {- plan_err!(- "{} function does not support data type: {}",- self.name(),- value_array.data_type()- )- }- }- }-}--fn bit_shift_coerce_types(arg_types: &[DataType], func: &str) -> Result<Vec<DataType>> {- if arg_types.len() != 2 {- return Err(invalid_arg_count_exec_err(func, (2, 2), arg_types.len()));- }- if !arg_types[0].is_integer() && !arg_types[0].is_null() {- return Err(unsupported_data_type_exec_err(- func,- "Integer Type",- &arg_types[0],- ));- }- if !arg_types[1].is_integer() && !arg_types[1].is_null() {- return Err(unsupported_data_type_exec_err(- func,- "Integer Type",- &arg_types[1],- ));- }-- // Coerce smaller integer types to Int32- let coerced_first = match &arg_types[0] {- DataType::Int8 | DataType::Int16 | DataType::Null => DataType::Int32,- DataType::UInt8 | DataType::UInt16 => DataType::UInt32,- _ => arg_types[0].clone(),- };-- Ok(vec![coerced_first, DataType::Int32])-}--#[derive(Debug, Hash, Eq, PartialEq)]-pub struct SparkShiftLeft {- signature: Signature,-}--impl Default for SparkShiftLeft {- fn default() -> Self {- Self::new()- }-}--impl SparkShiftLeft {- pub fn new() -> Self {- Self {- signature: Signature::user_defined(Volatility::Immutable),- }- }-}+fn shift_inner(+ arrays: &[ArrayRef],+ name: &str,+ bit_shift_type: BitShiftType,+) -> Result<ArrayRef> {+ let [value_array, shift_array] = take_function_args(name, arrays)?;+ let shift_array = shift_array.as_primitive::<Int32Type>();-impl BitShiftUDF for SparkShiftLeft {- fn shift<T: ArrowPrimitiveType>(- &self,+ fn shift<T>(value: &PrimitiveArray<T>,- shift: &PrimitiveArray<Int32Type>,+ shift: &Int32Array,+ bit_shift_type: BitShiftType,) -> Result<PrimitiveArray<T>>where- T::Native: ArrowNativeType- + std::ops::Shl<i32, Output = T::Native>+ T: ArrowPrimitiveType,+ T::Native: std::ops::Shl<i32, Output = T::Native>+ std::ops::Shr<i32, Output = T::Native>- + UShr<i32>,+ + UShr,{- shift_left(value, shift)- }-}--impl ScalarUDFImpl for SparkShiftLeft {- fn as_any(&self) -> &dyn Any {- self- }-- fn name(&self) -> &str {- "shiftleft"- }-- fn signature(&self) -> &Signature {- &self.signature- }-- fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {- bit_shift_coerce_types(arg_types, "shiftleft")- }-- fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {- if arg_types.len() != 2 {- return plan_err!("shiftleft expects exactly 2 arguments");+ match bit_shift_type {+ BitShiftType::Left => shift_left(value, shift),+ BitShiftType::Right => shift_right(value, shift),+ BitShiftType::RightUnsigned => shift_right_unsigned(value, shift),}- // Return type is the same as the first argument (the value to shift)- Ok(arg_types[0].clone())}- fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {- if args.args.len() != 2 {- return plan_err!("shiftleft expects exactly 2 arguments");+ match value_array.data_type() {+ DataType::Int32 => {+ let value_array = value_array.as_primitive::<Int32Type>();+ Ok(Arc::new(shift(value_array, shift_array, bit_shift_type)?))+ }+ DataType::Int64 => {+ let value_array = value_array.as_primitive::<Int64Type>();+ Ok(Arc::new(shift(value_array, shift_array, bit_shift_type)?))+ }+ DataType::UInt32 => {+ let value_array = value_array.as_primitive::<UInt32Type>();+ Ok(Arc::new(shift(value_array, shift_array, bit_shift_type)?))+ }+ DataType::UInt64 => {+ let value_array = value_array.as_primitive::<UInt64Type>();+ Ok(Arc::new(shift(value_array, shift_array, bit_shift_type)?))+ }+ dt => {+ internal_err!("{name} function does not support data type: {dt}")}- let inner = |arr: &[ArrayRef]| -> Result<ArrayRef> { self.spark_shift(arr) };- make_scalar_function(inner, vec![])(&args.args)}}-#[derive(Debug, Hash, Eq, PartialEq)]-pub struct SparkShiftRightUnsigned {- signature: Signature,+#[derive(Debug, Hash, Copy, Clone, Eq, PartialEq)]+enum BitShiftType {+ Left,+ Right,+ RightUnsigned,}-impl Default for SparkShiftRightUnsigned {- fn default() -> Self {- Self::new()- }+#[derive(Debug, Hash, Eq, PartialEq)]+pub struct SparkBitShift {+ signature: Signature,+ name: &'static str,+ bit_shift_type: BitShiftType,}-impl SparkShiftRightUnsigned {- pub fn new() -> Self {+impl SparkBitShift {+ fn new(name: &'static str, bit_shift_type: BitShiftType) -> Self {+ let shift_amount = Coercion::new_implicit(+ TypeSignatureClass::Native(logical_int32()),+ vec![TypeSignatureClass::Integer],+ NativeType::Int32,+ );Self {- signature: Signature::user_defined(Volatility::Immutable),+ signature: Signature::one_of(+ vec![+ // Upcast small ints to 32bit+ TypeSignature::Coercible(vec![+ Coercion::new_implicit(+ TypeSignatureClass::Native(logical_int32()),+ vec![+ TypeSignatureClass::Native(logical_int8()),+ TypeSignatureClass::Native(logical_int16()),+ ],+ NativeType::Int32,+ ),+ shift_amount.clone(),+ ]),+ TypeSignature::Coercible(vec![+ Coercion::new_implicit(+ TypeSignatureClass::Native(logical_uint32()),+ vec![+ TypeSignatureClass::Native(logical_uint8()),+ TypeSignatureClass::Native(logical_uint16()),+ ],+ NativeType::UInt32,+ ),+ shift_amount.clone(),+ ]),+ // Otherwise accept direct 64 bit integers+ TypeSignature::Coercible(vec![+ Coercion::new_exact(TypeSignatureClass::Native(logical_int64())),+ shift_amount.clone(),+ ]),+ TypeSignature::Coercible(vec![+ Coercion::new_exact(TypeSignatureClass::Native(logical_uint64())),+ shift_amount.clone(),+ ]),+ ],+ Volatility::Immutable,+ ),+ name,+ bit_shift_type,}}-}--impl BitShiftUDF for SparkShiftRightUnsigned {- fn shift<T: ArrowPrimitiveType>(- &self,- value: &PrimitiveArray<T>,- shift: &PrimitiveArray<Int32Type>,- ) -> Result<PrimitiveArray<T>>- where- T::Native: ArrowNativeType- + std::ops::Shl<i32, Output = T::Native>- + std::ops::Shr<i32, Output = T::Native>- + UShr<i32>,- {- shift_right_unsigned(value, shift)- }-}--impl ScalarUDFImpl for SparkShiftRightUnsigned {- fn as_any(&self) -> &dyn Any {- self- }- fn name(&self) -> &str {- "shiftrightunsigned"+ pub fn left() -> Self {+ Self::new("shiftleft", BitShiftType::Left)}- fn signature(&self) -> &Signature {- &self.signature+ pub fn right() -> Self {+ Self::new("shiftright", BitShiftType::Right)}- fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {- bit_shift_coerce_types(arg_types, "shiftrightunsigned")- }-- fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {- if arg_types.len() != 2 {- return plan_err!("shiftrightunsigned expects exactly 2 arguments");- }- // Return type is the same as the first argument (the value to shift)- Ok(arg_types[0].clone())- }-- fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {- if args.args.len() != 2 {- return plan_err!("shiftrightunsigned expects exactly 2 arguments");- }- let inner = |arr: &[ArrayRef]| -> Result<ArrayRef> { self.spark_shift(arr) };- make_scalar_function(inner, vec![])(&args.args)- }-}--#[derive(Debug, Hash, Eq, PartialEq)]-pub struct SparkShiftRight {- signature: Signature,-}--impl Default for SparkShiftRight {- fn default() -> Self {- Self::new()- }-}--impl SparkShiftRight {- pub fn new() -> Self {- Self {- signature: Signature::user_defined(Volatility::Immutable),- }+ pub fn right_unsigned() -> Self {+ Self::new("shiftrightunsigned", BitShiftType::RightUnsigned)}}-impl BitShiftUDF for SparkShiftRight {- fn shift<T: ArrowPrimitiveType>(- &self,- value: &PrimitiveArray<T>,- shift: &PrimitiveArray<Int32Type>,- ) -> Result<PrimitiveArray<T>>- where- T::Native: ArrowNativeType- + std::ops::Shl<i32, Output = T::Native>- + std::ops::Shr<i32, Output = T::Native>- + UShr<i32>,- {- shift_right(value, shift)- }-}--impl ScalarUDFImpl for SparkShiftRight {+impl ScalarUDFImpl for SparkBitShift {fn as_any(&self) -> &dyn Any {self}fn name(&self) -> &str {- "shiftright"+ self.name}fn signature(&self) -> &Signature {&self.signature}- fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {- bit_shift_coerce_types(arg_types, "shiftright")- }-fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {- if arg_types.len() != 2 {- return plan_err!("shiftright expects exactly 2 arguments");- }- // Return type is the same as the first argument (the value to shift)Ok(arg_types[0].clone())}fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {- if args.args.len() != 2 {- return plan_err!("shiftright expects exactly 2 arguments");- }- let inner = |arr: &[ArrayRef]| -> Result<ArrayRef> { self.spark_shift(arr) };+ let inner = |arr: &[ArrayRef]| -> Result<ArrayRef> {+ shift_inner(arr, self.name(), self.bit_shift_type)+ };make_scalar_function(inner, vec![])(&args.args)}}--#[cfg(test)]-mod tests {- use super::*;- use arrow::array::{Array, Int32Array, Int64Array, UInt32Array, UInt64Array};-- #[test]- fn test_shift_right_unsigned_int32() {- let value_array = Arc::new(Int32Array::from(vec![4, 8, 16, 32]));- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));- let result = SparkShiftRightUnsigned::new()- .spark_shift(&[value_array, shift_array])- .unwrap();- let arr = result.as_primitive::<Int32Type>();- assert_eq!(arr.value(0), 2); // 4 >>> 1 = 2- assert_eq!(arr.value(1), 2); // 8 >>> 2 = 2- assert_eq!(arr.value(2), 2); // 16 >>> 3 = 2- assert_eq!(arr.value(3), 2); // 32 >>> 4 = 2- }-- #[test]- fn test_shift_right_unsigned_int64() {- let value_array = Arc::new(Int64Array::from(vec![4i64, 8, 16]));- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3]));- let result = SparkShiftRightUnsigned::new()- .spark_shift(&[value_array, shift_array])- .unwrap();- let arr = result.as_primitive::<Int64Type>();- assert_eq!(arr.value(0), 2); // 4 >>> 1 = 2- assert_eq!(arr.value(1), 2); // 8 >>> 2 = 2- assert_eq!(arr.value(2), 2); // 16 >>> 3 = 2- }-- #[test]- fn test_shift_right_unsigned_uint32() {- let value_array = Arc::new(UInt32Array::from(vec![4u32, 8, 16]));- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3]));- let result = SparkShiftRightUnsigned::new()- .spark_shift(&[value_array, shift_array])- .unwrap();- let arr = result.as_primitive::<UInt32Type>();- assert_eq!(arr.value(0), 2); // 4 >>> 1 = 2- assert_eq!(arr.value(1), 2); // 8 >>> 2 = 2- assert_eq!(arr.value(2), 2); // 16 >>> 3 = 2- }-- #[test]- fn test_shift_right_unsigned_uint64() {- let value_array = Arc::new(UInt64Array::from(vec![4u64, 8, 16]));- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3]));- let result = SparkShiftRightUnsigned::new()- .spark_shift(&[value_array, shift_array])- .unwrap();- let arr = result.as_primitive::<UInt64Type>();- assert_eq!(arr.value(0), 2); // 4 >>> 1 = 2- assert_eq!(arr.value(1), 2); // 8 >>> 2 = 2- assert_eq!(arr.value(2), 2); // 16 >>> 3 = 2- }-- #[test]- fn test_shift_right_unsigned_nulls() {- let value_array = Arc::new(Int32Array::from(vec![Some(4), None, Some(8)]));- let shift_array = Arc::new(Int32Array::from(vec![Some(1), Some(2), None]));- let result = SparkShiftRightUnsigned::new()- .spark_shift(&[value_array, shift_array])- .unwrap();- let arr = result.as_primitive::<Int32Type>();- assert_eq!(arr.value(0), 2); // 4 >>> 1 = 2- assert!(arr.is_null(1)); // null >>> 2 = null- assert!(arr.is_null(2)); // 8 >>> null = null- }-- #[test]- fn test_shift_right_unsigned_negative_shift() {- let value_array = Arc::new(Int32Array::from(vec![4, 8, 16]));- let shift_array = Arc::new(Int32Array::from(vec![-1, -2, -3]));- let result = SparkShiftRightUnsigned::new()- .spark_shift(&[value_array, shift_array])- .unwrap();- let arr = result.as_primitive::<Int32Type>();- assert_eq!(arr.value(0), 0); // 4 >>> -1 = 0- assert_eq!(arr.value(1), 0); // 8 >>> -2 = 0- assert_eq!(arr.value(2), 0); // 16 >>> -3 = 0- }-- #[test]- fn test_shift_right_unsigned_negative_values() {- let value_array = Arc::new(Int32Array::from(vec![-4, -8, -16]));- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3]));- let result = SparkShiftRightUnsigned::new()- .spark_shift(&[value_array, shift_array])- .unwrap();- let arr = result.as_primitive::<Int32Type>();- // For unsigned right shift, negative values are treated as large positive values- // -4 as u32 = 4294967292, -4 >>> 1 = 2147483646- assert_eq!(arr.value(0), 2147483646);- // -8 as u32 = 4294967288, -8 >>> 2 = 1073741822- assert_eq!(arr.value(1), 1073741822);- // -16 as u32 = 4294967280, -16 >>> 3 = 536870910- assert_eq!(arr.value(2), 536870910);- }-- #[test]- fn test_shift_right_int32() {- let value_array = Arc::new(Int32Array::from(vec![4, 8, 16, 32]));- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));- let result = SparkShiftRight::new()- .spark_shift(&[value_array, shift_array])- .unwrap();- let arr = result.as_primitive::<Int32Type>();- assert_eq!(arr.value(0), 2); // 4 >> 1 = 2- assert_eq!(arr.value(1), 2); // 8 >> 2 = 2- assert_eq!(arr.value(2), 2); // 16 >> 3 = 2- assert_eq!(arr.value(3), 2); // 32 >> 4 = 2- }-- #[test]- fn test_shift_right_int64() {- let value_array = Arc::new(Int64Array::from(vec![4i64, 8, 16]));- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3]));- let result = SparkShiftRight::new()- .spark_shift(&[value_array, shift_array])- .unwrap();- let arr = result.as_primitive::<Int64Type>();- assert_eq!(arr.value(0), 2); // 4 >> 1 = 2- assert_eq!(arr.value(1), 2); // 8 >> 2 = 2- assert_eq!(arr.value(2), 2); // 16 >> 3 = 2- }-- #[test]- fn test_shift_right_uint32() {- let value_array = Arc::new(UInt32Array::from(vec![4u32, 8, 16]));- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3]));- let result = SparkShiftRight::new()- .spark_shift(&[value_array, shift_array])- .unwrap();- let arr = result.as_primitive::<UInt32Type>();- assert_eq!(arr.value(0), 2); // 4 >> 1 = 2- assert_eq!(arr.value(1), 2); // 8 >> 2 = 2- assert_eq!(arr.value(2), 2); // 16 >> 3 = 2- }-- #[test]- fn test_shift_right_uint64() {- let value_array = Arc::new(UInt64Array::from(vec![4u64, 8, 16]));- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3]));- let result = SparkShiftRight::new()- .spark_shift(&[value_array, shift_array])- .unwrap();- let arr = result.as_primitive::<UInt64Type>();- assert_eq!(arr.value(0), 2); // 4 >> 1 = 2- assert_eq!(arr.value(1), 2); // 8 >> 2 = 2- assert_eq!(arr.value(2), 2); // 16 >> 3 = 2- }-- #[test]- fn test_shift_right_nulls() {- let value_array = Arc::new(Int32Array::from(vec![Some(4), None, Some(8)]));- let shift_array = Arc::new(Int32Array::from(vec![Some(1), Some(2), None]));- let result = SparkShiftRight::new()- .spark_shift(&[value_array, shift_array])- .unwrap();- let arr = result.as_primitive::<Int32Type>();- assert_eq!(arr.value(0), 2); // 4 >> 1 = 2- assert!(arr.is_null(1)); // null >> 2 = null- assert!(arr.is_null(2)); // 8 >> null = null- }-- #[test]- fn test_shift_right_large_shift() {- let value_array = Arc::new(Int32Array::from(vec![1, 2, 3]));- let shift_array = Arc::new(Int32Array::from(vec![32, 33, 64]));- let result = SparkShiftRight::new()- .spark_shift(&[value_array, shift_array])- .unwrap();- let arr = result.as_primitive::<Int32Type>();- assert_eq!(arr.value(0), 1); // 1 >> 32 = 1- assert_eq!(arr.value(1), 1); // 2 >> 33 = 1- assert_eq!(arr.value(2), 3); // 3 >> 64 = 3- }-- #[test]- fn test_shift_right_negative_shift() {- let value_array = Arc::new(Int32Array::from(vec![4, 8, 16]));- let shift_array = Arc::new(Int32Array::from(vec![-1, -2, -3]));- let result = SparkShiftRight::new()- .spark_shift(&[value_array, shift_array])- .unwrap();- let arr = result.as_primitive::<Int32Type>();- assert_eq!(arr.value(0), 0); // 4 >> -1 = 0- assert_eq!(arr.value(1), 0); // 8 >> -2 = 0- assert_eq!(arr.value(2), 0); // 16 >> -3 = 0- }-- #[test]- fn test_shift_right_negative_values() {- let value_array = Arc::new(Int32Array::from(vec![-4, -8, -16]));- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3]));- let result = SparkShiftRight::new()- .spark_shift(&[value_array, shift_array])- .unwrap();- let arr = result.as_primitive::<Int32Type>();- // For signed integers, right shift preserves the sign bit- assert_eq!(arr.value(0), -2); // -4 >> 1 = -2- assert_eq!(arr.value(1), -2); // -8 >> 2 = -2- assert_eq!(arr.value(2), -2); // -16 >> 3 = -2- }-- #[test]- fn test_shift_left_int32() {- let value_array = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));- let result = SparkShiftLeft::new()- .spark_shift(&[value_array, shift_array])- .unwrap();- let arr = result.as_primitive::<Int32Type>();- assert_eq!(arr.value(0), 2); // 1 << 1 = 2- assert_eq!(arr.value(1), 8); // 2 << 2 = 8- assert_eq!(arr.value(2), 24); // 3 << 3 = 24- assert_eq!(arr.value(3), 64); // 4 << 4 = 64- }-- #[test]- fn test_shift_left_int64() {- let value_array = Arc::new(Int64Array::from(vec![1i64, 2, 3]));- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3]));- let result = SparkShiftLeft::new()- .spark_shift(&[value_array, shift_array])- .unwrap();- let arr = result.as_primitive::<Int64Type>();- assert_eq!(arr.value(0), 2); // 1 << 1 = 2- assert_eq!(arr.value(1), 8); // 2 << 2 = 8- assert_eq!(arr.value(2), 24); // 3 << 3 = 24- }-- #[test]- fn test_shift_left_uint32() {- let value_array = Arc::new(UInt32Array::from(vec![1u32, 2, 3]));- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3]));- let result = SparkShiftLeft::new()- .spark_shift(&[value_array, shift_array])- .unwrap();- let arr = result.as_primitive::<UInt32Type>();- assert_eq!(arr.value(0), 2); // 1 << 1 = 2- assert_eq!(arr.value(1), 8); // 2 << 2 = 8- assert_eq!(arr.value(2), 24); // 3 << 3 = 24- }-- #[test]- fn test_shift_left_uint64() {- let value_array = Arc::new(UInt64Array::from(vec![1u64, 2, 3]));- let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3]));- let result = SparkShiftLeft::new()- .spark_shift(&[value_array, shift_array])- .unwrap();- let arr = result.as_primitive::<UInt64Type>();- assert_eq!(arr.value(0), 2); // 1 << 1 = 2- assert_eq!(arr.value(1), 8); // 2 << 2 = 8- assert_eq!(arr.value(2), 24); // 3 << 3 = 24- }-- #[test]- fn test_shift_left_nulls() {- let value_array = Arc::new(Int32Array::from(vec![Some(2), None, Some(3)]));- let shift_array = Arc::new(Int32Array::from(vec![Some(1), Some(2), None]));- let result = SparkShiftLeft::new()- .spark_shift(&[value_array, shift_array])- .unwrap();- let arr = result.as_primitive::<Int32Type>();- assert_eq!(arr.value(0), 4); // 2 << 1 = 4- assert!(arr.is_null(1)); // null << 2 = null- assert!(arr.is_null(2)); // 3 << null = null- }-- #[test]- fn test_shift_left_large_shift() {- let value_array = Arc::new(Int32Array::from(vec![1, 2, 3]));- let shift_array = Arc::new(Int32Array::from(vec![32, 33, 64]));- let result = SparkShiftLeft::new()- .spark_shift(&[value_array, shift_array])- .unwrap();- let arr = result.as_primitive::<Int32Type>();- assert_eq!(arr.value(0), 1); // 1 << 32 = 0 (overflow)- assert_eq!(arr.value(1), 4); // 2 << 33 = 0 (overflow)- assert_eq!(arr.value(2), 3); // 3 << 64 = 0 (overflow)- }-- #[test]- fn test_shift_left_negative_shift() {- let value_array = Arc::new(Int32Array::from(vec![4, 8, 16]));- let shift_array = Arc::new(Int32Array::from(vec![-1, -2, -3]));- let result = SparkShiftLeft::new()- .spark_shift(&[value_array, shift_array])- .unwrap();- let arr = result.as_primitive::<Int32Type>();- assert_eq!(arr.value(0), 0); // 4 << -1 = 0- assert_eq!(arr.value(1), 0); // 8 << -2 = 0- assert_eq!(arr.value(2), 0); // 16 << -3 = 0- }-}diff --git a/datafusion/spark/src/function/bitwise/mod.rs b/datafusion/spark/src/function/bitwise/mod.rsindex d729a3ddd0..769ecf5c2f 100644--- a/datafusion/spark/src/function/bitwise/mod.rs+++ b/datafusion/spark/src/function/bitwise/mod.rs@@ -24,9 +24,21 @@ use datafusion_expr::ScalarUDF;use datafusion_functions::make_udf_function;use std::sync::Arc;-make_udf_function!(bit_shift::SparkShiftLeft, shiftleft);-make_udf_function!(bit_shift::SparkShiftRight, shiftright);-make_udf_function!(bit_shift::SparkShiftRightUnsigned, shiftrightunsigned);+make_udf_function!(+ bit_shift::SparkBitShift,+ shiftleft,+ bit_shift::SparkBitShift::left+);+make_udf_function!(+ bit_shift::SparkBitShift,+ shiftright,+ bit_shift::SparkBitShift::right+);+make_udf_function!(+ bit_shift::SparkBitShift,+ shiftrightunsigned,+ bit_shift::SparkBitShift::right_unsigned+);make_udf_function!(bit_get::SparkBitGet, bit_get);make_udf_function!(bit_count::SparkBitCount, bit_count);make_udf_function!(bitwise_not::SparkBitwiseNot, bitwise_not);diff --git a/datafusion/sqllogictest/test_files/spark/bitwise/shiftright.slt b/datafusion/sqllogictest/test_files/spark/bitwise/shiftright.sltindex 3587bcc7ca..3dd43509b1 100644--- a/datafusion/sqllogictest/test_files/spark/bitwise/shiftright.slt+++ b/datafusion/sqllogictest/test_files/spark/bitwise/shiftright.slt@@ -145,3 +145,118 @@ query Iselect shiftright(3::int,-32);----3++# i32 + nulls+query IT+SELECT+ shiftright(arrow_cast(value, 'Int32'), shift),+ arrow_typeof(shiftright(arrow_cast(value, 'Int32'), shift))+FROM VALUES+(4, 1),+(8, 2),+(16, 3),+(32, 4),+(null, 2),+(8, null)+t(value, shift)+----+2 Int32+2 Int32+2 Int32+2 Int32+NULL Int32+NULL Int32++# big shifts+query IT+SELECT+ shiftright(arrow_cast(value, 'Int32'), shift),+ arrow_typeof(shiftright(arrow_cast(value, 'Int32'), shift))+FROM VALUES+(1, 32),+(2, 33),+(3, 64)+t(value, shift)+----+1 Int32+1 Int32+3 Int32++# negative shift+query IT+SELECT+ shiftright(arrow_cast(value, 'Int32'), shift),+ arrow_typeof(shiftright(arrow_cast(value, 'Int32'), shift))+FROM VALUES+(4, -1),+(8, -2),+(16, -3)+t(value, shift)+----+0 Int32+0 Int32+0 Int32++# For signed integers, right shift preserves sign bit+query IT+SELECT+ shiftright(arrow_cast(value, 'Int32'), shift),+ arrow_typeof(shiftright(arrow_cast(value, 'Int32'), shift))+FROM VALUES+(-4, 1),+(-8, 2),+(-16, 3)+t(value, shift)+----+-2 Int32+-2 Int32+-2 Int32++# i64 value+query IT+SELECT+ shiftright(arrow_cast(value, 'Int64'), shift),+ arrow_typeof(shiftright(arrow_cast(value, 'Int64'), shift))+FROM VALUES (4, 1), (8, 2), (16, 3) t(value, shift)+----+2 Int64+2 Int64+2 Int64++# u32 value+query IT+SELECT+ shiftright(arrow_cast(value, 'UInt32'), shift),+ arrow_typeof(shiftright(arrow_cast(value, 'UInt32'), shift))+FROM VALUES (4, 1), (8, 2), (16, 3) t(value, shift)+----+2 UInt32+2 UInt32+2 UInt32++# u64 value+query IT+SELECT+ shiftright(arrow_cast(value, 'UInt64'), shift),+ arrow_typeof(shiftright(arrow_cast(value, 'UInt64'), shift))+FROM VALUES (4, 1), (8, 2), (16, 3) t(value, shift)+----+2 UInt64+2 UInt64+2 UInt64++# pure null handling+query IT+SELECT shiftright(null, 1), arrow_typeof(shiftright(null, 1));+----+NULL Int32++query IT+SELECT shiftright(null, null), arrow_typeof(shiftright(null, null));+----+NULL Int32++query IT+SELECT shiftright(1::bigint, null), arrow_typeof(shiftright(1::bigint, null));+----+NULL Int64diff --git a/datafusion/sqllogictest/test_files/spark/bitwise/shiftrightunsigned.slt b/datafusion/sqllogictest/test_files/spark/bitwise/shiftrightunsigned.sltindex b0d4cfaec7..b9ef530b36 100644--- a/datafusion/sqllogictest/test_files/spark/bitwise/shiftrightunsigned.slt+++ b/datafusion/sqllogictest/test_files/spark/bitwise/shiftrightunsigned.slt@@ -146,3 +146,106 @@ query Iselect shiftrightunsigned(3::int,-32);----3++# i32 + nulls+query IT+SELECT+ shiftrightunsigned(arrow_cast(value, 'Int32'), shift),+ arrow_typeof(shiftrightunsigned(arrow_cast(value, 'Int32'), shift))+FROM VALUES+(4, 1),+(8, 2),+(16, 3),+(32, 4),+(null, 2),+(8, null)+t(value, shift)+----+2 Int32+2 Int32+2 Int32+2 Int32+NULL Int32+NULL Int32++# negative shift+query IT+SELECT+ shiftrightunsigned(arrow_cast(value, 'Int32'), shift),+ arrow_typeof(shiftrightunsigned(arrow_cast(value, 'Int32'), shift))+FROM VALUES+(4, -1),+(8, -2),+(16, -3)+t(value, shift)+----+0 Int32+0 Int32+0 Int32++# Negative values are treated as large positive values+# -4 as u32 = 4294967292, 4294967292 >>> 1 = 2147483646+# -8 as u32 = 4294967288, 4294967288 >>> 2 = 1073741822+# -16 as u32 = 4294967280, 4294967280 >>> 3 = 536870910+query IT+SELECT+ shiftrightunsigned(arrow_cast(value, 'Int32'), shift),+ arrow_typeof(shiftrightunsigned(arrow_cast(value, 'Int32'), shift))+FROM VALUES+(-4, 1),+(-8, 2),+(-16, 3)+t(value, shift)+----+2147483646 Int32+1073741822 Int32+536870910 Int32++# i64 value+query IT+SELECT+ shiftrightunsigned(arrow_cast(value, 'Int64'), shift),+ arrow_typeof(shiftrightunsigned(arrow_cast(value, 'Int64'), shift))+FROM VALUES (4, 1), (8, 2), (16, 3) t(value, shift)+----+2 Int64+2 Int64+2 Int64++# u32 value+query IT+SELECT+ shiftrightunsigned(arrow_cast(value, 'UInt32'), shift),+ arrow_typeof(shiftrightunsigned(arrow_cast(value, 'UInt32'), shift))+FROM VALUES (4, 1), (8, 2), (16, 3) t(value, shift)+----+2 UInt32+2 UInt32+2 UInt32++# u64 value+query IT+SELECT+ shiftrightunsigned(arrow_cast(value, 'UInt64'), shift),+ arrow_typeof(shiftrightunsigned(arrow_cast(value, 'UInt64'), shift))+FROM VALUES (4, 1), (8, 2), (16, 3) t(value, shift)+----+2 UInt64+2 UInt64+2 UInt64++# pure null handling+query IT+SELECT shiftrightunsigned(null, 1), arrow_typeof(shiftrightunsigned(null, 1));+----+NULL Int32++query IT+SELECT shiftrightunsigned(null, null), arrow_typeof(shiftrightunsigned(null, null));+----+NULL Int32++query IT+SELECT shiftrightunsigned(1::bigint, null), arrow_typeof(shiftrightunsigned(1::bigint, null));+----+NULL Int64diff --git a/datafusion/sqllogictest/test_files/spark/math/shiftleft.slt b/datafusion/sqllogictest/test_files/spark/math/shiftleft.sltindex 3676e4c181..c8ddeb6740 100644--- a/datafusion/sqllogictest/test_files/spark/math/shiftleft.slt+++ b/datafusion/sqllogictest/test_files/spark/math/shiftleft.slt@@ -144,3 +144,103 @@ query Iselect shiftleft(3::int,-32);----3++# i32 + nulls+query IT+SELECT+ shiftleft(arrow_cast(value, 'Int32'), shift),+ arrow_typeof(shiftleft(arrow_cast(value, 'Int32'), shift))+FROM VALUES+(1, 1),+(2, 2),+(3, 3),+(4, 4),+(null, 2),+(8, null)+t(value, shift)+----+2 Int32+8 Int32+24 Int32+64 Int32+NULL Int32+NULL Int32++# big shifts+query IT+SELECT+ shiftleft(arrow_cast(value, 'Int32'), shift),+ arrow_typeof(shiftleft(arrow_cast(value, 'Int32'), shift))+FROM VALUES+(1, 32),+(2, 33),+(3, 64)+t(value, shift)+----+1 Int32+4 Int32+3 Int32++# negative shift+query IT+SELECT+ shiftleft(arrow_cast(value, 'Int32'), shift),+ arrow_typeof(shiftleft(arrow_cast(value, 'Int32'), shift))+FROM VALUES+(4, -1),+(8, -2),+(16, -3)+t(value, shift)+----+0 Int32+0 Int32+0 Int32++# i64 value+query IT+SELECT+ shiftleft(arrow_cast(value, 'Int64'), shift),+ arrow_typeof(shiftleft(arrow_cast(value, 'Int64'), shift))+FROM VALUES (1, 1), (2, 2), (3, 3) t(value, shift)+----+2 Int64+8 Int64+24 Int64++# u32 value+query IT+SELECT+ shiftleft(arrow_cast(value, 'UInt32'), shift),+ arrow_typeof(shiftleft(arrow_cast(value, 'UInt32'), shift))+FROM VALUES (1, 1), (2, 2), (3, 3) t(value, shift)+----+2 UInt32+8 UInt32+24 UInt32++# u64 value+query IT+SELECT+ shiftleft(arrow_cast(value, 'UInt64'), shift),+ arrow_typeof(shiftleft(arrow_cast(value, 'UInt64'), shift))+FROM VALUES (1, 1), (2, 2), (3, 3) t(value, shift)+----+2 UInt64+8 UInt64+24 UInt64++# pure null handling+query IT+SELECT shiftleft(null, 1), arrow_typeof(shiftleft(null, 1));+----+NULL Int32++query IT+SELECT shiftleft(null, null), arrow_typeof(shiftleft(null, null));+----+NULL Int32++query IT+SELECT shiftleft(1::bigint, null), arrow_typeof(shiftleft(1::bigint, null));+----+NULL Int64