Apache DataFusion SQL Query Engine
Refactor spark `bit_get()` signature away from user defined (#18836)
Jeffrey Vo5e36b052e961
diff --git a/datafusion/spark/src/function/bitwise/bit_get.rs b/datafusion/spark/src/function/bitwise/bit_get.rsindex a8562618cb..bc8d8cdbd1 100644--- a/datafusion/spark/src/function/bitwise/bit_get.rs+++ b/datafusion/spark/src/function/bitwise/bit_get.rs@@ -19,25 +19,21 @@ use std::any::Any;use std::mem::size_of;use std::sync::Arc;-use arrow::array::{Array, ArrayRef, ArrowPrimitiveType, AsArray, PrimitiveArray};-use arrow::compute::try_binary;-use arrow::datatypes::DataType::{- Int16, Int32, Int64, Int8, UInt16, UInt32, UInt64, UInt8,-};-use arrow::datatypes::{- ArrowNativeType, DataType, Int16Type, Int32Type, Int64Type, Int8Type, UInt16Type,- UInt32Type, UInt64Type, UInt8Type,+use arrow::array::{+ downcast_integer_array, Array, ArrayRef, ArrowPrimitiveType, AsArray, Int32Array,+ Int8Array, PrimitiveArray,};-use datafusion_common::{exec_err, Result};+use arrow::compute::try_binary;+use arrow::datatypes::{ArrowNativeType, DataType, Int32Type, Int8Type};+use datafusion_common::types::{logical_int32, NativeType};+use datafusion_common::utils::take_function_args;+use datafusion_common::{internal_err, Result};use datafusion_expr::{- ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,+ Coercion, ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature,+ TypeSignatureClass, Volatility,};use datafusion_functions::utils::make_scalar_function;-use crate::function::error_utils::{- invalid_arg_count_exec_err, unsupported_data_type_exec_err,-};-#[derive(Debug, PartialEq, Eq, Hash)]pub struct SparkBitGet {signature: Signature,@@ -53,7 +49,17 @@ impl Default for SparkBitGet {impl SparkBitGet {pub fn new() -> Self {Self {- signature: Signature::user_defined(Volatility::Immutable),+ signature: Signature::coercible(+ vec![+ Coercion::new_exact(TypeSignatureClass::Integer),+ Coercion::new_implicit(+ TypeSignatureClass::Native(logical_int32()),+ vec![TypeSignatureClass::Integer],+ NativeType::Int32,+ ),+ ],+ Volatility::Immutable,+ ),aliases: vec!["getbit".to_string()],}}@@ -64,34 +70,6 @@ impl ScalarUDFImpl for SparkBitGet {self}- fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {- if arg_types.len() != 2 {- return Err(invalid_arg_count_exec_err(- "bit_get",- (2, 2),- arg_types.len(),- ));- }- if !arg_types[0].is_integer() && !arg_types[0].is_null() {- return Err(unsupported_data_type_exec_err(- "bit_get",- "Integer Type",- &arg_types[0],- ));- }- if !arg_types[1].is_integer() && !arg_types[1].is_null() {- return Err(unsupported_data_type_exec_err(- "bit_get",- "Integer Type",- &arg_types[1],- ));- }- if arg_types[0].is_null() {- return Ok(vec![Int8, Int32]);- }- Ok(vec![arg_types[0].clone(), Int32])- }-fn name(&self) -> &str {"bit_get"}@@ -105,7 +83,7 @@ impl ScalarUDFImpl for SparkBitGet {}fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {- Ok(Int8)+ Ok(DataType::Int8)}fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {@@ -115,8 +93,8 @@ impl ScalarUDFImpl for SparkBitGet {fn spark_bit_get_inner<T: ArrowPrimitiveType>(value: &PrimitiveArray<T>,- pos: &PrimitiveArray<Int32Type>,-) -> Result<PrimitiveArray<Int8Type>> {+ pos: &Int32Array,+) -> Result<Int8Array> {let bit_length = (size_of::<T::Native>() * 8) as i32;let result: PrimitiveArray<Int8Type> = try_binary(value, pos, |value, pos| {@@ -130,164 +108,13 @@ fn spark_bit_get_inner<T: ArrowPrimitiveType>(Ok(result)}-pub fn spark_bit_get(args: &[ArrayRef]) -> Result<ArrayRef> {- if args.len() != 2 {- return exec_err!("`bit_get` expects exactly two arguments");- }-- if args[1].data_type() != &Int32 {- return exec_err!("`bit_get` expects Int32 as the second argument");- }-- let pos_arg = args[1].as_primitive::<Int32Type>();-- let ret = match &args[0].data_type() {- Int64 => {- let value_arg = args[0].as_primitive::<Int64Type>();- spark_bit_get_inner(value_arg, pos_arg)- }- Int32 => {- let value_arg = args[0].as_primitive::<Int32Type>();- spark_bit_get_inner(value_arg, pos_arg)- }- Int16 => {- let value_arg = args[0].as_primitive::<Int16Type>();- spark_bit_get_inner(value_arg, pos_arg)- }- Int8 => {- let value_arg = args[0].as_primitive::<Int8Type>();- spark_bit_get_inner(value_arg, pos_arg)- }- UInt64 => {- let value_arg = args[0].as_primitive::<UInt64Type>();- spark_bit_get_inner(value_arg, pos_arg)- }- UInt32 => {- let value_arg = args[0].as_primitive::<UInt32Type>();- spark_bit_get_inner(value_arg, pos_arg)- }- UInt16 => {- let value_arg = args[0].as_primitive::<UInt16Type>();- spark_bit_get_inner(value_arg, pos_arg)- }- UInt8 => {- let value_arg = args[0].as_primitive::<UInt8Type>();- spark_bit_get_inner(value_arg, pos_arg)- }- _ => {- exec_err!(- "`bit_get` expects Int64, Int32, Int16, or Int8 as the first argument"- )- }- }?;+fn spark_bit_get(args: &[ArrayRef]) -> Result<ArrayRef> {+ let [value, position] = take_function_args("bit_get", args)?;+ let pos_arg = position.as_primitive::<Int32Type>();+ let ret = downcast_integer_array!(+ value => spark_bit_get_inner(value, pos_arg),+ DataType::Null => Ok(Int8Array::new_null(value.len())),+ d => internal_err!("Unsupported datatype for bit_get: {d}"),+ )?;Ok(Arc::new(ret))}--#[cfg(test)]-mod tests {- use arrow::array::{Int32Array, Int64Array};-- use super::*;-- #[test]- fn test_bit_get_basic() {- // Test bit_get(11, 0) - 11 = 1011 in binary, bit 0 = 1- let result = spark_bit_get(&[- Arc::new(Int64Array::from(vec![11])),- Arc::new(Int32Array::from(vec![0])),- ])- .unwrap();-- assert_eq!(result.as_primitive::<Int8Type>().value(0), 1);-- // Test bit_get(11, 2) - 11 = 1011 in binary, bit 2 = 0- let result = spark_bit_get(&[- Arc::new(Int64Array::from(vec![11])),- Arc::new(Int32Array::from(vec![2])),- ])- .unwrap();-- assert_eq!(result.as_primitive::<Int8Type>().value(0), 0);-- // Test bit_get(11, 3) - 11 = 1011 in binary, bit 3 = 1- let result = spark_bit_get(&[- Arc::new(Int64Array::from(vec![11])),- Arc::new(Int32Array::from(vec![3])),- ])- .unwrap();-- assert_eq!(result.as_primitive::<Int8Type>().value(0), 1);- }-- #[test]- fn test_bit_get_edge_cases() {- // Test with 0- let result = spark_bit_get(&[- Arc::new(Int64Array::from(vec![0])),- Arc::new(Int32Array::from(vec![0])),- ])- .unwrap();-- assert_eq!(result.as_primitive::<Int8Type>().value(0), 0);-- let result = spark_bit_get(&[- Arc::new(Int64Array::from(vec![11])),- Arc::new(Int32Array::from(vec![-1])),- ]);- assert_eq!(- result.unwrap_err().message().lines().next().unwrap(),- "Compute error: bit_get: position -1 is out of bounds. Expected pos < 64 and pos >= 0"- );-- let result = spark_bit_get(&[- Arc::new(Int64Array::from(vec![11])),- Arc::new(Int32Array::from(vec![64])),- ]);-- assert_eq!(- result.unwrap_err().message().lines().next().unwrap(),- "Compute error: bit_get: position 64 is out of bounds. Expected pos < 64 and pos >= 0"- );- }-- #[test]- fn test_bit_get_null_inputs() {- // Test with NULL value- let result = spark_bit_get(&[- Arc::new(Int64Array::from(vec![None])),- Arc::new(Int32Array::from(vec![0])),- ])- .unwrap();-- assert_eq!(result.as_primitive::<Int8Type>().value(0), 0);-- // Test with NULL position- let result = spark_bit_get(&[- Arc::new(Int64Array::from(vec![11])),- Arc::new(Int32Array::from(vec![None])),- ])- .unwrap();-- assert_eq!(result.as_primitive::<Int8Type>().value(0), 0);- }-- #[test]- fn test_bit_get_large_numbers() {- // Test with larger number- let result = spark_bit_get(&[- Arc::new(Int64Array::from(vec![255])), // 11111111 in binary- Arc::new(Int32Array::from(vec![7])), // bit 7 = 1- ])- .unwrap();-- assert_eq!(result.as_primitive::<Int8Type>().value(0), 1);-- let result = spark_bit_get(&[- Arc::new(Int64Array::from(vec![255])), // 11111111 in binary- Arc::new(Int32Array::from(vec![8])), // bit 8 = 0- ])- .unwrap();-- assert_eq!(result.as_primitive::<Int8Type>().value(0), 0);- }-}diff --git a/datafusion/sqllogictest/test_files/spark/bitwise/bit_get.slt b/datafusion/sqllogictest/test_files/spark/bitwise/bit_get.sltindex 6a2b244d58..faba0b66c4 100644--- a/datafusion/sqllogictest/test_files/spark/bitwise/bit_get.slt+++ b/datafusion/sqllogictest/test_files/spark/bitwise/bit_get.slt@@ -69,7 +69,32 @@ SELECT bit_get(NULL, 0);----NULL+query I+SELECT bit_get(NULL::int, 0);+----+NULL+query ISELECT bit_get(11, NULL);----NULL++query I+SELECT bit_get(11, NULL::int);+----+NULL++query I+SELECT bit_get(11::tinyint, 0);+----+1++query I+SELECT bit_get(11::bigint, 0);+----+1++query I+SELECT bit_get(11, 3::bigint);+----+1