Apache DataFusion SQL Query Engine
Fix: spark bit_count function (#18322)
Kazantsev Maksimd445b619ba24
diff --git a/datafusion/spark/src/function/bitwise/bit_count.rs b/datafusion/spark/src/function/bitwise/bit_count.rsindex ba44d3bc0a..4b414b57cb 100644--- a/datafusion/spark/src/function/bitwise/bit_count.rs+++ b/datafusion/spark/src/function/bitwise/bit_count.rs@@ -23,6 +23,7 @@ use arrow::datatypes::{DataType, Int16Type, Int32Type, Int64Type, Int8Type, UInt16Type, UInt32Type,UInt64Type, UInt8Type,};+use datafusion_common::cast::as_boolean_array;use datafusion_common::{plan_err, Result};use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature,@@ -46,6 +47,7 @@ impl SparkBitCount {Self {signature: Signature::one_of(vec![+ TypeSignature::Exact(vec![DataType::Boolean]),TypeSignature::Exact(vec![DataType::Int8]),TypeSignature::Exact(vec![DataType::Int16]),TypeSignature::Exact(vec![DataType::Int32]),@@ -90,28 +92,34 @@ impl ScalarUDFImpl for SparkBitCount {fn spark_bit_count(value_array: &[ArrayRef]) -> Result<ArrayRef> {let value_array = value_array[0].as_ref();match value_array.data_type() {+ DataType::Boolean => {+ let result: Int32Array = as_boolean_array(value_array)?+ .iter()+ .map(|x| x.map(|y| y as i32))+ .collect();+ Ok(Arc::new(result))+ }DataType::Int8 => {let result: Int32Array = value_array.as_primitive::<Int8Type>()- .unary(|v| v.count_ones() as i32);+ .unary(|v| bit_count(v.into()));Ok(Arc::new(result))}DataType::Int16 => {let result: Int32Array = value_array.as_primitive::<Int16Type>()- .unary(|v| v.count_ones() as i32);+ .unary(|v| bit_count(v.into()));Ok(Arc::new(result))}DataType::Int32 => {let result: Int32Array = value_array.as_primitive::<Int32Type>()- .unary(|v| v.count_ones() as i32);+ .unary(|v| bit_count(v.into()));Ok(Arc::new(result))}DataType::Int64 => {- let result: Int32Array = value_array- .as_primitive::<Int64Type>()- .unary(|v| v.count_ones() as i32);+ let result: Int32Array =+ value_array.as_primitive::<Int64Type>().unary(bit_count);Ok(Arc::new(result))}DataType::UInt8 => {@@ -147,12 +155,26 @@ fn spark_bit_count(value_array: &[ArrayRef]) -> Result<ArrayRef> {}}+// Here’s the equivalent Rust implementation of the bitCount function (similar to Apache Spark's bitCount for LongType)+// Spark: https://github.com/apache/spark/blob/ac717dd7aec665de578d7c6b0070e8fcdde3cea9/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala#L243+// Java impl: https://github.com/openjdk/jdk/blob/d226023643f90027a8980d161ec6d423887ae3ce/src/java.base/share/classes/java/lang/Long.java#L1584+fn bit_count(i: i64) -> i32 {+ let mut u = i as u64;+ u = u - ((u >> 1) & 0x5555555555555555);+ u = (u & 0x3333333333333333) + ((u >> 2) & 0x3333333333333333);+ u = (u + (u >> 4)) & 0x0f0f0f0f0f0f0f0f;+ u = u + (u >> 8);+ u = u + (u >> 16);+ u = u + (u >> 32);+ (u as i32) & 0x7f+}+#[cfg(test)]mod tests {use super::*;use arrow::array::{- Array, Int16Array, Int32Array, Int64Array, Int8Array, UInt16Array, UInt32Array,- UInt64Array, UInt8Array,+ Array, BooleanArray, Int16Array, Int32Array, Int64Array, Int8Array, UInt16Array,+ UInt32Array, UInt64Array, UInt8Array,};use arrow::datatypes::Int32Type;@@ -192,7 +214,18 @@ mod tests {assert_eq!(arr.value(2), 2);assert_eq!(arr.value(3), 3);assert_eq!(arr.value(4), 4);- assert_eq!(arr.value(5), 8);+ assert_eq!(arr.value(5), 64);+ }++ #[test]+ fn test_bit_count_boolean() {+ // Test bit_count on BooleanArray+ let result =+ spark_bit_count(&[Arc::new(BooleanArray::from(vec![true, false]))]).unwrap();++ let arr = result.as_primitive::<Int32Type>();+ assert_eq!(arr.value(0), 1);+ assert_eq!(arr.value(1), 0);}#[test]@@ -207,7 +240,7 @@ mod tests {assert_eq!(arr.value(1), 1);assert_eq!(arr.value(2), 8);assert_eq!(arr.value(3), 10);- assert_eq!(arr.value(4), 16);+ assert_eq!(arr.value(4), 64);}#[test]@@ -222,7 +255,7 @@ mod tests {assert_eq!(arr.value(1), 1); // 0b00000000000000000000000000000001 = 1assert_eq!(arr.value(2), 8); // 0b00000000000000000000000011111111 = 8assert_eq!(arr.value(3), 10); // 0b00000000000000000000001111111111 = 10- assert_eq!(arr.value(4), 32); // -1 in two's complement = all 32 bits set+ assert_eq!(arr.value(4), 64); // -1 in two's complement = all 32 bits set}#[test]diff --git a/datafusion/sqllogictest/test_files/spark/bitwise/bit_count.slt b/datafusion/sqllogictest/test_files/spark/bitwise/bit_count.sltindex 2a75c7648d..216d990251 100644--- a/datafusion/sqllogictest/test_files/spark/bitwise/bit_count.slt+++ b/datafusion/sqllogictest/test_files/spark/bitwise/bit_count.slt@@ -59,17 +59,17 @@ SELECT bit_count(1023::int);query ISELECT bit_count(-1::int);-----32+64query ISELECT bit_count(-2::int);-----31+63query ISELECT bit_count(-3::int);-----31+63# Tests with different integer typesquery I@@ -85,7 +85,7 @@ SELECT bit_count(arrow_cast(15, 'Int8'));query ISELECT bit_count(arrow_cast(-1, 'Int8'));-----8+64query ISELECT bit_count(arrow_cast(0, 'Int16'));@@ -100,7 +100,7 @@ SELECT bit_count(arrow_cast(255, 'Int16'));query ISELECT bit_count(arrow_cast(-1, 'Int16'));-----16+64query ISELECT bit_count(arrow_cast(0, 'Int64'));@@ -214,7 +214,7 @@ SELECT bit_count(arrow_cast(2147483647, 'Int32'));query ISELECT bit_count(arrow_cast(-2147483648, 'Int32'));-----1+33query ISELECT bit_count(arrow_cast(9223372036854775807, 'Int64'));