Apache DataFusion SQL Query Engine
perf: avoid per-row String allocation in Spark bin and char (#23881)
Andy Grove36c417b6c142
diff --git a/datafusion/spark/src/function/math/bin.rs b/datafusion/spark/src/function/math/bin.rsindex 82afd48e8d..e6a0e1a735 100644--- a/datafusion/spark/src/function/math/bin.rs+++ b/datafusion/spark/src/function/math/bin.rs@@ -15,7 +15,7 @@// specific language governing permissions and limitations// under the License.-use arrow::array::{ArrayRef, AsArray, StringArray};+use arrow::array::{Array, ArrayRef, AsArray, StringBuilder};use arrow::datatypes::{DataType, Field, FieldRef, Int64Type};use datafusion_common::types::{NativeType, logical_int64};use datafusion_common::utils::take_function_args;@@ -88,12 +88,20 @@ fn spark_bin_inner(arg: &[ArrayRef]) -> Result<ArrayRef> {let [array] = take_function_args("bin", arg)?;match &array.data_type() {DataType::Int64 => {- let result: StringArray = array- .as_primitive::<Int64Type>()- .iter()- .map(|opt| opt.map(spark_bin))- .collect();- Ok(Arc::new(result))+ let array = array.as_primitive::<Int64Type>();+ let len = array.len();+ // Most values are small, so 8 digits per row is a reasonable estimate;+ // the buffer grows on its own for wider ones.+ let mut builder = StringBuilder::with_capacity(len, len * 8);+ // Digits are rendered into this stack buffer, so no row allocates.+ let mut digits = [0u8; MAX_BIN_DIGITS];+ for value in array.iter() {+ match value {+ Some(value) => builder.append_value(spark_bin(value, &mut digits)),+ None => builder.append_null(),+ }+ }+ Ok(Arc::new(builder.finish()))}data_type => {internal_err!("bin does not support: {data_type}")@@ -101,6 +109,24 @@ fn spark_bin_inner(arg: &[ArrayRef]) -> Result<ArrayRef> {}}-fn spark_bin(value: i64) -> String {- format!("{value:b}")+/// An `i64` renders as at most 64 binary digits.+const MAX_BIN_DIGITS: usize = 64;++/// Renders `value` as binary, right-aligned in `digits`, and returns the digits written.+///+/// Negative values render as their two's-complement bit pattern, matching `{:b}`.+fn spark_bin(value: i64, digits: &mut [u8; MAX_BIN_DIGITS]) -> &str {+ let mut pos = MAX_BIN_DIGITS;+ let mut remaining = value as u64;+ // `while` alone would produce an empty string for zero.+ loop {+ pos -= 1;+ digits[pos] = b'0' + (remaining & 1) as u8;+ remaining >>= 1;+ if remaining == 0 {+ break;+ }+ }+ // SAFETY: every byte written above is an ASCII '0' or '1'.+ unsafe { std::str::from_utf8_unchecked(&digits[pos..]) }}diff --git a/datafusion/spark/src/function/string/char.rs b/datafusion/spark/src/function/string/char.rsindex 15b00ee98f..5d6de3ae36 100644--- a/datafusion/spark/src/function/string/char.rs+++ b/datafusion/spark/src/function/string/char.rs@@ -112,6 +112,8 @@ fn chr(args: &[ArrayRef]) -> Result<ArrayRef> {integer_array.len(),);+ // Each character encodes into this stack buffer, so no row allocates a `String`.+ let mut encoded = [0u8; 4];for integer_opt in integer_array {match integer_opt {Some(integer) => {@@ -119,7 +121,7 @@ fn chr(args: &[ArrayRef]) -> Result<ArrayRef> {builder.append_value(""); // empty string for negative numbers.} else {match core::char::from_u32((integer % 256) as u32) {- Some(ch) => builder.append_value(ch.to_string()),+ Some(ch) => builder.append_value(ch.encode_utf8(&mut encoded)),None => {return exec_err!("requested character not compatible for encoding."