Skip to content

Apache DataFusion SQL Query Engine

perf: avoid per-row String allocation in Spark bin and char (#23881)

Andy Grove36c417b6c142
diff --git a/datafusion/spark/src/function/math/bin.rs b/datafusion/spark/src/function/math/bin.rs
index 82afd48e8d..e6a0e1a735 100644
--- a/datafusion/spark/src/function/math/bin.rs
+++ b/datafusion/spark/src/function/math/bin.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use arrow::array::{ArrayRef, AsArray, StringArray};
+use arrow::array::{Array, ArrayRef, AsArray, StringBuilder};
use arrow::datatypes::{DataType, Field, FieldRef, Int64Type};
use datafusion_common::types::{NativeType, logical_int64};
use datafusion_common::utils::take_function_args;
@@ -88,12 +88,20 @@ fn spark_bin_inner(arg: &[ArrayRef]) -> Result<ArrayRef> {
let [array] = take_function_args("bin", arg)?;
match &array.data_type() {
DataType::Int64 => {
- let result: StringArray = array
- .as_primitive::<Int64Type>()
- .iter()
- .map(|opt| opt.map(spark_bin))
- .collect();
- Ok(Arc::new(result))
+ let array = array.as_primitive::<Int64Type>();
+ let len = array.len();
+ // Most values are small, so 8 digits per row is a reasonable estimate;
+ // the buffer grows on its own for wider ones.
+ let mut builder = StringBuilder::with_capacity(len, len * 8);
+ // Digits are rendered into this stack buffer, so no row allocates.
+ let mut digits = [0u8; MAX_BIN_DIGITS];
+ for value in array.iter() {
+ match value {
+ Some(value) => builder.append_value(spark_bin(value, &mut digits)),
+ None => builder.append_null(),
+ }
+ }
+ Ok(Arc::new(builder.finish()))
}
data_type => {
internal_err!("bin does not support: {data_type}")
@@ -101,6 +109,24 @@ fn spark_bin_inner(arg: &[ArrayRef]) -> Result<ArrayRef> {
}
}
-fn spark_bin(value: i64) -> String {
- format!("{value:b}")
+/// An `i64` renders as at most 64 binary digits.
+const MAX_BIN_DIGITS: usize = 64;
+
+/// Renders `value` as binary, right-aligned in `digits`, and returns the digits written.
+///
+/// Negative values render as their two's-complement bit pattern, matching `{:b}`.
+fn spark_bin(value: i64, digits: &mut [u8; MAX_BIN_DIGITS]) -> &str {
+ let mut pos = MAX_BIN_DIGITS;
+ let mut remaining = value as u64;
+ // `while` alone would produce an empty string for zero.
+ loop {
+ pos -= 1;
+ digits[pos] = b'0' + (remaining & 1) as u8;
+ remaining >>= 1;
+ if remaining == 0 {
+ break;
+ }
+ }
+ // SAFETY: every byte written above is an ASCII '0' or '1'.
+ unsafe { std::str::from_utf8_unchecked(&digits[pos..]) }
}
diff --git a/datafusion/spark/src/function/string/char.rs b/datafusion/spark/src/function/string/char.rs
index 15b00ee98f..5d6de3ae36 100644
--- a/datafusion/spark/src/function/string/char.rs
+++ b/datafusion/spark/src/function/string/char.rs
@@ -112,6 +112,8 @@ fn chr(args: &[ArrayRef]) -> Result<ArrayRef> {
integer_array.len(),
);
+ // Each character encodes into this stack buffer, so no row allocates a `String`.
+ let mut encoded = [0u8; 4];
for integer_opt in integer_array {
match integer_opt {
Some(integer) => {
@@ -119,7 +121,7 @@ fn chr(args: &[ArrayRef]) -> Result<ArrayRef> {
builder.append_value(""); // empty string for negative numbers.
} else {
match core::char::from_u32((integer % 256) as u32) {
- Some(ch) => builder.append_value(ch.to_string()),
+ Some(ch) => builder.append_value(ch.encode_utf8(&mut encoded)),
None => {
return exec_err!(
"requested character not compatible for encoding."