Skip to content

Apache DataFusion SQL Query Engine

perf: preallocate memory in `pad` (#23586)

theirix2e785088018b
diff --git a/datafusion/functions/src/unicode/common.rs b/datafusion/functions/src/unicode/common.rs
index 5b7262e29d..9f91e4f1b0 100644
--- a/datafusion/functions/src/unicode/common.rs
+++ b/datafusion/functions/src/unicode/common.rs
@@ -50,6 +50,16 @@ pub(crate) fn try_as_scalar_i64(cv: &ColumnarValue) -> Option<i64> {
}
}
+/// Estimates data capacity for `pad` based on `length_array` with row length.
+/// For ASCII, one row is at most `target_len` bytes.
+/// For UTF8, it could be larger
+pub(crate) fn pad_data_capacity(length_array: &Int64Array) -> usize {
+ length_array
+ .iter()
+ .flatten()
+ .fold(0, |acc, len| acc.saturating_add(len as usize))
+}
+
/// A trait for `left` and `right` byte slicing operations
pub(crate) trait LeftRightSlicer {
fn slice(string: &str, n: i64) -> Range<usize>;
diff --git a/datafusion/functions/src/unicode/lpad.rs b/datafusion/functions/src/unicode/lpad.rs
index d27bc8633e..40bffeecf4 100644
--- a/datafusion/functions/src/unicode/lpad.rs
+++ b/datafusion/functions/src/unicode/lpad.rs
@@ -178,7 +178,8 @@ impl ScalarUDFImpl for LPadFunc {
}
use super::common::{
- StringCharLen, char_count_or_boundary, try_as_scalar_i64, try_as_scalar_str,
+ StringCharLen, char_count_or_boundary, pad_data_capacity, try_as_scalar_i64,
+ try_as_scalar_str,
};
/// Optimized lpad for constant target_len and fill arguments.
@@ -373,7 +374,10 @@ where
T: OffsetSizeTrait,
{
let array = if let Some(fill_array) = fill_array {
- let mut builder: GenericStringBuilder<T> = GenericStringBuilder::new();
+ let mut builder: GenericStringBuilder<T> = GenericStringBuilder::with_capacity(
+ string_array.len(),
+ pad_data_capacity(length_array),
+ );
let mut fill_chars_buf = Vec::new();
for ((string, target_len), fill) in string_array
@@ -449,7 +453,10 @@ where
builder.finish()
} else {
- let mut builder: GenericStringBuilder<T> = GenericStringBuilder::new();
+ let mut builder: GenericStringBuilder<T> = GenericStringBuilder::with_capacity(
+ string_array.len(),
+ pad_data_capacity(length_array),
+ );
for (string, target_len) in string_array.iter().zip(length_array.iter()) {
if let (Some(string), Some(target_len)) = (string, target_len) {
diff --git a/datafusion/functions/src/unicode/rpad.rs b/datafusion/functions/src/unicode/rpad.rs
index b3e14f9352..784a2037cf 100644
--- a/datafusion/functions/src/unicode/rpad.rs
+++ b/datafusion/functions/src/unicode/rpad.rs
@@ -178,7 +178,8 @@ impl ScalarUDFImpl for RPadFunc {
}
use super::common::{
- StringCharLen, char_count_or_boundary, try_as_scalar_i64, try_as_scalar_str,
+ StringCharLen, char_count_or_boundary, pad_data_capacity, try_as_scalar_i64,
+ try_as_scalar_str,
};
/// Optimized rpad for constant target_len and fill arguments.
@@ -372,7 +373,10 @@ where
T: OffsetSizeTrait,
{
let array = if let Some(fill_array) = fill_array {
- let mut builder: GenericStringBuilder<T> = GenericStringBuilder::new();
+ let mut builder: GenericStringBuilder<T> = GenericStringBuilder::with_capacity(
+ string_array.len(),
+ pad_data_capacity(length_array),
+ );
let mut fill_chars_buf = Vec::new();
for ((string, target_len), fill) in string_array
@@ -450,7 +454,10 @@ where
builder.finish()
} else {
- let mut builder: GenericStringBuilder<T> = GenericStringBuilder::new();
+ let mut builder: GenericStringBuilder<T> = GenericStringBuilder::with_capacity(
+ string_array.len(),
+ pad_data_capacity(length_array),
+ );
for (string, target_len) in string_array.iter().zip(length_array.iter()) {
if let (Some(string), Some(target_len)) = (string, target_len) {