Skip to content

Apache DataFusion SQL Query Engine

refactor: share a single pad kernel between lpad and rpad (#26367)

shazilkhan29580fa3d9f0
diff --git a/datafusion/functions/benches/unicode_expressions/pad.rs b/datafusion/functions/benches/unicode_expressions/pad.rs
index b766203383..46c3411809 100644
--- a/datafusion/functions/benches/unicode_expressions/pad.rs
+++ b/datafusion/functions/benches/unicode_expressions/pad.rs
@@ -363,6 +363,60 @@ fn criterion_benchmark(c: &mut Criterion) {
},
);
+ // Utf8 type without fill (pads with spaces)
+ let mut args = create_pad_args::<i32>(size, 5, 20, false);
+ args.truncate(2);
+ let arg_fields = args
+ .iter()
+ .enumerate()
+ .map(|(idx, arg)| {
+ Field::new(format!("arg_{idx}"), arg.data_type(), true).into()
+ })
+ .collect::<Vec<_>>();
+
+ group.bench_function(
+ format!("lpad utf8 no fill [size={size}, str_len=5, target=20]"),
+ |b| {
+ b.iter(|| {
+ let args_cloned = args.clone();
+ black_box(unicode::lpad().invoke_with_args(ScalarFunctionArgs {
+ args: args_cloned,
+ arg_fields: arg_fields.clone(),
+ number_rows: size,
+ return_field: Field::new("f", DataType::Utf8, true).into(),
+ config_options: Arc::clone(&config_options),
+ }))
+ })
+ },
+ );
+
+ // Utf8 type with Unicode strings, without fill
+ let mut args = create_unicode_pad_args(size, 20, false);
+ args.truncate(2);
+ let arg_fields = args
+ .iter()
+ .enumerate()
+ .map(|(idx, arg)| {
+ Field::new(format!("arg_{idx}"), arg.data_type(), true).into()
+ })
+ .collect::<Vec<_>>();
+
+ group.bench_function(
+ format!("lpad utf8 unicode no fill [size={size}, target=20]"),
+ |b| {
+ b.iter(|| {
+ let args_cloned = args.clone();
+ black_box(unicode::lpad().invoke_with_args(ScalarFunctionArgs {
+ args: args_cloned,
+ arg_fields: arg_fields.clone(),
+ number_rows: size,
+ return_field: Field::new("f", DataType::Utf8, true).into(),
+ config_options: Arc::clone(&config_options),
+ }))
+ })
+ },
+ );
+
// --- Scalar length + fill benchmarks ---
// Utf8 with scalar length and fill (3-arg)
@@ -641,6 +695,60 @@ fn criterion_benchmark(c: &mut Criterion) {
},
);
+ // Utf8 type without fill (pads with spaces)
+ let mut args = create_pad_args::<i32>(size, 5, 20, false);
+ args.truncate(2);
+ let arg_fields = args
+ .iter()
+ .enumerate()
+ .map(|(idx, arg)| {
+ Field::new(format!("arg_{idx}"), arg.data_type(), true).into()
+ })
+ .collect::<Vec<_>>();
+
+ group.bench_function(
+ format!("rpad utf8 no fill [size={size}, str_len=5, target=20]"),
+ |b| {
+ b.iter(|| {
+ let args_cloned = args.clone();
+ black_box(unicode::rpad().invoke_with_args(ScalarFunctionArgs {
+ args: args_cloned,
+ arg_fields: arg_fields.clone(),
+ number_rows: size,
+ return_field: Field::new("f", DataType::Utf8, true).into(),
+ config_options: Arc::clone(&config_options),
+ }))
+ })
+ },
+ );
+
+ // Utf8 type with Unicode strings, without fill
+ let mut args = create_unicode_pad_args(size, 20, false);
+ args.truncate(2);
+ let arg_fields = args
+ .iter()
+ .enumerate()
+ .map(|(idx, arg)| {
+ Field::new(format!("arg_{idx}"), arg.data_type(), true).into()
+ })
+ .collect::<Vec<_>>();
+
+ group.bench_function(
+ format!("rpad utf8 unicode no fill [size={size}, target=20]"),
+ |b| {
+ b.iter(|| {
+ let args_cloned = args.clone();
+ black_box(unicode::rpad().invoke_with_args(ScalarFunctionArgs {
+ args: args_cloned,
+ arg_fields: arg_fields.clone(),
+ number_rows: size,
+ return_field: Field::new("f", DataType::Utf8, true).into(),
+ config_options: Arc::clone(&config_options),
+ }))
+ })
+ },
+ );
+
// --- Scalar length + fill benchmarks ---
// Utf8 with scalar length and fill (3-arg)
diff --git a/datafusion/functions/src/unicode/lpad.rs b/datafusion/functions/src/unicode/lpad.rs
index 0ffd027149..7368a1f90c 100644
--- a/datafusion/functions/src/unicode/lpad.rs
+++ b/datafusion/functions/src/unicode/lpad.rs
@@ -15,23 +15,13 @@
// specific language governing permissions and limitations
// under the License.
-use std::fmt::Write;
-use std::sync::Arc;
-
-use DataType::{LargeUtf8, Utf8, Utf8View};
-use arrow::array::{
- Array, ArrayRef, AsArray, GenericStringArray, GenericStringBuilder, Int64Array,
- OffsetSizeTrait, StringArrayType, StringViewArray,
-};
use arrow::datatypes::DataType;
-use crate::utils::{make_scalar_function, utf8_to_str_type};
-use datafusion_common::cast::as_int64_array;
-use datafusion_common::{Result, exec_err};
-use datafusion_expr::TypeSignature::Exact;
+use super::pad::{LeftPad, invoke_pad, pad_signature};
+use crate::utils::utf8_to_str_type;
+use datafusion_common::Result;
use datafusion_expr::{
ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
- Volatility,
};
use datafusion_macros::user_doc;
@@ -71,25 +61,8 @@ impl Default for LPadFunc {
impl LPadFunc {
pub fn new() -> Self {
- use DataType::*;
Self {
- signature: Signature::one_of(
- vec![
- Exact(vec![Utf8View, Int64]),
- Exact(vec![Utf8View, Int64, Utf8View]),
- Exact(vec![Utf8View, Int64, Utf8]),
- Exact(vec![Utf8View, Int64, LargeUtf8]),
- Exact(vec![Utf8, Int64]),
- Exact(vec![Utf8, Int64, Utf8View]),
- Exact(vec![Utf8, Int64, Utf8]),
- Exact(vec![Utf8, Int64, LargeUtf8]),
- Exact(vec![LargeUtf8, Int64]),
- Exact(vec![LargeUtf8, Int64, Utf8View]),
- Exact(vec![LargeUtf8, Int64, Utf8]),
- Exact(vec![LargeUtf8, Int64, LargeUtf8]),
- ],
- Volatility::Immutable,
- ),
+ signature: pad_signature(),
}
}
}
@@ -108,68 +81,7 @@ impl ScalarUDFImpl for LPadFunc {
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
- let ScalarFunctionArgs {
- args, number_rows, ..
- } = args;
-
- const MAX_SCALAR_TARGET_LEN: usize = 16384;
-
- // If target_len and fill (if specified) are constants, use the scalar
- // fast path.
- if let Some(target_len) = try_as_scalar_i64(&args[1]) {
- let target_len: usize = match usize::try_from(target_len) {
- Ok(n) if i32::try_from(n).is_ok() => n,
- Ok(n) => {
- return exec_err!(
- "lpad requested length {n} too large, maximum allowed length is {}",
- i32::MAX
- );
- }
- Err(_) => 0, // negative → 0
- };
-
- let fill_str = if args.len() == 3 {
- try_as_scalar_str(&args[2])
- } else {
- Some(" ")
- };
-
- // Skip the fast path for very large `target_len` values to avoid
- // consuming too much memory. Such large padding values are uncommon
- // in practice.
- if target_len <= MAX_SCALAR_TARGET_LEN
- && let Some(fill) = fill_str
- {
- let string_array = args[0].to_array_of_size(number_rows)?;
- let result = match string_array.data_type() {
- Utf8View => lpad_scalar_args::<_, i32>(
- string_array.as_string_view(),
- target_len,
- fill,
- ),
- Utf8 => lpad_scalar_args::<_, i32>(
- string_array.as_string::<i32>(),
- target_len,
- fill,
- ),
- LargeUtf8 => lpad_scalar_args::<_, i64>(
- string_array.as_string::<i64>(),
- target_len,
- fill,
- ),
- other => {
- exec_err!("Unsupported data type {other:?} for function lpad")
- }
- }?;
- return Ok(ColumnarValue::Array(result));
- }
- }
-
- match args[0].data_type() {
- Utf8 | Utf8View => make_scalar_function(lpad::<i32>, vec![])(&args),
- LargeUtf8 => make_scalar_function(lpad::<i64>, vec![])(&args),
- other => exec_err!("Unsupported data type {other:?} for function lpad"),
- }
+ invoke_pad::<LeftPad>(args)
}
fn documentation(&self) -> Option<&Documentation> {
@@ -177,341 +89,6 @@ impl ScalarUDFImpl for LPadFunc {
}
}
-use super::common::{
- StringCharLen, char_count_or_boundary, pad_data_capacity, try_as_scalar_i64,
- try_as_scalar_str,
-};
-
-/// Optimized lpad for constant target_len and fill arguments.
-fn lpad_scalar_args<'a, V: StringArrayType<'a> + Copy, T: OffsetSizeTrait>(
- string_array: V,
- target_len: usize,
- fill: &str,
-) -> Result<ArrayRef> {
- if string_array.is_ascii() && fill.is_ascii() {
- lpad_scalar_ascii::<V, T>(string_array, target_len, fill)
- } else {
- lpad_scalar_unicode::<V, T>(string_array, target_len, fill)
- }
-}
-
-fn lpad_scalar_ascii<'a, V: StringArrayType<'a> + Copy, T: OffsetSizeTrait>(
- string_array: V,
- target_len: usize,
- fill: &str,
-) -> Result<ArrayRef> {
- // With a scalar `target_len` and `fill`, we can precompute a padding
- // buffer of `target_len` fill characters repeated cyclically.
- let padding_buf = if !fill.is_empty() {
- let mut buf = String::with_capacity(target_len);
- while buf.len() < target_len {
- let remaining = target_len - buf.len();
- if remaining >= fill.len() {
- buf.push_str(fill);
- } else {
- buf.push_str(&fill[..remaining]);
- }
- }
- buf
- } else {
- String::new()
- };
-
- // Each output row is exactly `target_len` ASCII bytes (padding + string).
- let data_capacity = string_array.len().saturating_mul(target_len);
- let mut builder =
- GenericStringBuilder::<T>::with_capacity(string_array.len(), data_capacity);
-
- for maybe_string in string_array.iter() {
- match maybe_string {
- Some(string) => {
- let str_len = string.len();
- if target_len <= str_len {
- builder.append_value(&string[..target_len]);
- } else if fill.is_empty() {
- builder.append_value(string);
- } else {
- let pad_needed = target_len - str_len;
- builder.write_str(&padding_buf[..pad_needed])?;
- builder.append_value(string);
- }
- }
- None => builder.append_null(),
- }
- }
-
- Ok(Arc::new(builder.finish()) as ArrayRef)
-}
-
-fn lpad_scalar_unicode<'a, V: StringArrayType<'a> + Copy, T: OffsetSizeTrait>(
- string_array: V,
- target_len: usize,
- fill: &str,
-) -> Result<ArrayRef> {
- let fill_chars: Vec<char> = fill.chars().collect();
-
- // With a scalar `target_len` and `fill`, we can precompute a padding buffer
- // of `target_len` fill characters repeated cyclically. Because Unicode
- // characters are variable-width, we build a byte-offset table to map from
- // character count to the corresponding byte position in the padding buffer.
- let (padding_buf, char_byte_offsets) = if !fill_chars.is_empty() {
- let mut buf = String::new();
- let mut offsets = Vec::with_capacity(target_len + 1);
- offsets.push(0usize);
- for i in 0..target_len {
- buf.push(fill_chars[i % fill_chars.len()]);
- offsets.push(buf.len());
- }
- (buf, offsets)
- } else {
- (String::new(), vec![0])
- };
-
- // Each output row is `target_len` chars; multiply by 4 (max UTF-8 bytes
- // per char) for an upper bound in bytes.
- let data_capacity = string_array.len().saturating_mul(target_len * 4);
- let mut builder =
- GenericStringBuilder::<T>::with_capacity(string_array.len(), data_capacity);
-
- for maybe_string in string_array.iter() {
- match maybe_string {
- Some(string) => match char_count_or_boundary(string, target_len) {
- StringCharLen::ByteOffset(offset) => {
- builder.append_value(&string[..offset]);
- }
- StringCharLen::CharCount(char_count) => {
- if !fill_chars.is_empty() {
- let pad_chars = target_len - char_count;
- let pad_bytes = char_byte_offsets[pad_chars];
- builder.write_str(&padding_buf[..pad_bytes])?;
- }
- builder.append_value(string);
- }
- },
- None => builder.append_null(),
- }
- }
-
- Ok(Arc::new(builder.finish()) as ArrayRef)
-}
-
-/// Left-pads `string` to `target_len` using the fill string (default: space).
-/// Truncates from the right if `string` is already longer than `target_len`.
-/// lpad('hi', 5, 'xy') = 'xyxhi'
-fn lpad<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
- if args.len() <= 1 || args.len() > 3 {
- return exec_err!(
- "lpad was called with {} arguments. It requires at least 2 and at most 3.",
- args.len()
- );
- }
-
- let length_array = as_int64_array(&args[1])?;
-
- match (args.len(), args[0].data_type()) {
- (2, Utf8View) => lpad_impl::<&StringViewArray, &GenericStringArray<i32>, T>(
- &args[0].as_string_view(),
- length_array,
- None,
- ),
- (2, Utf8 | LargeUtf8) => lpad_impl::<
- &GenericStringArray<T>,
- &GenericStringArray<T>,
- T,
- >(&args[0].as_string::<T>(), length_array, None),
- (3, Utf8View) => lpad_with_replace::<&StringViewArray, T>(
- &args[0].as_string_view(),
- length_array,
- &args[2],
- ),
- (3, Utf8 | LargeUtf8) => lpad_with_replace::<&GenericStringArray<T>, T>(
- &args[0].as_string::<T>(),
- length_array,
- &args[2],
- ),
- (len, dt) => unreachable!("lpad: unexpected arg count ({len}) or type ({dt})"),
- }
-}
-
-fn lpad_with_replace<'a, V, T: OffsetSizeTrait>(
- string_array: &V,
- length_array: &Int64Array,
- fill_array: &'a ArrayRef,
-) -> Result<ArrayRef>
-where
- V: StringArrayType<'a>,
-{
- match fill_array.data_type() {
- Utf8View => lpad_impl::<V, &StringViewArray, T>(
- string_array,
- length_array,
- Some(fill_array.as_string_view()),
- ),
- LargeUtf8 => lpad_impl::<V, &GenericStringArray<i64>, T>(
- string_array,
- length_array,
- Some(fill_array.as_string::<i64>()),
- ),
- Utf8 => lpad_impl::<V, &GenericStringArray<i32>, T>(
- string_array,
- length_array,
- Some(fill_array.as_string::<i32>()),
- ),
- other => {
- exec_err!("Unsupported data type {other:?} for function lpad")
- }
- }
-}
-
-fn lpad_impl<'a, V, V2, T>(
- string_array: &V,
- length_array: &Int64Array,
- fill_array: Option<V2>,
-) -> Result<ArrayRef>
-where
- V: StringArrayType<'a>,
- V2: StringArrayType<'a>,
- T: OffsetSizeTrait,
-{
- let array = if let Some(fill_array) = fill_array {
- let mut builder: GenericStringBuilder<T> = GenericStringBuilder::with_capacity(
- string_array.len(),
- pad_data_capacity(length_array),
- );
- let mut fill_chars_buf = Vec::new();
-
- for ((string, target_len), fill) in string_array
- .iter()
- .zip(length_array.iter())
- .zip(fill_array.iter())
- {
- if let (Some(string), Some(target_len), Some(fill)) =
- (string, target_len, fill)
- {
- if target_len > i32::MAX as i64 {
- return exec_err!(
- "lpad requested length {target_len} too large, maximum allowed length is {}",
- i32::MAX
- );
- }
-
- let target_len = if target_len < 0 {
- 0
- } else {
- target_len as usize
- };
- if target_len == 0 {
- builder.append_value("");
- continue;
- }
-
- if string.is_ascii() && fill.is_ascii() {
- // ASCII fast path: byte length == character length.
- let str_len = string.len();
- if target_len < str_len {
- builder.append_value(&string[..target_len]);
- } else if fill.is_empty() {
- builder.append_value(string);
- } else {
- let pad_len = target_len - str_len;
- let fill_len = fill.len();
- let full_reps = pad_len / fill_len;
- let remainder = pad_len % fill_len;
- for _ in 0..full_reps {
- builder.write_str(fill)?;
- }
- if remainder > 0 {
- builder.write_str(&fill[..remainder])?;
- }
- builder.append_value(string);
- }
- } else {
- fill_chars_buf.clear();
- fill_chars_buf.extend(fill.chars());
-
- match char_count_or_boundary(string, target_len) {
- StringCharLen::ByteOffset(offset) => {
- builder.append_value(&string[..offset]);
- }
- StringCharLen::CharCount(char_count) => {
- if !fill_chars_buf.is_empty() {
- for l in 0..target_len - char_count {
- let c = *fill_chars_buf
- .get(l % fill_chars_buf.len())
- .unwrap();
- builder.write_char(c)?;
- }
- }
- builder.append_value(string);
- }
- }
- }
- } else {
- builder.append_null();
- }
- }
-
- builder.finish()
- } else {
- let mut builder: GenericStringBuilder<T> = GenericStringBuilder::with_capacity(
- string_array.len(),
- pad_data_capacity(length_array),
- );
-
- for (string, target_len) in string_array.iter().zip(length_array.iter()) {
- if let (Some(string), Some(target_len)) = (string, target_len) {
- if target_len > i32::MAX as i64 {
- return exec_err!(
- "lpad requested length {target_len} too large, maximum allowed length is {}",
- i32::MAX
- );
- }
-
- let target_len = if target_len < 0 {
- 0
- } else {
- target_len as usize
- };
- if target_len == 0 {
- builder.append_value("");
- continue;
- }
-
- if string.is_ascii() {
- // ASCII fast path: byte length == character length
- let str_len = string.len();
- if target_len < str_len {
- builder.append_value(&string[..target_len]);
- } else {
- for _ in 0..(target_len - str_len) {
- builder.write_str(" ")?;
- }
- builder.append_value(string);
- }
- } else {
- match char_count_or_boundary(string, target_len) {
- StringCharLen::ByteOffset(offset) => {
- builder.append_value(&string[..offset]);
- }
- StringCharLen::CharCount(char_count) => {
- for _ in 0..(target_len - char_count) {
- builder.write_str(" ")?;
- }
- builder.append_value(string);
- }
- }
- }
- } else {
- builder.append_null();
- }
- }
-
- builder.finish()
- };
-
- Ok(Arc::new(array) as ArrayRef)
-}
-
#[cfg(test)]
mod tests {
use crate::unicode::lpad::LPadFunc;
diff --git a/datafusion/functions/src/unicode/mod.rs b/datafusion/functions/src/unicode/mod.rs
index 7250b3915f..84ac658753 100644
--- a/datafusion/functions/src/unicode/mod.rs
+++ b/datafusion/functions/src/unicode/mod.rs
@@ -27,6 +27,7 @@ pub mod find_in_set;
pub mod initcap;
pub mod left;
pub mod lpad;
+mod pad;
pub mod planner;
pub mod reverse;
pub mod right;
diff --git a/datafusion/functions/src/unicode/pad.rs b/datafusion/functions/src/unicode/pad.rs
new file mode 100644
index 0000000000..2e8ad22725
--- /dev/null
+++ b/datafusion/functions/src/unicode/pad.rs
@@ -0,0 +1,460 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Shared implementation of the `lpad` and `rpad` functions
+
+use std::fmt::Write;
+use std::iter;
+use std::sync::Arc;
+
+use DataType::{LargeUtf8, Utf8, Utf8View};
+use arrow::array::{
+ Array, ArrayRef, AsArray, GenericStringBuilder, Int64Array, OffsetSizeTrait,
+ StringArrayType,
+};
+use arrow::datatypes::DataType;
+use datafusion_common::cast::as_int64_array;
+use datafusion_common::{Result, exec_err};
+use datafusion_expr::TypeSignature::Exact;
+use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, Signature, Volatility};
+
+use super::common::{
+ StringCharLen, byte_offset_of_char, char_count_or_boundary, pad_data_capacity,
+ try_as_scalar_i64, try_as_scalar_str,
+};
+use crate::utils::make_scalar_function;
+
+/// The side of the string that padding is written on
+pub(crate) trait PadSide {
+ /// Name of the SQL function, used in error messages
+ const NAME: &'static str;
+ /// `true` to pad on the left (`lpad`), `false` to pad on the right (`rpad`)
+ const LEFT: bool;
+}
+
+pub(crate) struct LeftPad {}
+
+impl PadSide for LeftPad {
+ const NAME: &'static str = "lpad";
+ const LEFT: bool = true;
+}
+
+pub(crate) struct RightPad {}
+
+impl PadSide for RightPad {
+ const NAME: &'static str = "rpad";
+ const LEFT: bool = false;
+}
+
+/// Signature shared by `lpad` and `rpad`
+pub(crate) fn pad_signature() -> Signature {
+ use DataType::*;
+ Signature::one_of(
+ vec![
+ Exact(vec![Utf8View, Int64]),
+ Exact(vec![Utf8View, Int64, Utf8View]),
+ Exact(vec![Utf8View, Int64, Utf8]),
+ Exact(vec![Utf8View, Int64, LargeUtf8]),
+ Exact(vec![Utf8, Int64]),
+ Exact(vec![Utf8, Int64, Utf8View]),
+ Exact(vec![Utf8, Int64, Utf8]),
+ Exact(vec![Utf8, Int64, LargeUtf8]),
+ Exact(vec![LargeUtf8, Int64]),
+ Exact(vec![LargeUtf8, Int64, Utf8View]),
+ Exact(vec![LargeUtf8, Int64, Utf8]),
+ Exact(vec![LargeUtf8, Int64, LargeUtf8]),
+ ],
+ Volatility::Immutable,
+ )
+}
+
+/// `invoke_with_args` for `lpad` and `rpad`
+pub(crate) fn invoke_pad<S: PadSide>(args: ScalarFunctionArgs) -> Result<ColumnarValue> {
+ let ScalarFunctionArgs {
+ args, number_rows, ..
+ } = args;
+
+ const MAX_SCALAR_TARGET_LEN: usize = 16384;
+
+ // If target_len and fill (if specified) are constants, use the scalar
+ // fast path.
+ if let Some(target_len) = try_as_scalar_i64(&args[1]) {
+ let target_len = parse_target_len(target_len, S::NAME)?;
+
+ let fill_str = if args.len() == 3 {
+ try_as_scalar_str(&args[2])
+ } else {
+ Some(" ")
+ };
+
+ // Skip the fast path for very large `target_len` values to avoid
+ // consuming too much memory. Such large padding values are uncommon
+ // in practice.
+ if target_len <= MAX_SCALAR_TARGET_LEN
+ && let Some(fill) = fill_str
+ {
+ let string_array = args[0].to_array_of_size(number_rows)?;
+ let result = match string_array.data_type() {
+ Utf8View => pad_scalar_args::<S, _, i32>(
+ string_array.as_string_view(),
+ target_len,
+ fill,
+ ),
+ Utf8 => pad_scalar_args::<S, _, i32>(
+ string_array.as_string::<i32>(),
+ target_len,
+ fill,
+ ),
+ LargeUtf8 => pad_scalar_args::<S, _, i64>(
+ string_array.as_string::<i64>(),
+ target_len,
+ fill,
+ ),
+ other => {
+ exec_err!("Unsupported data type {other:?} for function {}", S::NAME)
+ }
+ }?;
+ return Ok(ColumnarValue::Array(result));
+ }
+ }
+
+ match args[0].data_type() {
+ Utf8 | Utf8View => make_scalar_function(pad::<S, i32>, vec![])(&args),
+ LargeUtf8 => make_scalar_function(pad::<S, i64>, vec![])(&args),
+ other => exec_err!("Unsupported data type {other:?} for function {}", S::NAME),
+ }
+}
+
+/// Validates a requested pad length. Lengths above `i32::MAX` are an error and
+/// negative lengths are treated as 0.
+#[inline(always)]
+fn parse_target_len(target_len: i64, fn_name: &str) -> Result<usize> {
+ if target_len > i32::MAX as i64 {
+ return target_len_too_large(target_len, fn_name);
+ }
+ Ok(target_len.max(0) as usize)
+}
+
+/// Kept out of line so the per-row length check stays small.
+#[cold]
+#[inline(never)]
+fn target_len_too_large(target_len: i64, fn_name: &str) -> Result<usize> {
+ exec_err!(
+ "{fn_name} requested length {target_len} too large, maximum allowed length is {}",
+ i32::MAX
+ )
+}
+
+/// Appends one output row made of `string` and the padding written by
+/// `write_padding`, placing the padding on the side given by `S`.
+#[inline(always)]
+fn append_padded<S: PadSide, T: OffsetSizeTrait>(
+ builder: &mut GenericStringBuilder<T>,
+ string: &str,
+ write_padding: impl FnOnce(&mut GenericStringBuilder<T>) -> std::fmt::Result,
+) -> Result<()> {
+ if S::LEFT {
+ write_padding(builder)?;
+ builder.append_value(string);
+ } else {
+ builder.write_str(string)?;
+ write_padding(builder)?;
+ builder.append_value("");
+ }
+ Ok(())
+}
+
+/// Writes `pad_chars` characters of `fill`, repeated cyclically. Whole copies
+/// of `fill` are written at once rather than one character at a time.
+#[inline(always)]
+fn write_fill_chars(
+ w: &mut impl Write,
+ fill: &str,
+ pad_chars: usize,
+) -> std::fmt::Result {
+ let fill_chars = if fill.is_ascii() {
+ fill.len()
+ } else {
+ fill.chars().count()
+ };
+ if fill_chars == 0 {
+ return Ok(());
+ }
+ let mut remaining = pad_chars;
+ while remaining >= fill_chars {
+ w.write_str(fill)?;
+ remaining -= fill_chars;
+ }
+ if remaining > 0 {
+ w.write_str(&fill[..byte_offset_of_char(fill, remaining)])?;
+ }
+ Ok(())
+}
+
+/// Optimized pad for constant target_len and fill arguments.
+fn pad_scalar_args<'a, S: PadSide, V: StringArrayType<'a> + Copy, T: OffsetSizeTrait>(
+ string_array: V,
+ target_len: usize,
+ fill: &str,
+) -> Result<ArrayRef> {
+ if string_array.is_ascii() && fill.is_ascii() {
+ pad_scalar_ascii::<S, V, T>(string_array, target_len, fill)
+ } else {
+ pad_scalar_unicode::<S, V, T>(string_array, target_len, fill)
+ }
+}
+
+fn pad_scalar_ascii<'a, S: PadSide, V: StringArrayType<'a> + Copy, T: OffsetSizeTrait>(
+ string_array: V,
+ target_len: usize,
+ fill: &str,
+) -> Result<ArrayRef> {
+ // With a scalar `target_len` and `fill`, we can precompute a padding
+ // buffer of `target_len` fill characters repeated cyclically.
+ let padding_buf = if !fill.is_empty() {
+ let mut buf = String::with_capacity(target_len);
+ while buf.len() < target_len {
+ let remaining = target_len - buf.len();
+ if remaining >= fill.len() {
+ buf.push_str(fill);
+ } else {
+ buf.push_str(&fill[..remaining]);
+ }
+ }
+ buf
+ } else {
+ String::new()
+ };
+
+ // Each output row is exactly `target_len` ASCII bytes (string + padding).
+ let data_capacity = string_array.len().saturating_mul(target_len);
+ let mut builder =
+ GenericStringBuilder::<T>::with_capacity(string_array.len(), data_capacity);
+
+ for maybe_string in string_array.iter() {
+ match maybe_string {
+ Some(string) => {
+ let str_len = string.len();
+ if target_len <= str_len {
+ builder.append_value(&string[..target_len]);
+ } else if fill.is_empty() {
+ builder.append_value(string);
+ } else {
+ let pad_needed = target_len - str_len;
+ append_padded::<S, T>(&mut builder, string, |b| {
+ b.write_str(&padding_buf[..pad_needed])
+ })?;
+ }
+ }
+ None => builder.append_null(),
+ }
+ }
+
+ Ok(Arc::new(builder.finish()) as ArrayRef)
+}
+
+fn pad_scalar_unicode<
+ 'a,
+ S: PadSide,
+ V: StringArrayType<'a> + Copy,
+ T: OffsetSizeTrait,
+>(
+ string_array: V,
+ target_len: usize,
+ fill: &str,
+) -> Result<ArrayRef> {
+ let fill_chars: Vec<char> = fill.chars().collect();
+
+ // With a scalar `target_len` and `fill`, we can precompute a padding buffer
+ // of `target_len` fill characters repeated cyclically. Because Unicode
+ // characters are variable-width, we build a byte-offset table to map from
+ // character count to the corresponding byte position in the padding buffer.
+ let (padding_buf, char_byte_offsets) = if !fill_chars.is_empty() {
+ let mut buf = String::new();
+ let mut offsets = Vec::with_capacity(target_len + 1);
+ offsets.push(0usize);
+ for i in 0..target_len {
+ buf.push(fill_chars[i % fill_chars.len()]);
+ offsets.push(buf.len());
+ }
+ (buf, offsets)
+ } else {
+ (String::new(), vec![0])
+ };
+
+ // Each output row is `target_len` chars; multiply by 4 (max UTF-8 bytes
+ // per char) for an upper bound in bytes.
+ let data_capacity = string_array.len().saturating_mul(target_len * 4);
+ let mut builder =
+ GenericStringBuilder::<T>::with_capacity(string_array.len(), data_capacity);
+
+ for maybe_string in string_array.iter() {
+ match maybe_string {
+ Some(string) => match char_count_or_boundary(string, target_len) {
+ StringCharLen::ByteOffset(offset) => {
+ builder.append_value(&string[..offset]);
+ }
+ StringCharLen::CharCount(char_count) => {
+ append_padded::<S, T>(&mut builder, string, |b| {
+ if fill_chars.is_empty() {
+ return Ok(());
+ }
+ let pad_chars = target_len - char_count;
+ let pad_bytes = char_byte_offsets[pad_chars];
+ b.write_str(&padding_buf[..pad_bytes])
+ })?;
+ }
+ },
+ None => builder.append_null(),
+ }
+ }
+
+ Ok(Arc::new(builder.finish()) as ArrayRef)
+}
+
+/// Pads `string` to `target_len` using the fill string (default: space), on
+/// the side given by `S`. Truncates from the right if `string` is already
+/// longer than `target_len`.
+/// lpad('hi', 5, 'xy') = 'xyxhi'
+/// rpad('hi', 5, 'xy') = 'hixyx'
+fn pad<S: PadSide, T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
+ if args.len() <= 1 || args.len() > 3 {
+ return exec_err!(
+ "{} was called with {} arguments. It requires at least 2 and at most 3.",
+ S::NAME,
+ args.len()
+ );
+ }
+
+ let length_array = as_int64_array(&args[1])?;
+
+ // Without a fill argument, pad with spaces. `repeat_with` keeps the fill a
+ // compile-time constant in `pad_impl`, so this case is as fast as a
+ // dedicated space-padding loop.
+ let spaces = || iter::repeat_with(|| Some(" "));
+
+ match (args.len(), args[0].data_type()) {
+ (2, Utf8View) => {
+ pad_impl::<S, _, T>(&args[0].as_string_view(), length_array, spaces())
+ }
+ (2, Utf8 | LargeUtf8) => {
+ pad_impl::<S, _, T>(&args[0].as_string::<T>(), length_array, spaces())
+ }
+ (3, Utf8View) => {
+ pad_with_fill::<S, _, T>(&args[0].as_string_view(), length_array, &args[2])
+ }
+ (3, Utf8 | LargeUtf8) => {
+ pad_with_fill::<S, _, T>(&args[0].as_string::<T>(), length_array, &args[2])
+ }
+ (len, dt) => {
+ unreachable!("{}: unexpected arg count ({len}) or type ({dt})", S::NAME)
+ }
+ }
+}
+
+fn pad_with_fill<'a, S: PadSide, V: StringArrayType<'a>, T: OffsetSizeTrait>(
+ string_array: &V,
+ length_array: &Int64Array,
+ fill_array: &'a ArrayRef,
+) -> Result<ArrayRef> {
+ match fill_array.data_type() {
+ Utf8View => pad_impl::<S, V, T>(
+ string_array,
+ length_array,
+ fill_array.as_string_view().iter(),
+ ),
+ LargeUtf8 => pad_impl::<S, V, T>(
+ string_array,
+ length_array,
+ fill_array.as_string::<i64>().iter(),
+ ),
+ Utf8 => pad_impl::<S, V, T>(
+ string_array,
+ length_array,
+ fill_array.as_string::<i32>().iter(),
+ ),
+ other => {
+ exec_err!("Unsupported data type {other:?} for function {}", S::NAME)
+ }
+ }
+}
+
+fn pad_impl<'a, S, V, T>(
+ string_array: &V,
+ length_array: &Int64Array,
+ fills: impl Iterator<Item = Option<&'a str>>,
+) -> Result<ArrayRef>
+where
+ S: PadSide,
+ V: StringArrayType<'a>,
+ T: OffsetSizeTrait,
+{
+ let mut builder: GenericStringBuilder<T> = GenericStringBuilder::with_capacity(
+ string_array.len(),
+ pad_data_capacity(length_array),
+ );
+ for ((string, target_len), fill) in
+ string_array.iter().zip(length_array.iter()).zip(fills)
+ {
+ let (Some(string), Some(target_len), Some(fill)) = (string, target_len, fill)
+ else {
+ builder.append_null();
+ continue;
+ };
+
+ let target_len = parse_target_len(target_len, S::NAME)?;
+ if target_len == 0 {
+ builder.append_value("");
+ continue;
+ }
+
+ if string.is_ascii() && fill.is_ascii() {
+ // ASCII fast path: byte length == character length.
+ let str_len = string.len();
+ if target_len < str_len {
+ builder.append_value(&string[..target_len]);
+ } else if fill.is_empty() {
+ builder.append_value(string);
+ } else {
+ append_padded::<S, T>(&mut builder, string, |b| {
+ let mut remaining = target_len - str_len;
+ while remaining >= fill.len() {
+ b.write_str(fill)?;
+ remaining -= fill.len();
+ }
+ if remaining > 0 {
+ b.write_str(&fill[..remaining])?;
+ }
+ Ok(())
+ })?;
+ }
+ } else {
+ match char_count_or_boundary(string, target_len) {
+ StringCharLen::ByteOffset(offset) => {
+ builder.append_value(&string[..offset]);
+ }
+ StringCharLen::CharCount(char_count) => {
+ append_padded::<S, T>(&mut builder, string, |b| {
+ write_fill_chars(b, fill, target_len - char_count)
+ })?;
+ }
+ }
+ }
+ }
+
+ Ok(Arc::new(builder.finish()) as ArrayRef)
+}
diff --git a/datafusion/functions/src/unicode/rpad.rs b/datafusion/functions/src/unicode/rpad.rs
index d89ebd0057..f882661ae3 100644
--- a/datafusion/functions/src/unicode/rpad.rs
+++ b/datafusion/functions/src/unicode/rpad.rs
@@ -15,23 +15,13 @@
// specific language governing permissions and limitations
// under the License.
-use std::fmt::Write;
-use std::sync::Arc;
-
-use DataType::{LargeUtf8, Utf8, Utf8View};
-use arrow::array::{
- ArrayRef, AsArray, GenericStringArray, GenericStringBuilder, Int64Array,
- OffsetSizeTrait, StringArrayType, StringViewArray,
-};
use arrow::datatypes::DataType;
-use crate::utils::{make_scalar_function, utf8_to_str_type};
-use datafusion_common::cast::as_int64_array;
-use datafusion_common::{Result, exec_err};
-use datafusion_expr::TypeSignature::Exact;
+use super::pad::{RightPad, invoke_pad, pad_signature};
+use crate::utils::utf8_to_str_type;
+use datafusion_common::Result;
use datafusion_expr::{
ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
- Volatility,
};
use datafusion_macros::user_doc;
@@ -71,25 +61,8 @@ impl Default for RPadFunc {
impl RPadFunc {
pub fn new() -> Self {
- use DataType::*;
Self {
- signature: Signature::one_of(
- vec![
- Exact(vec![Utf8View, Int64]),
- Exact(vec![Utf8View, Int64, Utf8View]),
- Exact(vec![Utf8View, Int64, Utf8]),
- Exact(vec![Utf8View, Int64, LargeUtf8]),
- Exact(vec![Utf8, Int64]),
- Exact(vec![Utf8, Int64, Utf8View]),
- Exact(vec![Utf8, Int64, Utf8]),
- Exact(vec![Utf8, Int64, LargeUtf8]),
- Exact(vec![LargeUtf8, Int64]),
- Exact(vec![LargeUtf8, Int64, Utf8View]),
- Exact(vec![LargeUtf8, Int64, Utf8]),
- Exact(vec![LargeUtf8, Int64, LargeUtf8]),
- ],
- Volatility::Immutable,
- ),
+ signature: pad_signature(),
}
}
}
@@ -108,68 +81,7 @@ impl ScalarUDFImpl for RPadFunc {
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
- let ScalarFunctionArgs {
- args, number_rows, ..
- } = args;
-
- const MAX_SCALAR_TARGET_LEN: usize = 16384;
-
- // If target_len and fill (if specified) are constants, use the
- // scalar fast path.
- if let Some(target_len) = try_as_scalar_i64(&args[1]) {
- let target_len: usize = match usize::try_from(target_len) {
- Ok(n) if i32::try_from(n).is_ok() => n,
- Ok(n) => {
- return exec_err!(
- "rpad requested length {n} too large, maximum allowed length is {}",
- i32::MAX
- );
- }
- Err(_) => 0, // negative → 0
- };
-
- let fill_str = if args.len() == 3 {
- try_as_scalar_str(&args[2])
- } else {
- Some(" ")
- };
-
- // Skip the fast path for very large `target_len` values to avoid
- // consuming too much memory. Such large padding values are uncommon
- // in practice.
- if target_len <= MAX_SCALAR_TARGET_LEN
- && let Some(fill) = fill_str
- {
- let string_array = args[0].to_array_of_size(number_rows)?;
- let result = match string_array.data_type() {
- Utf8View => rpad_scalar_args::<_, i32>(
- string_array.as_string_view(),
- target_len,
- fill,
- ),
- Utf8 => rpad_scalar_args::<_, i32>(
- string_array.as_string::<i32>(),
- target_len,
- fill,
- ),
- LargeUtf8 => rpad_scalar_args::<_, i64>(
- string_array.as_string::<i64>(),
- target_len,
- fill,
- ),
- other => {
- exec_err!("Unsupported data type {other:?} for function rpad")
- }
- }?;
- return Ok(ColumnarValue::Array(result));
- }
- }
-
- match args[0].data_type() {
- Utf8 | Utf8View => make_scalar_function(rpad::<i32>, vec![])(&args),
- LargeUtf8 => make_scalar_function(rpad::<i64>, vec![])(&args),
- other => exec_err!("Unsupported data type {other:?} for function rpad"),
- }
+ invoke_pad::<RightPad>(args)
}
fn documentation(&self) -> Option<&Documentation> {
@@ -177,344 +89,6 @@ impl ScalarUDFImpl for RPadFunc {
}
}
-use super::common::{
- StringCharLen, char_count_or_boundary, pad_data_capacity, try_as_scalar_i64,
- try_as_scalar_str,
-};
-
-/// Optimized rpad for constant target_len and fill arguments.
-fn rpad_scalar_args<'a, V: StringArrayType<'a> + Copy, T: OffsetSizeTrait>(
- string_array: V,
- target_len: usize,
- fill: &str,
-) -> Result<ArrayRef> {
- if string_array.is_ascii() && fill.is_ascii() {
- rpad_scalar_ascii::<V, T>(string_array, target_len, fill)
- } else {
- rpad_scalar_unicode::<V, T>(string_array, target_len, fill)
- }
-}
-
-fn rpad_scalar_ascii<'a, V: StringArrayType<'a> + Copy, T: OffsetSizeTrait>(
- string_array: V,
- target_len: usize,
- fill: &str,
-) -> Result<ArrayRef> {
- // With a scalar `target_len` and `fill`, we can precompute a padding
- // buffer of `target_len` fill characters repeated cyclically.
- let padding_buf = if !fill.is_empty() {
- let mut buf = String::with_capacity(target_len);
- while buf.len() < target_len {
- let remaining = target_len - buf.len();
- if remaining >= fill.len() {
- buf.push_str(fill);
- } else {
- buf.push_str(&fill[..remaining]);
- }
- }
- buf
- } else {
- String::new()
- };
-
- // Each output row is exactly `target_len` ASCII bytes (string + padding).
- let data_capacity = string_array.len().saturating_mul(target_len);
- let mut builder =
- GenericStringBuilder::<T>::with_capacity(string_array.len(), data_capacity);
-
- for maybe_string in string_array.iter() {
- match maybe_string {
- Some(string) => {
- let str_len = string.len();
- if target_len <= str_len {
- builder.append_value(&string[..target_len]);
- } else if fill.is_empty() {
- builder.append_value(string);
- } else {
- let pad_needed = target_len - str_len;
- builder.write_str(string)?;
- builder.write_str(&padding_buf[..pad_needed])?;
- builder.append_value("");
- }
- }
- None => builder.append_null(),
- }
- }
-
- Ok(Arc::new(builder.finish()) as ArrayRef)
-}
-
-fn rpad_scalar_unicode<'a, V: StringArrayType<'a> + Copy, T: OffsetSizeTrait>(
- string_array: V,
- target_len: usize,
- fill: &str,
-) -> Result<ArrayRef> {
- let fill_chars: Vec<char> = fill.chars().collect();
-
- // With a scalar `target_len` and `fill`, we can precompute a padding buffer
- // of `target_len` fill characters repeated cyclically. Because Unicode
- // characters are variable-width, we build a byte-offset table to map from
- // character count to the corresponding byte position in the padding buffer.
- let (padding_buf, char_byte_offsets) = if !fill_chars.is_empty() {
- let mut buf = String::new();
- let mut offsets = Vec::with_capacity(target_len + 1);
- offsets.push(0usize);
- for i in 0..target_len {
- buf.push(fill_chars[i % fill_chars.len()]);
- offsets.push(buf.len());
- }
- (buf, offsets)
- } else {
- (String::new(), vec![0])
- };
-
- // Each output row is `target_len` chars; multiply by 4 (max UTF-8 bytes
- // per char) for an upper bound in bytes.
- let data_capacity = string_array.len().saturating_mul(target_len * 4);
- let mut builder =
- GenericStringBuilder::<T>::with_capacity(string_array.len(), data_capacity);
-
- for maybe_string in string_array.iter() {
- match maybe_string {
- Some(string) => match char_count_or_boundary(string, target_len) {
- StringCharLen::ByteOffset(offset) => {
- builder.append_value(&string[..offset]);
- }
- StringCharLen::CharCount(char_count) => {
- builder.write_str(string)?;
- if !fill_chars.is_empty() {
- let pad_chars = target_len - char_count;
- let pad_bytes = char_byte_offsets[pad_chars];
- builder.write_str(&padding_buf[..pad_bytes])?;
- }
- builder.append_value("");
- }
- },
- None => builder.append_null(),
- }
- }
-
- Ok(Arc::new(builder.finish()) as ArrayRef)
-}
-
-fn rpad<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
- if args.len() <= 1 || args.len() > 3 {
- return exec_err!(
- "rpad was called with {} arguments. It requires at least 2 and at most 3.",
- args.len()
- );
- }
-
- let length_array = as_int64_array(&args[1])?;
-
- match (args.len(), args[0].data_type()) {
- (2, Utf8View) => rpad_impl::<&StringViewArray, &GenericStringArray<i32>, T>(
- &args[0].as_string_view(),
- length_array,
- None,
- ),
- (2, Utf8 | LargeUtf8) => rpad_impl::<
- &GenericStringArray<T>,
- &GenericStringArray<T>,
- T,
- >(&args[0].as_string::<T>(), length_array, None),
- (3, Utf8View) => rpad_with_replace::<&StringViewArray, T>(
- &args[0].as_string_view(),
- length_array,
- &args[2],
- ),
- (3, Utf8 | LargeUtf8) => rpad_with_replace::<&GenericStringArray<T>, T>(
- &args[0].as_string::<T>(),
- length_array,
- &args[2],
- ),
- (len, dt) => unreachable!("rpad: unexpected arg count ({len}) or type ({dt})"),
- }
-}
-
-fn rpad_with_replace<'a, V, T: OffsetSizeTrait>(
- string_array: &V,
- length_array: &Int64Array,
- fill_array: &'a ArrayRef,
-) -> Result<ArrayRef>
-where
- V: StringArrayType<'a>,
-{
- match fill_array.data_type() {
- Utf8View => rpad_impl::<V, &StringViewArray, T>(
- string_array,
- length_array,
- Some(fill_array.as_string_view()),
- ),
- LargeUtf8 => rpad_impl::<V, &GenericStringArray<i64>, T>(
- string_array,
- length_array,
- Some(fill_array.as_string::<i64>()),
- ),
- Utf8 => rpad_impl::<V, &GenericStringArray<i32>, T>(
- string_array,
- length_array,
- Some(fill_array.as_string::<i32>()),
- ),
- other => {
- exec_err!("Unsupported data type {other:?} for function rpad")
- }
- }
-}
-
-fn rpad_impl<'a, V, V2, T>(
- string_array: &V,
- length_array: &Int64Array,
- fill_array: Option<V2>,
-) -> Result<ArrayRef>
-where
- V: StringArrayType<'a>,
- V2: StringArrayType<'a>,
- T: OffsetSizeTrait,
-{
- let array = if let Some(fill_array) = fill_array {
- let mut builder: GenericStringBuilder<T> = GenericStringBuilder::with_capacity(
- string_array.len(),
- pad_data_capacity(length_array),
- );
- let mut fill_chars_buf = Vec::new();
-
- for ((string, target_len), fill) in string_array
- .iter()
- .zip(length_array.iter())
- .zip(fill_array.iter())
- {
- if let (Some(string), Some(target_len), Some(fill)) =
- (string, target_len, fill)
- {
- if target_len > i32::MAX as i64 {
- return exec_err!(
- "rpad requested length {target_len} too large, maximum allowed length is {}",
- i32::MAX
- );
- }
-
- let target_len = if target_len < 0 {
- 0
- } else {
- target_len as usize
- };
- if target_len == 0 {
- builder.append_value("");
- continue;
- }
-
- if string.is_ascii() && fill.is_ascii() {
- // ASCII fast path: byte length == character length.
- let str_len = string.len();
- if target_len < str_len {
- builder.append_value(&string[..target_len]);
- } else if fill.is_empty() {
- builder.append_value(string);
- } else {
- let pad_len = target_len - str_len;
- let fill_len = fill.len();
- let full_reps = pad_len / fill_len;
- let remainder = pad_len % fill_len;
- builder.write_str(string)?;
- for _ in 0..full_reps {
- builder.write_str(fill)?;
- }
- if remainder > 0 {
- builder.write_str(&fill[..remainder])?;
- }
- builder.append_value("");
- }
- } else {
- fill_chars_buf.clear();
- fill_chars_buf.extend(fill.chars());
-
- match char_count_or_boundary(string, target_len) {
- StringCharLen::ByteOffset(offset) => {
- builder.append_value(&string[..offset]);
- }
- StringCharLen::CharCount(char_count) => {
- builder.write_str(string)?;
- if !fill_chars_buf.is_empty() {
- for l in 0..target_len - char_count {
- let c = *fill_chars_buf
- .get(l % fill_chars_buf.len())
- .unwrap();
- builder.write_char(c)?;
- }
- }
- builder.append_value("");
- }
- }
- }
- } else {
- builder.append_null();
- }
- }
-
- builder.finish()
- } else {
- let mut builder: GenericStringBuilder<T> = GenericStringBuilder::with_capacity(
- string_array.len(),
- pad_data_capacity(length_array),
- );
-
- for (string, target_len) in string_array.iter().zip(length_array.iter()) {
- if let (Some(string), Some(target_len)) = (string, target_len) {
- if target_len > i32::MAX as i64 {
- return exec_err!(
- "rpad requested length {target_len} too large, maximum allowed length is {}",
- i32::MAX
- );
- }
-
- let target_len = if target_len < 0 {
- 0
- } else {
- target_len as usize
- };
- if target_len == 0 {
- builder.append_value("");
- continue;
- }
-
- if string.is_ascii() {
- // ASCII fast path: byte length == character length
- let str_len = string.len();
- if target_len < str_len {
- builder.append_value(&string[..target_len]);
- } else {
- builder.write_str(string)?;
- for _ in 0..(target_len - str_len) {
- builder.write_str(" ")?;
- }
- builder.append_value("");
- }
- } else {
- match char_count_or_boundary(string, target_len) {
- StringCharLen::ByteOffset(offset) => {
- builder.append_value(&string[..offset]);
- }
- StringCharLen::CharCount(char_count) => {
- builder.write_str(string)?;
- for _ in 0..(target_len - char_count) {
- builder.write_str(" ")?;
- }
- builder.append_value("");
- }
- }
- }
- } else {
- builder.append_null();
- }
- }
-
- builder.finish()
- };
-
- Ok(Arc::new(array) as ArrayRef)
-}
-
#[cfg(test)]
mod tests {
use arrow::array::{Array, StringArray};
diff --git a/datafusion/sqllogictest/test_files/string/string_literal.slt b/datafusion/sqllogictest/test_files/string/string_literal.slt
index 07aacaad93..93ce3cfaa3 100644
--- a/datafusion/sqllogictest/test_files/string/string_literal.slt
+++ b/datafusion/sqllogictest/test_files/string/string_literal.slt
@@ -377,6 +377,39 @@ true true true
true true true
true true true
+# lpad array path where only part of a multi-character fill fits
+query TT
+SELECT lpad(column1, column2, column3), lpad(arrow_cast(column1, 'Utf8View'), column2, arrow_cast(column3, 'Utf8View'))
+FROM (VALUES ('josé', 6, 'ñéx'), ('josé', 9, 'ñéx')) AS t(column1, column2, column3);
+----
+ñéjosé ñéjosé
+ñéxñéjosé ñéxñéjosé
+
+# lpad array path with a Unicode string, a zero length and a NULL length
+query T
+SELECT lpad(column1, column2) FROM (VALUES ('josé', 6), ('hi', 0), ('hello', 3), ('hi', NULL)) AS t(column1, column2);
+----
+ josé
+(empty)
+hel
+NULL
+
+# lpad array path with different string widths for the input and the fill
+query BBB
+SELECT
+ lpad(arrow_cast(column1, 'Utf8View'), column2, arrow_cast(column3, 'LargeUtf8')) IS NOT DISTINCT FROM column4,
+ lpad(arrow_cast(column1, 'LargeUtf8'), column2, arrow_cast(column3, 'Utf8View')) IS NOT DISTINCT FROM column4,
+ lpad(arrow_cast(column1, 'Utf8'), column2, arrow_cast(column3, 'Utf8View')) IS NOT DISTINCT FROM column4
+FROM (VALUES ('hi', 5, 'xy', 'xyxhi'), ('josé', 5, 'ñ', 'ñjosé'), (NULL, 4, 'ab', NULL)) AS t(column1, column2, column3, column4);
+----
+true true true
+true true true
+true true true
+
+# a length column above i32::MAX is an error on the array path
+query error lpad requested length 2147483648 too large
+SELECT lpad('hi', column1) FROM (VALUES (2147483648), (1)) AS t(column1);
+
# a large scalar target length skips the scalar fast path
query I
SELECT character_length(lpad('x', 16385, 'a'));
@@ -763,6 +796,40 @@ true true true
true true true
true true true
+# rpad array path where only part of a multi-character fill fits
+query TT
+SELECT rpad(column1, column2, column3), rpad(arrow_cast(column1, 'Utf8View'), column2, arrow_cast(column3, 'Utf8View'))
+FROM (VALUES ('josé', 6, 'ñéx'), ('josé', 9, 'ñéx')) AS t(column1, column2, column3);
+----
+joséñé joséñé
+joséñéxñé joséñéxñé
+
+# rpad array path with a Unicode string, a zero length and a NULL length; the
+# trailing '|' keeps the padding visible, since trailing whitespace is trimmed
+query T
+SELECT rpad(column1, column2) || '|' FROM (VALUES ('josé', 6), ('hi', 0), ('hello', 3), ('hi', NULL)) AS t(column1, column2);
+----
+josé |
+|
+hel|
+NULL
+
+# rpad array path with different string widths for the input and the fill
+query BBB
+SELECT
+ rpad(arrow_cast(column1, 'Utf8View'), column2, arrow_cast(column3, 'LargeUtf8')) IS NOT DISTINCT FROM column4,
+ rpad(arrow_cast(column1, 'LargeUtf8'), column2, arrow_cast(column3, 'Utf8View')) IS NOT DISTINCT FROM column4,
+ rpad(arrow_cast(column1, 'Utf8'), column2, arrow_cast(column3, 'Utf8View')) IS NOT DISTINCT FROM column4
+FROM (VALUES ('hi', 4, 'ab', 'hiab'), ('josé', 5, 'ñ', 'joséñ'), (NULL, 4, 'ab', NULL)) AS t(column1, column2, column3, column4);
+----
+true true true
+true true true
+true true true
+
+# a length column above i32::MAX is an error on the array path
+query error rpad requested length 2147483648 too large
+SELECT rpad('hi', column1) FROM (VALUES (2147483648), (1)) AS t(column1);
+
# a large scalar target length skips the scalar fast path
query I
SELECT character_length(rpad('x', 16385, 'a'));