Apache DataFusion SQL Query Engine
chore: Rename concat-specific string builders, make pub(crate) (#21695)
Neil Conway766dff161fea
diff --git a/datafusion/functions/src/string/concat.rs b/datafusion/functions/src/string/concat.rsindex e22c3b80da..b10db23472 100644--- a/datafusion/functions/src/string/concat.rs+++ b/datafusion/functions/src/string/concat.rs@@ -22,7 +22,8 @@ use std::sync::Arc;use crate::string::concat;use crate::strings::{- ColumnarValueRef, LargeStringArrayBuilder, StringArrayBuilder, StringViewArrayBuilder,+ ColumnarValueRef, ConcatLargeStringBuilder, ConcatStringBuilder,+ ConcatStringViewBuilder,};use datafusion_common::cast::{as_binary_array, as_string_array, as_string_view_array};use datafusion_common::{@@ -242,7 +243,7 @@ impl ScalarUDFImpl for ConcatFunc {match return_datatype {DataType::Utf8 => {- let mut builder = StringArrayBuilder::with_capacity(len, data_size);+ let mut builder = ConcatStringBuilder::with_capacity(len, data_size);for i in 0..len {columns.iter()@@ -254,7 +255,7 @@ impl ScalarUDFImpl for ConcatFunc {Ok(ColumnarValue::Array(Arc::new(string_array)))}DataType::Utf8View => {- let mut builder = StringViewArrayBuilder::with_capacity(len, data_size);+ let mut builder = ConcatStringViewBuilder::with_capacity(len, data_size);for i in 0..len {columns.iter()@@ -266,7 +267,7 @@ impl ScalarUDFImpl for ConcatFunc {Ok(ColumnarValue::Array(Arc::new(string_array)))}DataType::LargeUtf8 => {- let mut builder = LargeStringArrayBuilder::with_capacity(len, data_size);+ let mut builder = ConcatLargeStringBuilder::with_capacity(len, data_size);for i in 0..len {columns.iter()diff --git a/datafusion/functions/src/string/concat_ws.rs b/datafusion/functions/src/string/concat_ws.rsindex fc4cc6e43b..2c2d4bd421 100644--- a/datafusion/functions/src/string/concat_ws.rs+++ b/datafusion/functions/src/string/concat_ws.rs@@ -24,7 +24,8 @@ use crate::string::concat;use crate::string::concat::simplify_concat;use crate::string::concat_ws;use crate::strings::{- ColumnarValueRef, LargeStringArrayBuilder, StringArrayBuilder, StringViewArrayBuilder,+ ColumnarValueRef, ConcatLargeStringBuilder, ConcatStringBuilder,+ ConcatStringViewBuilder,};use datafusion_common::cast::{as_large_string_array, as_string_array, as_string_view_array,@@ -311,7 +312,7 @@ impl ScalarUDFImpl for ConcatWsFunc {match return_datatype {DataType::Utf8View => {- let mut builder = StringViewArrayBuilder::with_capacity(len, data_size);+ let mut builder = ConcatStringViewBuilder::with_capacity(len, data_size);for i in 0..len {if !sep.is_valid(i) {builder.append_offset()?;@@ -332,7 +333,7 @@ impl ScalarUDFImpl for ConcatWsFunc {Ok(ColumnarValue::Array(Arc::new(builder.finish(sep.nulls())?)))}DataType::LargeUtf8 => {- let mut builder = LargeStringArrayBuilder::with_capacity(len, data_size);+ let mut builder = ConcatLargeStringBuilder::with_capacity(len, data_size);for i in 0..len {if !sep.is_valid(i) {builder.append_offset()?;@@ -353,7 +354,7 @@ impl ScalarUDFImpl for ConcatWsFunc {Ok(ColumnarValue::Array(Arc::new(builder.finish(sep.nulls())?)))}_ => {- let mut builder = StringArrayBuilder::with_capacity(len, data_size);+ let mut builder = ConcatStringBuilder::with_capacity(len, data_size);for i in 0..len {if !sep.is_valid(i) {builder.append_offset()?;diff --git a/datafusion/functions/src/strings.rs b/datafusion/functions/src/strings.rsindex bcfb3d666c..f986ffd2e3 100644--- a/datafusion/functions/src/strings.rs+++ b/datafusion/functions/src/strings.rs@@ -26,17 +26,20 @@ use arrow::array::{use arrow::buffer::{Buffer, MutableBuffer, NullBuffer, ScalarBuffer};use arrow::datatypes::DataType;-/// Optimized version of the StringBuilder in Arrow that:-/// 1. Precalculating the expected length of the result, avoiding reallocations.-/// 2. Avoids creating / incrementally creating a `NullBufferBuilder`-pub struct StringArrayBuilder {+/// Builder used by `concat`/`concat_ws` to assemble a [`StringArray`] one row+/// at a time from multiple input columns.+///+/// Each row is written via repeated `write` calls, followed by a single+/// `append_offset` call to commit the row. The output null buffer is supplied+/// by the caller at `finish` time.+pub(crate) struct ConcatStringBuilder {offsets_buffer: MutableBuffer,value_buffer: MutableBuffer,/// If true, a safety check is required during the `finish` calltainted: bool,}-impl StringArrayBuilder {+impl ConcatStringBuilder {pub fn with_capacity(item_capacity: usize, data_capacity: usize) -> Self {let capacity = item_capacity.checked_add(1)@@ -151,13 +154,13 @@ impl StringArrayBuilder {}}-/// Optimized version of Arrow's [`StringViewBuilder`]. Rather than adding NULLs-/// on a row-by-row basis, the caller should provide nulls when calling-/// [`finish`](Self::finish). This allows callers to compute nulls more-/// efficiently (e.g., via bulk bitmap operations).+/// Builder used by `concat`/`concat_ws` to assemble a [`StringViewArray`] one+/// row at a time from multiple input columns.///-/// [`StringViewBuilder`]: arrow::array::StringViewBuilder-pub struct StringViewArrayBuilder {+/// Each row is written via repeated `write` calls, followed by a single+/// `append_offset` call to commit the row as a single string view. The output+/// null buffer is supplied by the caller at `finish` time.+pub(crate) struct ConcatStringViewBuilder {views: Vec<u128>,data: Vec<u8>,block: Vec<u8>,@@ -165,7 +168,7 @@ pub struct StringViewArrayBuilder {tainted: bool,}-impl StringViewArrayBuilder {+impl ConcatStringViewBuilder {pub fn with_capacity(item_capacity: usize, data_capacity: usize) -> Self {Self {views: Vec::with_capacity(item_capacity),@@ -286,14 +289,17 @@ impl StringViewArrayBuilder {}}-pub struct LargeStringArrayBuilder {+/// Builder used by `concat`/`concat_ws` to assemble a [`LargeStringArray`] one+/// row at a time from multiple input columns. See [`ConcatStringBuilder`] for+/// details on the row-composition contract.+pub(crate) struct ConcatLargeStringBuilder {offsets_buffer: MutableBuffer,value_buffer: MutableBuffer,/// If true, a safety check is required during the `finish` calltainted: bool,}-impl LargeStringArrayBuilder {+impl ConcatLargeStringBuilder {pub fn with_capacity(item_capacity: usize, data_capacity: usize) -> Self {let capacity = item_capacity.checked_add(1)@@ -426,7 +432,7 @@ impl LargeStringArrayBuilder {/// LLVM is apparently overly eager to inline this function into some hot loops,/// which bloats them and regresses performance, so we disable inlining for now.#[inline(never)]-pub fn append_view(+pub(crate) fn append_view(views_buffer: &mut Vec<u128>,original_view: &u128,substr: &str,@@ -447,7 +453,7 @@ pub fn append_view(}#[derive(Debug)]-pub enum ColumnarValueRef<'a> {+pub(crate) enum ColumnarValueRef<'a> {Scalar(&'a [u8]),NullableArray(&'a StringArray),NonNullableArray(&'a StringArray),@@ -497,13 +503,13 @@ mod tests {#[test]#[should_panic(expected = "capacity integer overflow")]- fn test_overflow_string_array_builder() {- let _builder = StringArrayBuilder::with_capacity(usize::MAX, usize::MAX);+ fn test_overflow_concat_string_builder() {+ let _builder = ConcatStringBuilder::with_capacity(usize::MAX, usize::MAX);}#[test]#[should_panic(expected = "capacity integer overflow")]- fn test_overflow_large_string_array_builder() {- let _builder = LargeStringArrayBuilder::with_capacity(usize::MAX, usize::MAX);+ fn test_overflow_concat_large_string_builder() {+ let _builder = ConcatLargeStringBuilder::with_capacity(usize::MAX, usize::MAX);}}diff --git a/docs/source/library-user-guide/upgrading/54.0.0.md b/docs/source/library-user-guide/upgrading/54.0.0.mdindex 030ca729f2..cadd365e1f 100644--- a/docs/source/library-user-guide/upgrading/54.0.0.md+++ b/docs/source/library-user-guide/upgrading/54.0.0.md@@ -320,6 +320,14 @@ The difference is only observable for strings containing combining charactersclusters (e.g., ZWJ emoji sequences). For ASCII and most common Unicode text,behavior is unchanged.+### Items in `datafusion_functions::strings` are no longer public++`StringArrayBuilder`, `LargeStringArrayBuilder`, `StringViewArrayBuilder`,+`ColumnarValueRef`, and `append_view` have been reduced to `pub(crate)`. They+were only ever used to implement `concat` and `concat_ws` inside the crate. If+you were importing them externally, use Arrow's corresponding builders with a+caller-computed `NullBuffer`.+[#17861]: https://github.com/apache/datafusion/pull/17861### `approx_percentile_cont`, `approx_percentile_cont_with_weight`, `approx_median` now coerce to floats