Apache DataFusion SQL Query Engine
refactor: deduplicate `concat` string and binary builders (#26318)
Andy Grovea99fb4ad72eb
diff --git a/datafusion/functions/src/binaries.rs b/datafusion/functions/src/binaries.rsindex b335ba10ef..5534e916a2 100644--- a/datafusion/functions/src/binaries.rs+++ b/datafusion/functions/src/binaries.rs@@ -15,243 +15,9 @@// specific language governing permissions and limitations// under the License.-use crate::strings::{ColumnarValueRef, ConcatBuilder};-use arrow::array::{- Array, ArrayDataBuilder, ArrayRef, BinaryViewArray, GenericBinaryArray,- OffsetSizeTrait, make_view,-};-use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer, NullBuffer, ScalarBuffer};-use datafusion_common::{Result, exec_datafusion_err, exec_err, internal_err};-use std::marker::PhantomData;-use std::sync::Arc;+use crate::strings::{ConcatByteArrayBuilder, ConcatByteViewBuilder};+use arrow::datatypes::{BinaryType, BinaryViewType, LargeBinaryType};-pub(crate) struct ConcatGenericBinaryBuilder<O: OffsetSizeTrait + ArrowNativeType> {- offsets_buffer: MutableBuffer,- value_buffer: MutableBuffer,- _phantom: PhantomData<O>,-}-pub(crate) type ConcatBinaryBuilder = ConcatGenericBinaryBuilder<i32>;-pub(crate) type ConcatLargeBinaryBuilder = ConcatGenericBinaryBuilder<i64>;--impl<O: OffsetSizeTrait + ArrowNativeType> ConcatGenericBinaryBuilder<O> {- pub fn with_capacity(item_capacity: usize, data_capacity: usize) -> Self {- let capacity = item_capacity- .checked_add(1)- .map(|i| i.saturating_mul(size_of::<O>()))- .expect("capacity integer overflow");-- let mut offsets_buffer = MutableBuffer::with_capacity(capacity);- // SAFETY: the first offset value is definitely not going to exceed the bounds.- unsafe { offsets_buffer.push_unchecked(O::usize_as(0)) };- Self {- offsets_buffer,- value_buffer: MutableBuffer::with_capacity(data_capacity),- _phantom: PhantomData,- }- }-}--impl<O: OffsetSizeTrait + ArrowNativeType> ConcatBuilder- for ConcatGenericBinaryBuilder<O>-{- fn write<const CHECK_VALID: bool>(- &mut self,- column: &ColumnarValueRef,- i: usize,- ) -> Result<()> {- match column {- ColumnarValueRef::Scalar(s) => {- self.value_buffer.extend_from_slice(s);- }- ColumnarValueRef::NullableBinaryArray(array) => {- if !CHECK_VALID || array.is_valid(i) {- self.value_buffer.extend_from_slice(array.value(i));- }- }- ColumnarValueRef::NullableLargeBinaryArray(array) => {- if !CHECK_VALID || array.is_valid(i) {- self.value_buffer.extend_from_slice(array.value(i));- }- }- ColumnarValueRef::NullableBinaryViewArray(array) => {- if !CHECK_VALID || array.is_valid(i) {- self.value_buffer.extend_from_slice(array.value(i));- }- }- ColumnarValueRef::NonNullableBinaryArray(array) => {- self.value_buffer.extend_from_slice(array.value(i));- }- ColumnarValueRef::NonNullableLargeBinaryArray(array) => {- self.value_buffer.extend_from_slice(array.value(i));- }- ColumnarValueRef::NonNullableBinaryViewArray(array) => {- self.value_buffer.extend_from_slice(array.value(i));- }- _ => {- return exec_err!(- "concat: unexpected column type for binary builder: {column:?}"- );- }- }- Ok(())- }-- fn append_offset(&mut self) -> Result<()> {- let next_offset: O = O::from_usize(self.value_buffer.len())- .ok_or_else(|| exec_datafusion_err!("byte array offset overflow"))?;- self.offsets_buffer.push(next_offset);- Ok(())- }-- /// Finalize the builder into a concrete [`GenericBinaryArray<O>`].- ///- /// # Errors- ///- /// Returns an error when:- ///- /// - the provided `null_buffer` is not the same length as the `offsets_buffer`.- fn finish(self, null_buffer: Option<NullBuffer>) -> Result<ArrayRef> {- let row_count = self.offsets_buffer.len() / size_of::<O>() - 1;- if let Some(ref null_buffer) = null_buffer- && null_buffer.len() != row_count- {- return internal_err!(- "Null buffer and offsets buffer must be the same length"- );- }- let array_builder = ArrayDataBuilder::new(GenericBinaryArray::<O>::DATA_TYPE)- .len(row_count)- .add_buffer(self.offsets_buffer.into())- .add_buffer(self.value_buffer.into())- .nulls(null_buffer);- // SAFETY: all data that was appended was valid and the values- // and offsets were created correctly- let array_data = unsafe { array_builder.build_unchecked() };- let array = GenericBinaryArray::<O>::from(array_data);- Ok(Arc::new(array))- }-}--/// Builder used by `concat`/`concat_ws` to assemble a [`BinaryViewArray`] one-/// row at a time from multiple input columns.-///-/// Each row is written via repeated `write` calls (one per input-/// fragment) followed by a single `append_offset` to commit the row-/// as a single binary view. The output null buffer is supplied by the caller-/// at `finish` time, avoiding per-row NULL handling work.-///-pub(crate) struct ConcatBinaryViewBuilder {- views: Vec<u128>,- data: Vec<u8>,- block: Vec<u8>,-}--impl ConcatBinaryViewBuilder {- pub fn with_capacity(item_capacity: usize, data_capacity: usize) -> Self {- Self {- views: Vec::with_capacity(item_capacity),- data: Vec::with_capacity(data_capacity),- block: vec![],- }- }-}--impl ConcatBuilder for ConcatBinaryViewBuilder {- fn write<const CHECK_VALID: bool>(- &mut self,- column: &ColumnarValueRef,- i: usize,- ) -> Result<()> {- match column {- ColumnarValueRef::Scalar(s) => {- self.block.extend_from_slice(s);- }- ColumnarValueRef::NullableBinaryArray(array) => {- if !CHECK_VALID || array.is_valid(i) {- self.block.extend_from_slice(array.value(i));- }- }- ColumnarValueRef::NullableLargeBinaryArray(array) => {- if !CHECK_VALID || array.is_valid(i) {- self.block.extend_from_slice(array.value(i));- }- }- ColumnarValueRef::NullableBinaryViewArray(array) => {- if !CHECK_VALID || array.is_valid(i) {- self.block.extend_from_slice(array.value(i));- }- }- ColumnarValueRef::NonNullableBinaryArray(array) => {- self.block.extend_from_slice(array.value(i));- }- ColumnarValueRef::NonNullableLargeBinaryArray(array) => {- self.block.extend_from_slice(array.value(i));- }- ColumnarValueRef::NonNullableBinaryViewArray(array) => {- self.block.extend_from_slice(array.value(i));- }- _ => {- return exec_err!(- "concat: unexpected column type for binary view builder: {column:?}"- );- }- }- Ok(())- }-- /// Finalizes the current row by converting the accumulated data into a- /// StringView and appending it to the views buffer.- fn append_offset(&mut self) -> Result<()> {- let v = &self.block;- if v.len() > 12 {- let offset: u32 = self- .data- .len()- .try_into()- .map_err(|_| exec_datafusion_err!("byte array offset overflow"))?;- self.data.extend_from_slice(v);- self.views.push(make_view(v, 0, offset));- } else {- self.views.push(make_view(v, 0, 0));- }-- self.block.clear();- Ok(())- }-- /// Finalize the builder into a concrete [`BinaryViewArray`].- ///- /// # Errors- ///- /// Returns an error when:- ///- /// - the provided `null_buffer` length does not match the row count.- fn finish(self, null_buffer: Option<NullBuffer>) -> Result<ArrayRef> {- if let Some(ref nulls) = null_buffer- && nulls.len() != self.views.len()- {- return internal_err!(- "Null buffer length ({}) must match row count ({})",- nulls.len(),- self.views.len()- );- }-- let buffers: Vec<Buffer> = if self.data.is_empty() {- vec![]- } else {- vec![Buffer::from(self.data)]- };-- // SAFETY: views were constructed with correct lengths, offsets, and- // prefixes.- let array = unsafe {- BinaryViewArray::new_unchecked(- ScalarBuffer::from(self.views),- buffers.into(),- null_buffer,- )- };- Ok(Arc::new(array))- }-}+pub(crate) type ConcatBinaryBuilder = ConcatByteArrayBuilder<BinaryType>;+pub(crate) type ConcatLargeBinaryBuilder = ConcatByteArrayBuilder<LargeBinaryType>;+pub(crate) type ConcatBinaryViewBuilder = ConcatByteViewBuilder<BinaryViewType>;diff --git a/datafusion/functions/src/strings.rs b/datafusion/functions/src/strings.rsindex 671bca0d4a..1de61efa1d 100644--- a/datafusion/functions/src/strings.rs+++ b/datafusion/functions/src/strings.rs@@ -28,12 +28,15 @@ use datafusion_common::{use arrow::array::{Array, ArrayAccessor, ArrayDataBuilder, ArrayRef, BinaryArray, BinaryViewArray,- ByteView, GenericStringArray, LargeBinaryArray, LargeStringArray,- MAX_INLINE_VIEW_LEN, OffsetSizeTrait, StringArray, StringViewArray,- as_largestring_array, make_view,+ ByteView, GenericByteArray, GenericByteViewArray, GenericStringArray,+ LargeBinaryArray, LargeStringArray, MAX_INLINE_VIEW_LEN, OffsetSizeTrait,+ StringArray, StringViewArray, as_largestring_array, make_view,};use arrow::buffer::{Buffer, MutableBuffer, NullBuffer, ScalarBuffer};-use arrow::datatypes::DataType;+use arrow::datatypes::{+ ByteArrayType, ByteViewType, DataType, GenericBinaryType, GenericStringType,+ LargeUtf8Type, StringViewType, Utf8Type,+};use arrow_buffer::ArrowNativeType;use datafusion_common::cast::{as_binary_array, as_binary_view_array, as_large_binary_array, as_string_array,@@ -54,8 +57,26 @@ pub(crate) trait ConcatBuilder {fn finish(self, null_buffer: Option<NullBuffer>) -> Result<ArrayRef>;}-/// Builder used by `concat`/`concat_ws` to assemble a [`GenericStringArray<O>`]-/// (`StringArray` or `LargeStringArray`) one row at a time from multiple input columns.+/// A [`ByteArrayType`] that [`ConcatByteArrayBuilder`] can build.+///+/// Like [`ByteViewType::IS_UTF8`], `IS_UTF8` tells string types from binary+/// ones. Using `T::DATA_TYPE.is_string()` in [`ColumnarValueRef::write_bytes`]+/// instead would build a `DataType` and call its destructor for every value.+pub(crate) trait ConcatByteArrayType: ByteArrayType {+ const IS_UTF8: bool;+}++impl<O: OffsetSizeTrait> ConcatByteArrayType for GenericStringType<O> {+ const IS_UTF8: bool = true;+}++impl<O: OffsetSizeTrait> ConcatByteArrayType for GenericBinaryType<O> {+ const IS_UTF8: bool = false;+}++/// Builder used by `concat`/`concat_ws` to assemble a [`GenericByteArray<T>`]+/// (`StringArray`, `LargeStringArray`, `BinaryArray` or `LargeBinaryArray`)+/// one row at a time from multiple input columns.////// Each row is written via repeated `write` calls (one per input fragment)/// followed by a single `append_offset` to commit the row. The output null@@ -64,24 +85,24 @@ pub(crate) trait ConcatBuilder {////// For the common "produce one `&str` per row" pattern, prefer/// `GenericStringArrayBuilder` instead.-pub(crate) struct ConcatGenericStringBuilder<O: OffsetSizeTrait + ArrowNativeType> {+pub(crate) struct ConcatByteArrayBuilder<T: ConcatByteArrayType> {offsets_buffer: MutableBuffer,value_buffer: MutableBuffer,- _phantom: PhantomData<O>,+ _phantom: PhantomData<T>,}-pub(crate) type ConcatStringBuilder = ConcatGenericStringBuilder<i32>;-pub(crate) type ConcatLargeStringBuilder = ConcatGenericStringBuilder<i64>;+pub(crate) type ConcatStringBuilder = ConcatByteArrayBuilder<Utf8Type>;+pub(crate) type ConcatLargeStringBuilder = ConcatByteArrayBuilder<LargeUtf8Type>;-impl<O: OffsetSizeTrait + ArrowNativeType> ConcatGenericStringBuilder<O> {+impl<T: ConcatByteArrayType> ConcatByteArrayBuilder<T> {pub fn with_capacity(item_capacity: usize, data_capacity: usize) -> Self {let capacity = item_capacity.checked_add(1)- .map(|i| i.saturating_mul(size_of::<O>()))+ .map(|i| i.saturating_mul(size_of::<T::Offset>())).expect("capacity integer overflow");let mut offsets_buffer = MutableBuffer::with_capacity(capacity);// SAFETY: the first offset value is definitely not going to exceed the bounds.- unsafe { offsets_buffer.push_unchecked(O::usize_as(0)) };+ unsafe { offsets_buffer.push_unchecked(T::Offset::usize_as(0)) };Self {offsets_buffer,value_buffer: MutableBuffer::with_capacity(data_capacity),@@ -90,68 +111,25 @@ impl<O: OffsetSizeTrait + ArrowNativeType> ConcatGenericStringBuilder<O> {}}-impl<O: OffsetSizeTrait + ArrowNativeType> ConcatBuilder- for ConcatGenericStringBuilder<O>-{+impl<T: ConcatByteArrayType> ConcatBuilder for ConcatByteArrayBuilder<T> {fn write<const CHECK_VALID: bool>(&mut self,column: &ColumnarValueRef,i: usize,) -> Result<()> {- match column {- ColumnarValueRef::Scalar(s) => {- std::str::from_utf8(s).map_err(|_| {- exec_datafusion_err!("concat: scalar bytes are not valid UTF-8")- })?;- self.value_buffer.extend_from_slice(s);- }- ColumnarValueRef::NullableArray(array) => {- if !CHECK_VALID || array.is_valid(i) {- self.value_buffer- .extend_from_slice(array.value(i).as_bytes());- }- }- ColumnarValueRef::NullableLargeStringArray(array) => {- if !CHECK_VALID || array.is_valid(i) {- self.value_buffer- .extend_from_slice(array.value(i).as_bytes());- }- }- ColumnarValueRef::NullableStringViewArray(array) => {- if !CHECK_VALID || array.is_valid(i) {- self.value_buffer- .extend_from_slice(array.value(i).as_bytes());- }- }- ColumnarValueRef::NonNullableArray(array) => {- self.value_buffer- .extend_from_slice(array.value(i).as_bytes());- }- ColumnarValueRef::NonNullableLargeStringArray(array) => {- self.value_buffer- .extend_from_slice(array.value(i).as_bytes());- }- ColumnarValueRef::NonNullableStringViewArray(array) => {- self.value_buffer- .extend_from_slice(array.value(i).as_bytes());- }- _ => {- return exec_err!(- "concat: unexpected column type for string builder: {column:?}"- );- }- }- Ok(())+ column.write_bytes::<CHECK_VALID>(i, T::IS_UTF8, |bytes| {+ self.value_buffer.extend_from_slice(bytes)+ })}fn append_offset(&mut self) -> Result<()> {- let next_offset: O = O::from_usize(self.value_buffer.len())+ let next_offset = T::Offset::from_usize(self.value_buffer.len()).ok_or_else(|| exec_datafusion_err!("byte array offset overflow"))?;self.offsets_buffer.push(next_offset);Ok(())}- /// Finalize the builder into a concrete [`GenericStringArray<O>`].+ /// Finalize the builder into a concrete [`GenericByteArray<T>`].////// # Errors///@@ -159,7 +137,7 @@ impl<O: OffsetSizeTrait + ArrowNativeType> ConcatBuilder////// - the provided `null_buffer` is not the same length as the `offsets_buffer`.fn finish(self, null_buffer: Option<NullBuffer>) -> Result<ArrayRef> {- let row_count = self.offsets_buffer.len() / size_of::<O>() - 1;+ let row_count = self.offsets_buffer.len() / size_of::<T::Offset>() - 1;if let Some(ref null_buffer) = null_buffer&& null_buffer.len() != row_count{@@ -167,96 +145,65 @@ impl<O: OffsetSizeTrait + ArrowNativeType> ConcatBuilder"Null buffer and offsets buffer must be the same length");}- let array_builder = ArrayDataBuilder::new(GenericStringArray::<O>::DATA_TYPE)+ let array_builder = ArrayDataBuilder::new(T::DATA_TYPE).len(row_count).add_buffer(self.offsets_buffer.into()).add_buffer(self.value_buffer.into()).nulls(null_buffer);- // SAFETY: all data that was appended was valid UTF8 and the values- // and offsets were created correctly+ // SAFETY: the values and offsets were created correctly, and when `T`+ // is a string type, `write_bytes` only wrote valid UTF-8let array_data = unsafe { array_builder.build_unchecked() };- let array = GenericStringArray::<O>::from(array_data);+ let array = GenericByteArray::<T>::from(array_data);Ok(Arc::new(array))}}-/// Builder used by `concat`/`concat_ws` to assemble a [`StringViewArray`] one+/// Builder used by `concat`/`concat_ws` to assemble a+/// [`GenericByteViewArray<T>`] (`StringViewArray` or `BinaryViewArray`) one/// row at a time from multiple input columns.////// Each row is written via repeated `write` calls (one per input/// fragment) followed by a single `append_offset` to commit the row-/// as a single string view. The output null buffer is supplied by the caller+/// as a single view. The output null buffer is supplied by the caller/// at `finish` time, avoiding per-row NULL handling work.////// For the common "produce one `&str` per row" pattern, prefer/// [`StringViewArrayBuilder`] instead.-pub(crate) struct ConcatStringViewBuilder {+pub(crate) struct ConcatByteViewBuilder<T: ByteViewType> {views: Vec<u128>,data: Vec<u8>,block: Vec<u8>,+ _phantom: PhantomData<T>,}+pub(crate) type ConcatStringViewBuilder = ConcatByteViewBuilder<StringViewType>;-impl ConcatStringViewBuilder {+impl<T: ByteViewType> ConcatByteViewBuilder<T> {pub fn with_capacity(item_capacity: usize, data_capacity: usize) -> Self {Self {views: Vec::with_capacity(item_capacity),data: Vec::with_capacity(data_capacity),block: vec![],+ _phantom: PhantomData,}}}-impl ConcatBuilder for ConcatStringViewBuilder {+impl<T: ByteViewType> ConcatBuilder for ConcatByteViewBuilder<T> {fn write<const CHECK_VALID: bool>(&mut self,column: &ColumnarValueRef,i: usize,) -> Result<()> {- match column {- ColumnarValueRef::Scalar(s) => {- std::str::from_utf8(s).map_err(|_| {- exec_datafusion_err!("concat: scalar bytes are not valid UTF-8")- })?;- self.block.extend_from_slice(s);- }- ColumnarValueRef::NullableArray(array) => {- if !CHECK_VALID || array.is_valid(i) {- self.block.extend_from_slice(array.value(i).as_bytes());- }- }- ColumnarValueRef::NullableLargeStringArray(array) => {- if !CHECK_VALID || array.is_valid(i) {- self.block.extend_from_slice(array.value(i).as_bytes());- }- }- ColumnarValueRef::NullableStringViewArray(array) => {- if !CHECK_VALID || array.is_valid(i) {- self.block.extend_from_slice(array.value(i).as_bytes());- }- }- ColumnarValueRef::NonNullableArray(array) => {- self.block.extend_from_slice(array.value(i).as_bytes());- }- ColumnarValueRef::NonNullableLargeStringArray(array) => {- self.block.extend_from_slice(array.value(i).as_bytes());- }- ColumnarValueRef::NonNullableStringViewArray(array) => {- self.block.extend_from_slice(array.value(i).as_bytes());- }- _ => {- return exec_err!(- "concat: unexpected column type for string view builder: {column:?}"- );- }- }- Ok(())+ column.write_bytes::<CHECK_VALID>(i, T::IS_UTF8, |bytes| {+ self.block.extend_from_slice(bytes)+ })}/// Finalizes the current row by converting the accumulated data into a- /// StringView and appending it to the views buffer.+ /// view and appending it to the views buffer.fn append_offset(&mut self) -> Result<()> {let v = &self.block;- if v.len() > 12 {+ if v.len() > MAX_INLINE_LEN {let offset: u32 = self.data.len()@@ -272,7 +219,7 @@ impl ConcatBuilder for ConcatStringViewBuilder {Ok(())}- /// Finalize the builder into a concrete [`StringViewArray`].+ /// Finalize the builder into a concrete [`GenericByteViewArray<T>`].////// # Errors///@@ -297,10 +244,10 @@ impl ConcatBuilder for ConcatStringViewBuilder {};// SAFETY: views were constructed with correct lengths, offsets, and- // prefixes. All input fragments came from string arrays or string- // scalars, all of which are valid UTF-8.+ // prefixes, and when `T` is `StringViewType`, `write_bytes` only+ // wrote valid UTF-8.let array = unsafe {- StringViewArray::new_unchecked(+ GenericByteViewArray::<T>::new_unchecked(ScalarBuffer::from(self.views),buffers.into(),null_buffer,@@ -1356,6 +1303,81 @@ impl ColumnarValueRef<'_> {}}+ /// Calls `write` with the bytes of the value at row `i`, for appending to a+ /// string (`utf8`) or binary array, unless the value is null.+ ///+ /// Validity is only checked if `CHECK_VALID` is true, so callers passing+ /// `false` must have checked it already.+ ///+ /// # Errors+ ///+ /// String arrays only take values from string arrays and valid UTF-8+ /// scalars, and binary arrays only from binary arrays and scalars. Returns+ /// an error, without calling `write`, for any other value.+ #[inline]+ pub fn write_bytes<const CHECK_VALID: bool>(+ &self,+ i: usize,+ utf8: bool,+ write: impl FnOnce(&[u8]),+ ) -> Result<()> {+ match self {+ Self::Scalar(s) => {+ if utf8 && std::str::from_utf8(s).is_err() {+ return exec_err!("concat: scalar bytes are not valid UTF-8");+ }+ write(s)+ }+ Self::NullableArray(array) if utf8 => {+ if !CHECK_VALID || array.is_valid(i) {+ write(array.value(i).as_bytes())+ }+ }+ Self::NonNullableArray(array) if utf8 => write(array.value(i).as_bytes()),+ Self::NullableLargeStringArray(array) if utf8 => {+ if !CHECK_VALID || array.is_valid(i) {+ write(array.value(i).as_bytes())+ }+ }+ Self::NonNullableLargeStringArray(array) if utf8 => {+ write(array.value(i).as_bytes())+ }+ Self::NullableStringViewArray(array) if utf8 => {+ if !CHECK_VALID || array.is_valid(i) {+ write(array.value(i).as_bytes())+ }+ }+ Self::NonNullableStringViewArray(array) if utf8 => {+ write(array.value(i).as_bytes())+ }+ Self::NullableBinaryArray(array) if !utf8 => {+ if !CHECK_VALID || array.is_valid(i) {+ write(array.value(i))+ }+ }+ Self::NonNullableBinaryArray(array) if !utf8 => write(array.value(i)),+ Self::NullableLargeBinaryArray(array) if !utf8 => {+ if !CHECK_VALID || array.is_valid(i) {+ write(array.value(i))+ }+ }+ Self::NonNullableLargeBinaryArray(array) if !utf8 => write(array.value(i)),+ Self::NullableBinaryViewArray(array) if !utf8 => {+ if !CHECK_VALID || array.is_valid(i) {+ write(array.value(i))+ }+ }+ Self::NonNullableBinaryViewArray(array) if !utf8 => write(array.value(i)),+ _ => {+ let kind = if utf8 { "string" } else { "binary" };+ return exec_err!(+ "concat: unexpected column type for {kind} builder: {self:?}"+ );+ }+ }+ Ok(())+ }+/// Parse a [`ColumnarValue`] argument into `ColumnarValueRef`./// Returns `None` when the argument is null or null scalar/// Returns an error when a columnar value type is not supported.@@ -1511,6 +1533,9 @@ pub(crate) fn widest_string_type(types: &[DataType]) -> DataType {#[cfg(test)]mod tests {use super::*;+ use crate::binaries::{+ ConcatBinaryBuilder, ConcatBinaryViewBuilder, ConcatLargeBinaryBuilder,+ };use crate::utils::test::sliced_byte_array;#[test]@@ -1644,6 +1669,90 @@ mod tests {let _builder = ConcatLargeStringBuilder::with_capacity(usize::MAX, usize::MAX);}+ /// Asserts that builders made by `new` accept the `valid` columns and+ /// reject the `invalid` ones.+ fn check_concat_inputs<B: ConcatBuilder>(+ new: impl Fn() -> B,+ valid: &[&ColumnarValueRef],+ invalid: &[&ColumnarValueRef],+ ) {+ for column in valid {+ assert!(new().write::<true>(column, 0).is_ok(), "{column:?}");+ }+ for column in invalid {+ assert!(new().write::<true>(column, 0).is_err(), "{column:?}");+ }+ }++ #[test]+ fn concat_builders_check_input_types() {+ let strings = StringArray::from(vec!["a"]);+ let large_strings = LargeStringArray::from(vec!["a"]);+ let string_views = StringViewArray::from(vec!["a"]);+ let binaries = BinaryArray::from_vec(vec![b"a"]);+ let large_binaries = LargeBinaryArray::from_vec(vec![b"a"]);+ let binary_views = BinaryViewArray::from_iter_values([b"a"]);+ let string_columns = [+ ColumnarValueRef::NullableArray(&strings),+ ColumnarValueRef::NonNullableArray(&strings),+ ColumnarValueRef::NullableLargeStringArray(&large_strings),+ ColumnarValueRef::NonNullableLargeStringArray(&large_strings),+ ColumnarValueRef::NullableStringViewArray(&string_views),+ ColumnarValueRef::NonNullableStringViewArray(&string_views),+ ];+ let binary_columns = [+ ColumnarValueRef::NullableBinaryArray(&binaries),+ ColumnarValueRef::NonNullableBinaryArray(&binaries),+ ColumnarValueRef::NullableLargeBinaryArray(&large_binaries),+ ColumnarValueRef::NonNullableLargeBinaryArray(&large_binaries),+ ColumnarValueRef::NullableBinaryViewArray(&binary_views),+ ColumnarValueRef::NonNullableBinaryViewArray(&binary_views),+ ];+ let utf8_scalar = ColumnarValueRef::Scalar(b"a");+ let non_utf8_scalar = ColumnarValueRef::Scalar(&[0xff]);++ // String builders take string arrays and UTF-8 scalars+ let valid: Vec<_> = string_columns.iter().chain([&utf8_scalar]).collect();+ let invalid: Vec<_> = binary_columns.iter().chain([&non_utf8_scalar]).collect();+ check_concat_inputs(+ || ConcatStringBuilder::with_capacity(1, 1),+ &valid,+ &invalid,+ );+ check_concat_inputs(+ || ConcatLargeStringBuilder::with_capacity(1, 1),+ &valid,+ &invalid,+ );+ check_concat_inputs(+ || ConcatStringViewBuilder::with_capacity(1, 1),+ &valid,+ &invalid,+ );++ // Binary builders take binary arrays and any scalar+ let valid: Vec<_> = binary_columns+ .iter()+ .chain([&utf8_scalar, &non_utf8_scalar])+ .collect();+ let invalid: Vec<_> = string_columns.iter().collect();+ check_concat_inputs(+ || ConcatBinaryBuilder::with_capacity(1, 1),+ &valid,+ &invalid,+ );+ check_concat_inputs(+ || ConcatLargeBinaryBuilder::with_capacity(1, 1),+ &valid,+ &invalid,+ );+ check_concat_inputs(+ || ConcatBinaryViewBuilder::with_capacity(1, 1),+ &valid,+ &invalid,+ );+ }+#[test]fn bulk_append_value_with_nulls() {check_on_all_builders!(diff --git a/datafusion/sqllogictest/test_files/string/concat.slt b/datafusion/sqllogictest/test_files/string/concat.sltindex 1749a57591..eb39c062ed 100644--- a/datafusion/sqllogictest/test_files/string/concat.slt+++ b/datafusion/sqllogictest/test_files/string/concat.slt@@ -136,3 +136,72 @@ SELECT concat_ws(x'7c', column1, column2) from t;statement okdrop table t++# Binary columns of each type, with a NULL+statement ok+create table binary_columns as+select+ arrow_cast(column1, 'Binary') as b,+ arrow_cast(column1, 'LargeBinary') as lb,+ arrow_cast(column1, 'BinaryView') as bv,+ column2 as sep+from (values+ (x'0102030405', x'7c'),+ (x'010203040506', NULL),+ (x'0102030405060708090a0b0c0d0e', x'2c2c'),+ (NULL, x'7c')+);++# NULLs are skipped, and the non-UTF-8 x'ff' is copied as is+query ?T+SELECT concat(b, x'ff', b), arrow_typeof(concat(b, x'ff', b)) FROM binary_columns;+----+0102030405ff0102030405 Binary+010203040506ff010203040506 Binary+0102030405060708090a0b0c0d0eff0102030405060708090a0b0c0d0e Binary+ff Binary++query ?T+SELECT concat(lb, x'ff', lb), arrow_typeof(concat(lb, x'ff', lb)) FROM binary_columns;+----+0102030405ff0102030405 LargeBinary+010203040506ff010203040506 LargeBinary+0102030405060708090a0b0c0d0eff0102030405060708090a0b0c0d0e LargeBinary+ff LargeBinary++# The second result is 13 bytes, one more than a BinaryView stores inline+query ?T+SELECT concat(bv, x'ff', bv), arrow_typeof(concat(bv, x'ff', bv)) FROM binary_columns;+----+0102030405ff0102030405 BinaryView+010203040506ff010203040506 BinaryView+0102030405060708090a0b0c0d0eff0102030405060708090a0b0c0d0e BinaryView+ff BinaryView++# A NULL separator makes the result NULL+query ?T+SELECT concat_ws(sep, b, x'ff', b), arrow_typeof(concat_ws(sep, b, x'ff', b)) FROM binary_columns;+----+01020304057cff7c0102030405 Binary+NULL Binary+0102030405060708090a0b0c0d0e2c2cff2c2c0102030405060708090a0b0c0d0e Binary+ff Binary++query ?T+SELECT concat_ws(sep, lb, x'ff', lb), arrow_typeof(concat_ws(sep, lb, x'ff', lb)) FROM binary_columns;+----+01020304057cff7c0102030405 LargeBinary+NULL LargeBinary+0102030405060708090a0b0c0d0e2c2cff2c2c0102030405060708090a0b0c0d0e LargeBinary+ff LargeBinary++query ?T+SELECT concat_ws(sep, bv, x'ff', bv), arrow_typeof(concat_ws(sep, bv, x'ff', bv)) FROM binary_columns;+----+01020304057cff7c0102030405 BinaryView+NULL BinaryView+0102030405060708090a0b0c0d0e2c2cff2c2c0102030405060708090a0b0c0d0e BinaryView+ff BinaryView++statement ok+drop table binary_columns