Apache DataFusion SQL Query Engine
feat: support binary arguments for StringConcat operator (#21883)
theirixea0928c19ccf
diff --git a/datafusion/expr-common/src/type_coercion/binary.rs b/datafusion/expr-common/src/type_coercion/binary.rsindex 02bfdd9b34..aec87ec5ff 100644--- a/datafusion/expr-common/src/type_coercion/binary.rs+++ b/datafusion/expr-common/src/type_coercion/binary.rs@@ -1611,9 +1611,37 @@ fn ree_coercion(/// This is a union of string coercion rules and specified rules:/// 1. At least one side of lhs and rhs should be string type (Utf8 / LargeUtf8)/// 2. Data type of the other side should be able to cast to string type+/// 3. Binary and string types cannot be mixedfn string_concat_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataType> {use arrow::datatypes::DataType::*;string_coercion(lhs_type, rhs_type).or_else(|| match (lhs_type, rhs_type) {+ // Allow pure binary + binary+ (+ Binary | LargeBinary | BinaryView | FixedSizeBinary(_),+ Binary | LargeBinary | BinaryView | FixedSizeBinary(_),+ ) => {+ // Coerce fixed-sized binary to variable-sized `Binary` to make uniform signature+ // with the `Binary` result+ let lhs_type = match lhs_type {+ FixedSizeBinary(_) => &Binary,+ val => val,+ };+ let rhs_type = match rhs_type {+ FixedSizeBinary(_) => &Binary,+ val => val,+ };+ binary_coercion(lhs_type, rhs_type)+ }+ // Deny other mixed binary + string combinations+ (+ Binary | LargeBinary | BinaryView | FixedSizeBinary(_),+ Utf8 | LargeUtf8 | Utf8View,+ ) => None,+ (+ Utf8 | LargeUtf8 | Utf8View,+ Binary | LargeBinary | BinaryView | FixedSizeBinary(_),+ ) => None,+ // Predicate-based coercion rules are following(Utf8View, from_type) | (from_type, Utf8View) => {string_concat_internal_coercion(from_type, &Utf8View)}@@ -1626,7 +1654,6 @@ fn string_concat_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<Da(Dictionary(_, lhs_value_type), Dictionary(_, rhs_value_type)) => {string_coercion(lhs_value_type, rhs_value_type).or(None)}- (Binary, Binary) => Some(Utf8),_ => None,})}@@ -1756,13 +1783,17 @@ pub fn binary_to_string_coercion(fn binary_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataType> {use arrow::datatypes::DataType::*;match (lhs_type, rhs_type) {+ // Prefer symmetric coercion (in case the function is called directly)+ (Binary, Binary) => Some(Binary),+ (LargeBinary, LargeBinary) => Some(LargeBinary),+ (BinaryView, BinaryView) => Some(BinaryView),// If BinaryView is in any side, we coerce to BinaryView.- (BinaryView, BinaryView | Binary | LargeBinary | Utf8 | LargeUtf8 | Utf8View)+ (BinaryView, Binary | LargeBinary | Utf8 | LargeUtf8 | Utf8View)| (LargeBinary | Binary | Utf8 | LargeUtf8 | Utf8View, BinaryView) => {Some(BinaryView)}// Prefer LargeBinary over Binary- (LargeBinary | Binary | Utf8 | LargeUtf8 | Utf8View, LargeBinary)+ (Binary | Utf8 | LargeUtf8 | Utf8View, LargeBinary)| (LargeBinary, Binary | Utf8 | LargeUtf8 | Utf8View) => Some(LargeBinary),// If Utf8View/LargeUtf8 presents need to be large Binarydiff --git a/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rs b/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rsindex 317b022238..f8bff3ca90 100644--- a/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rs+++ b/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rs@@ -897,3 +897,109 @@ fn test_binary_comparison_string_numeric_coercion() -> Result<()> {}Ok(())}++#[test]+fn test_string_concat_coercion() -> Result<()> {+ // Binary+ test_coercion_binary_rule!(+ DataType::Binary,+ DataType::Binary,+ Operator::StringConcat,+ DataType::Binary+ );+ test_coercion_binary_rule!(+ DataType::LargeBinary,+ DataType::LargeBinary,+ Operator::StringConcat,+ DataType::LargeBinary+ );+ test_coercion_binary_rule!(+ DataType::BinaryView,+ DataType::BinaryView,+ Operator::StringConcat,+ DataType::BinaryView+ );+ test_coercion_binary_rule!(+ DataType::Binary,+ DataType::LargeBinary,+ Operator::StringConcat,+ DataType::LargeBinary+ );+ test_coercion_binary_rule!(+ DataType::BinaryView,+ DataType::Binary,+ Operator::StringConcat,+ DataType::BinaryView+ );+ test_coercion_binary_rule!(+ DataType::FixedSizeBinary(4),+ DataType::FixedSizeBinary(16),+ Operator::StringConcat,+ DataType::Binary+ );+ test_coercion_binary_rule!(+ DataType::FixedSizeBinary(4),+ DataType::LargeBinary,+ Operator::StringConcat,+ DataType::LargeBinary+ );+ test_coercion_binary_rule!(+ DataType::FixedSizeBinary(4),+ DataType::BinaryView,+ Operator::StringConcat,+ DataType::BinaryView+ );++ // String+ test_coercion_binary_rule!(+ DataType::Utf8,+ DataType::Utf8,+ Operator::StringConcat,+ DataType::Utf8+ );+ test_coercion_binary_rule!(+ DataType::LargeUtf8,+ DataType::LargeUtf8,+ Operator::StringConcat,+ DataType::LargeUtf8+ );+ test_coercion_binary_rule!(+ DataType::Utf8View,+ DataType::Utf8View,+ Operator::StringConcat,+ DataType::Utf8View+ );++ // Mixed string-binary+ for string_dt in [DataType::Utf8, DataType::LargeUtf8, DataType::Utf8View] {+ for binary_dt in [+ DataType::Binary,+ DataType::LargeBinary,+ DataType::BinaryView,+ DataType::FixedSizeBinary(8),+ ] {+ assert!(+ BinaryTypeCoercer::new(&binary_dt, &Operator::StringConcat, &string_dt,)+ .get_input_types()+ .is_err(),+ "{binary_dt} || {string_dt}"+ );+ assert!(+ BinaryTypeCoercer::new(&string_dt, &Operator::StringConcat, &binary_dt,)+ .get_input_types()+ .is_err(),+ "{string_dt} || {binary_dt}"+ );+ }+ }++ // Mixed string-other+ test_coercion_binary_rule!(+ DataType::Utf8,+ DataType::Timestamp(Second, None),+ Operator::StringConcat,+ DataType::Utf8+ );++ Ok(())+}diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rsindex ae93b1d03a..396c0b87c4 100644--- a/datafusion/physical-expr/src/expressions/binary.rs+++ b/datafusion/physical-expr/src/expressions/binary.rs@@ -24,7 +24,9 @@ use std::sync::Arc;use arrow::array::*;use arrow::compute::kernels::boolean::{and_kleene, or_kleene};-use arrow::compute::kernels::concat_elements::concat_elements_utf8;+use arrow::compute::kernels::concat_elements::{+ concat_element_binary, concat_elements_utf8,+};use arrow::compute::{SlicesIterator, cast, filter_record_batch};use arrow::datatypes::*;use arrow::error::ArrowError;@@ -46,7 +48,8 @@ use kernels::{bitwise_and_dyn, bitwise_and_dyn_scalar, bitwise_or_dyn, bitwise_or_dyn_scalar,bitwise_shift_left_dyn, bitwise_shift_left_dyn_scalar, bitwise_shift_right_dyn,bitwise_shift_right_dyn_scalar, bitwise_xor_dyn, bitwise_xor_dyn_scalar,- concat_elements_utf8view, regex_match_dyn, regex_match_dyn_scalar,+ concat_elements_binary_view_array, concat_elements_utf8view, regex_match_dyn,+ regex_match_dyn_scalar,};/// Binary expression@@ -928,18 +931,6 @@ fn pre_selection_scatter(}fn concat_elements(left: &ArrayRef, right: &ArrayRef) -> Result<ArrayRef> {- if *left.data_type() == DataType::Binary && *right.data_type() == DataType::Binary {- // Cast Binary to Utf8 to validate UTF-8 encoding before concatenation- // Follow widespread approach of PostgreSQL, sqlite, DuckDB, Snowflake- // Spark does it in a different way by making a binary-to-binary concatenation- let left = cast(left.as_ref(), &DataType::Utf8)?;- let right = cast(right.as_ref(), &DataType::Utf8)?;- return Ok(Arc::new(concat_elements_utf8(- left.as_string::<i32>(),- right.as_string::<i32>(),- )?));- }-Ok(match left.data_type() {DataType::Utf8 => Arc::new(concat_elements_utf8(left.as_string::<i32>(),@@ -953,6 +944,18 @@ fn concat_elements(left: &ArrayRef, right: &ArrayRef) -> Result<ArrayRef> {left.as_string_view(),right.as_string_view(),)?),+ DataType::Binary => Arc::new(concat_element_binary::<i32>(+ left.as_binary(),+ right.as_binary(),+ )?),+ DataType::LargeBinary => Arc::new(concat_element_binary::<i64>(+ left.as_binary(),+ right.as_binary(),+ )?),+ DataType::BinaryView => Arc::new(concat_elements_binary_view_array(+ left.as_binary_view(),+ right.as_binary_view(),+ )?),other => {return internal_err!("Data type {other:?} not supported for binary operation 'concat_elements' on string arrays"diff --git a/datafusion/physical-expr/src/expressions/binary/kernels.rs b/datafusion/physical-expr/src/expressions/binary/kernels.rsindex 9818861c2f..e573d7ece2 100644--- a/datafusion/physical-expr/src/expressions/binary/kernels.rs+++ b/datafusion/physical-expr/src/expressions/binary/kernels.rs@@ -18,7 +18,7 @@//! This module contains computation kernels that are specific to//! datafusion and not (yet) targeted to port upstream to arrowuse arrow::array::*;-use arrow::buffer::NullBuffer;+use arrow::buffer::{MutableBuffer, NullBuffer};use arrow::compute::kernels::bitwise::{bitwise_and, bitwise_and_scalar, bitwise_or, bitwise_or_scalar, bitwise_shift_left,bitwise_shift_left_scalar, bitwise_shift_right, bitwise_shift_right_scalar,@@ -161,11 +161,11 @@ create_left_integral_dyn_scalar_kernel!(bitwise_shift_left_scalar);-/// Concatenates two `StringViewArray`s element-wise.+/// Concatenates two `StringViewArray`s element-wise./// If either element is `Null`, the result element is also `Null`.////// # Errors-/// - Returns an error if the input arrays have different lengths.+/// - Returns an error if the input arrays have different lengths./// - Returns an error if any concatenated string exceeds `u32::MAX` (≈4 GB) in length.pub fn concat_elements_utf8view(left: &StringViewArray,@@ -204,6 +204,50 @@ pub fn concat_elements_utf8view(Ok(result.finish())}+/// Concatenates two `BinaryViewArray`s element-wise.+/// If either element is `Null`, the result element is also `Null`.+///+/// # Errors+/// - Returns an error if the input arrays have different lengths.+/// - Returns an error if any concatenated string exceeds `u32::MAX` in length.+pub fn concat_elements_binary_view_array(+ left: &BinaryViewArray,+ right: &BinaryViewArray,+) -> std::result::Result<BinaryViewArray, ArrowError> {+ if left.len() != right.len() {+ return Err(ArrowError::ComputeError(format!(+ "Arrays must have the same length: {} != {}",+ left.len(),+ right.len()+ )));+ }+ let mut result = BinaryViewBuilder::with_capacity(left.len());++ // Avoid reallocations by writing to a reused buffer (note we could be even+ // more efficient by creating the view directly here and avoid the buffer+ // but that would be more complex)+ let mut buffer = MutableBuffer::new(0);++ // Pre-compute combined null bitmap, so the per-row NULL check is more+ // efficient+ let nulls = NullBuffer::union(left.nulls(), right.nulls());++ for i in 0..left.len() {+ if nulls.as_ref().is_some_and(|n| n.is_null(i)) {+ result.append_null();+ } else {+ let l = left.value(i);+ let r = right.value(i);+ buffer.clear();+ buffer.extend_from_slice(l);+ buffer.extend_from_slice(r);+ // No try-version of append_value+ result.try_append_value(&buffer)?;+ }+ }+ Ok(result.finish())+}+/// Invoke a compute kernel on a pair of binary data arrays with flagsmacro_rules! regexp_is_match_flag {($LEFT:expr, $RIGHT:expr, $ARRAYTYPE:ident, $NOT:expr, $FLAG:expr) => {{diff --git a/datafusion/sqllogictest/test_files/binary.slt b/datafusion/sqllogictest/test_files/binary.sltindex 54ac51d9e7..a57c31547f 100644--- a/datafusion/sqllogictest/test_files/binary.slt+++ b/datafusion/sqllogictest/test_files/binary.slt@@ -321,3 +321,40 @@ query TSELECT split_part(CAST(binary AS VARCHAR), 'o', 2) FROM t WHERE binary = X'466f6f';----(empty)++# Pipe concatenation of binaries always provides a binary+query ?+SELECT x'636166c3a9' || x'68656c6c6f';+----+636166c3a968656c6c6f++# Pipe concatenation of binary and other kind of binary also provides a binary+query ?+SELECT x'636166c3a9' || arrow_cast(x'68656c6c6f', 'LargeBinary');+----+636166c3a968656c6c6f++query ?+SELECT x'636166c3a9' || arrow_cast(arrow_cast(x'68656c6c6f', 'LargeBinary'), 'BinaryView');+----+636166c3a968656c6c6f++query ?T+SELECT x'636166c3a9' || arrow_cast(x'68656c6c6f', 'FixedSizeBinary(5)'), arrow_typeof(x'636166c3a9' || arrow_cast(x'68656c6c6f', 'FixedSizeBinary(5)'));+----+636166c3a968656c6c6f Binary++query ?T+SELECT arrow_cast(x'6361', 'FixedSizeBinary(2)') || arrow_cast(x'68656c6c6f', 'FixedSizeBinary(5)'), arrow_typeof(arrow_cast(x'6361', 'FixedSizeBinary(2)') || arrow_cast(x'68656c6c6f', 'FixedSizeBinary(5)'));+----+636168656c6c6f Binary++# Byte pipe operator is forbidden for mixed binary and text+query error DataFusion error: Error during planning: Cannot infer common string type for string concat operation Binary || Utf8+SELECT x'c3a9' || 'hello';++query error DataFusion error: Error during planning: Cannot infer common string type for string concat operation Utf8 || LargeBinary+SELECT 'hello' || arrow_cast(arrow_cast('hello', 'Binary'), 'LargeBinary');++query error DataFusion error: Error during planning: Cannot infer common string type for string concat operation Utf8 || BinaryView+SELECT 'hello' || arrow_cast(arrow_cast('hello', 'Binary'), 'BinaryView');diff --git a/datafusion/sqllogictest/test_files/scalar.slt b/datafusion/sqllogictest/test_files/scalar.sltindex aac6abfe2f..89ae30e3c0 100644--- a/datafusion/sqllogictest/test_files/scalar.slt+++ b/datafusion/sqllogictest/test_files/scalar.slt@@ -1739,23 +1739,11 @@ SELECT 'a' || 42 || 23.3----a4223.3-# concat of binary and text provides a text output-query T-select arrow_cast('Café', 'Utf8') || arrow_cast('Foobar', 'Binary');------CaféFoobar--query T-select arrow_cast('Café', 'Binary') || arrow_cast('Foobar', 'Utf8');------CaféFoobar--# Concat of two binaries should cast arguments to text and produce a text output,-# following common behaviour of PostreSQL. However, Spark is providing binary-query T+# Concat operator of two binaries uses their binary representation without text at all+query ?select arrow_cast('Café', 'Binary') || arrow_cast('Foobar', 'Binary');-----CaféFoobar+436166c3a9466f6f626172# test_not_expressions()