Apache DataFusion SQL Query Engine
chore: `datafusion-spark` substring to support Binary types (#21979)
Oleks V2c7af1752953
diff --git a/datafusion/spark/src/function/string/substring.rs b/datafusion/spark/src/function/string/substring.rsindex 21b2242329..1c26564e03 100644--- a/datafusion/spark/src/function/string/substring.rs+++ b/datafusion/spark/src/function/string/substring.rs@@ -16,8 +16,9 @@// under the License.use arrow::array::{- Array, ArrayBuilder, ArrayRef, AsArray, GenericStringBuilder, Int64Array,- OffsetSizeTrait, StringArrayType, StringViewBuilder,+ Array, ArrayAccessor, ArrayBuilder, ArrayRef, AsArray, BinaryViewBuilder,+ GenericBinaryBuilder, GenericStringBuilder, Int64Array, OffsetSizeTrait,+ StringViewBuilder,};use arrow::datatypes::DataType;use datafusion_common::arrow::datatypes::{Field, FieldRef};@@ -56,6 +57,7 @@ impl Default for SparkSubstring {impl SparkSubstring {pub fn new() -> Self {let string = Coercion::new_exact(TypeSignatureClass::Native(logical_string()));+ let binary = Coercion::new_exact(TypeSignatureClass::Binary);let int64 = Coercion::new_implicit(TypeSignatureClass::Native(logical_int64()),vec![TypeSignatureClass::Native(logical_int32())],@@ -70,6 +72,12 @@ impl SparkSubstring {int64.clone(),int64.clone(),]),+ TypeSignature::Coercible(vec![binary.clone(), int64.clone()]),+ TypeSignature::Coercible(vec![+ binary.clone(),+ int64.clone(),+ int64.clone(),+ ]),],Volatility::Immutable,)@@ -128,26 +136,65 @@ fn spark_substring(args: &[ArrayRef]) -> Result<ArrayRef> {};match args[0].data_type() {- DataType::Utf8 => spark_substring_impl(- &args[0].as_string::<i32>(),+ DataType::Utf8 => {+ let array = args[0].as_string::<i32>();+ let is_ascii = enable_ascii_fast_path(&array, start_array, length_array);+ spark_substring_generic(+ &array,+ start_array,+ length_array,+ GenericStringBuilder::<i32>::new(),+ is_ascii,+ )+ }+ DataType::LargeUtf8 => {+ let array = args[0].as_string::<i64>();+ let is_ascii = enable_ascii_fast_path(&array, start_array, length_array);+ spark_substring_generic(+ &array,+ start_array,+ length_array,+ GenericStringBuilder::<i64>::new(),+ is_ascii,+ )+ }+ DataType::Utf8View => {+ let array = args[0].as_string_view();+ let is_ascii = enable_ascii_fast_path(&array, start_array, length_array);+ spark_substring_generic(+ &array,+ start_array,+ length_array,+ StringViewBuilder::new(),+ is_ascii,+ )+ }+ // Binary paths always use byte-level indexing, so `is_ascii` is irrelevant+ // and set to `true` (its value is ignored by the `[u8]` impl of+ // `SubstringItem`).+ DataType::Binary => spark_substring_generic(+ &args[0].as_binary::<i32>(),start_array,length_array,- GenericStringBuilder::<i32>::new(),+ GenericBinaryBuilder::<i32>::new(),+ true,),- DataType::LargeUtf8 => spark_substring_impl(- &args[0].as_string::<i64>(),+ DataType::LargeBinary => spark_substring_generic(+ &args[0].as_binary::<i64>(),start_array,length_array,- GenericStringBuilder::<i64>::new(),+ GenericBinaryBuilder::<i64>::new(),+ true,),- DataType::Utf8View => spark_substring_impl(- &args[0].as_string_view(),+ DataType::BinaryView => spark_substring_generic(+ &args[0].as_binary_view(),start_array,length_array,- StringViewBuilder::new(),+ BinaryViewBuilder::new(),+ true,),other => exec_err!(- "Unsupported data type {other:?} for function spark_substring, expected Utf8View, Utf8 or LargeUtf8."+ "Unsupported data type {other:?} for function spark_substring, expected Utf8View, Utf8, LargeUtf8, Binary, LargeBinary or BinaryView."),}}@@ -173,43 +220,156 @@ fn spark_start_to_datafusion_start(start: i64, len: usize) -> i64 {}}-trait StringArrayBuilder: ArrayBuilder {- fn append_value(&mut self, val: &str);+trait SubstringItem {+ /// Length used for Spark's negative-position adjustment.+ /// For `str` this is characters (or bytes in ASCII mode); for `[u8]` it is+ /// always byte count.+ fn positional_len(&self, is_ascii: bool) -> usize;++ /// Converts Spark's 1-indexed adjusted start + optional length into a+ /// byte range clamped to `[0, byte_len]`.+ fn byte_range(+ &self,+ adjusted_start: i64,+ len: Option<i64>,+ is_ascii: bool,+ ) -> Result<(usize, usize)>;++ fn byte_slice(&self, start: usize, end: usize) -> &Self;+}++impl SubstringItem for str {+ fn positional_len(&self, is_ascii: bool) -> usize {+ if is_ascii {+ self.len()+ } else {+ self.chars().count()+ }+ }++ fn byte_range(+ &self,+ adjusted_start: i64,+ len: Option<i64>,+ is_ascii: bool,+ ) -> Result<(usize, usize)> {+ get_true_start_end(self, adjusted_start, len, is_ascii)+ }++ fn byte_slice(&self, start: usize, end: usize) -> &Self {+ &self[start..end]+ }+}++impl SubstringItem for [u8] {+ fn positional_len(&self, _is_ascii: bool) -> usize {+ self.len()+ }++ fn byte_range(+ &self,+ adjusted_start: i64,+ len: Option<i64>,+ _is_ascii: bool,+ ) -> Result<(usize, usize)> {+ let byte_len = self.len();+ let start0 = adjusted_start.saturating_sub(1);+ let end0 = match len {+ Some(l) => start0.saturating_add(l),+ None => byte_len as i64,+ };+ let byte_len_i64 = byte_len as i64;+ Ok((+ start0.clamp(0, byte_len_i64) as usize,+ end0.clamp(0, byte_len_i64) as usize,+ ))+ }++ fn byte_slice(&self, start: usize, end: usize) -> &Self {+ &self[start..end]+ }+}++trait SubstringBuilder: ArrayBuilder {+ type Item: SubstringItem + ?Sized;+ fn append_value(&mut self, val: &Self::Item);fn append_null(&mut self);+ /// Spark's semantic "empty" for this builder's item type, used for the+ /// negative-length short-circuit.+ fn append_empty(&mut self);}-impl<O: OffsetSizeTrait> StringArrayBuilder for GenericStringBuilder<O> {+impl<O: OffsetSizeTrait> SubstringBuilder for GenericStringBuilder<O> {+ type Item = str;fn append_value(&mut self, val: &str) {GenericStringBuilder::append_value(self, val);}fn append_null(&mut self) {GenericStringBuilder::append_null(self);}+ fn append_empty(&mut self) {+ GenericStringBuilder::append_value(self, "");+ }}-impl StringArrayBuilder for StringViewBuilder {+impl SubstringBuilder for StringViewBuilder {+ type Item = str;fn append_value(&mut self, val: &str) {StringViewBuilder::append_value(self, val);}fn append_null(&mut self) {StringViewBuilder::append_null(self);}+ fn append_empty(&mut self) {+ StringViewBuilder::append_value(self, "");+ }}-fn spark_substring_impl<'a, V, B>(- string_array: &V,+impl<O: OffsetSizeTrait> SubstringBuilder for GenericBinaryBuilder<O> {+ type Item = [u8];+ fn append_value(&mut self, val: &[u8]) {+ GenericBinaryBuilder::append_value(self, val);+ }+ fn append_null(&mut self) {+ GenericBinaryBuilder::append_null(self);+ }+ fn append_empty(&mut self) {+ GenericBinaryBuilder::append_value(self, &[]);+ }+}++impl SubstringBuilder for BinaryViewBuilder {+ type Item = [u8];+ fn append_value(&mut self, val: &[u8]) {+ BinaryViewBuilder::append_value(self, val);+ }+ fn append_null(&mut self) {+ BinaryViewBuilder::append_null(self);+ }+ fn append_empty(&mut self) {+ BinaryViewBuilder::append_value(self, []);+ }+}++/// Unified implementation of Spark's `substring`, generic over the source+/// array (`StringArrayType`/`BinaryArrayType` via `ArrayAccessor`) and its+/// corresponding builder. Per-row indexing semantics are delegated to+/// [`SubstringItem`], which differs between `str` (char-aware when+/// `is_ascii` is false) and `[u8]` (always byte-level).+fn spark_substring_generic<'a, Source, Item, Builder>(+ array: &Source,start_array: &Int64Array,length_array: Option<&Int64Array>,- mut builder: B,+ mut builder: Builder,+ is_ascii: bool,) -> Result<ArrayRef>where- V: StringArrayType<'a>,- B: StringArrayBuilder,+ Source: ArrayAccessor<Item = &'a Item>,+ Item: SubstringItem + ?Sized + 'a,+ Builder: SubstringBuilder<Item = Item>,{- let is_ascii = enable_ascii_fast_path(string_array, start_array, length_array);-- for i in 0..string_array.len() {- if string_array.is_null(i) || start_array.is_null(i) {+ for i in 0..array.len() {+ if array.is_null(i) || start_array.is_null(i) {builder.append_null();continue;}@@ -221,30 +381,23 @@ wherecontinue;}- let string = string_array.value(i);+ let value = array.value(i);let start = start_array.value(i);let len_opt = length_array.map(|arr| arr.value(i));- // Spark: negative length returns empty string+ // Spark: negative length yields an empty valueif let Some(len) = len_opt&& len < 0{- builder.append_value("");+ builder.append_empty();continue;}- let string_len = if is_ascii {- string.len()- } else {- string.chars().count()- };-- let adjusted_start = spark_start_to_datafusion_start(start, string_len);-+ let positional_len = value.positional_len(is_ascii);+ let adjusted_start = spark_start_to_datafusion_start(start, positional_len);let (byte_start, byte_end) =- get_true_start_end(string, adjusted_start, len_opt, is_ascii)?;- let substr = &string[byte_start..byte_end];- builder.append_value(substr);+ value.byte_range(adjusted_start, len_opt, is_ascii)?;+ builder.append_value(value.byte_slice(byte_start, byte_end));}Ok(builder.finish())diff --git a/datafusion/sqllogictest/test_files/spark/string/substring.slt b/datafusion/sqllogictest/test_files/spark/string/substring.sltindex 9189ce50c2..1670de2c82 100644--- a/datafusion/sqllogictest/test_files/spark/string/substring.slt+++ b/datafusion/sqllogictest/test_files/spark/string/substring.slt@@ -207,4 +207,164 @@ SELECT substr(column1, -10, 3)FROM VALUES('abc'::string)-----(empty)\ No newline at end of file+(empty)++# ----------------------------------------------------------------------+# Binary / LargeBinary / BinaryView support+# ----------------------------------------------------------------------++# 'Spark SQL' bytes => 53 70 61 72 6b 20 53 51 4c+# byte 1..9 => 's' 'p' 'a' 'r' 'k' ' ' 'S' 'Q' 'L'++# Binary: basic substring with only pos+query ?+SELECT substring(arrow_cast('Spark SQL', 'Binary'), 5::int);+----+6b2053514c++# Binary: substring with pos and length+query ?+SELECT substring(arrow_cast('Spark SQL', 'Binary'), 3::int, 1::int);+----+61++# Binary: zero pos is treated as 1+query ?+SELECT substring(arrow_cast('Spark SQL', 'Binary'), 0::int);+----+537061726b2053514c++# Binary: negative pos counts from the end+query ?+SELECT substring(arrow_cast('Spark SQL', 'Binary'), -3::int);+----+53514c++query ?+SELECT substring(arrow_cast('Spark SQL', 'Binary'), -3::int, 2::int);+----+5351++# Binary: length exceeding input length+query ?+SELECT substring(arrow_cast('Spark SQL', 'Binary'), 2::int, 700::int);+----+7061726b2053514c++# Binary: pos beyond end returns empty binary+query I+SELECT character_length(substring(arrow_cast('Spark SQL', 'Binary'), 30::int));+----+0++# Binary: large negative pos returns the full value+query ?+SELECT substring(arrow_cast('Spark SQL', 'Binary'), -30::int);+----+537061726b2053514c++# Binary: negative length returns empty binary+query I+SELECT character_length(substring(arrow_cast('Spark SQL', 'Binary'), 3::int, -1::int));+----+0++# Binary: zero length returns empty binary+query I+SELECT character_length(substring(arrow_cast('Spark SQL', 'Binary'), 3::int, 0::int));+----+0++# Binary: NULL handling+query ?+SELECT substring(arrow_cast('Spark SQL', 'Binary'), NULL::int);+----+NULL++query ?+SELECT substring(arrow_cast(NULL, 'Binary'), 5::int);+----+NULL++query ?+SELECT substring(arrow_cast('Spark SQL', 'Binary'), NULL::int, 1::int);+----+NULL++query ?+SELECT substring(arrow_cast('Spark SQL', 'Binary'), 3::int, NULL::int);+----+NULL++# Binary: return type matches input type+query ?T+SELECT substring(arrow_cast('Spark SQL', 'Binary'), 3::int, 2::int),+ arrow_typeof(substring(arrow_cast('Spark SQL', 'Binary'), 3::int, 2::int));+----+6172 Binary++# LargeBinary: return type is preserved+query ?T+SELECT substring(arrow_cast('Spark SQL', 'LargeBinary'), 3::int, 2::int),+ arrow_typeof(substring(arrow_cast('Spark SQL', 'LargeBinary'), 3::int, 2::int));+----+6172 LargeBinary++# BinaryView: return type is preserved+query ?T+SELECT substring(arrow_cast('Spark SQL', 'BinaryView'), 3::int, 2::int),+ arrow_typeof(substring(arrow_cast('Spark SQL', 'BinaryView'), 3::int, 2::int));+----+6172 BinaryView++# Binary with non-UTF-8 bytes: byte-level slicing is preserved+# X'ff00aabb1122' has 6 bytes and is typed as Binary by default+query ?+SELECT substring(X'ff00aabb1122', 2::int, 3::int);+----+00aabb++query ?+SELECT substring(X'ff00aabb1122', -2::int, 2::int);+----+1122++# substr alias works with binary input+query ?+SELECT substr(arrow_cast('Spark SQL', 'Binary'), 5::int);+----+6b2053514c++query ?+SELECT substr(arrow_cast('Spark SQL', 'LargeBinary'), 3::int, 1::int);+----+61++# Array input (Binary column) produces per-row byte-level substrings.+# The second column is the byte length of the result so empty-binary+# rows remain unambiguous in the expected output.+query ?I+SELECT substring(column1, column2, column3),+ character_length(substring(column1, column2, column3))+FROM VALUES+(arrow_cast('Spark SQL', 'Binary'), -3::int, 2::int),+(arrow_cast('Spark SQL', 'Binary'), 3::int, 1::int),+(arrow_cast('Spark SQL', 'Binary'), 3::int, 700::int),+(arrow_cast('Spark SQL', 'Binary'), 3::int, -1::int),+(arrow_cast('Spark SQL', 'Binary'), 3::int, 0::int),+(arrow_cast('Spark SQL', 'Binary'), 300::int, 3::int),+(arrow_cast('Spark SQL', 'Binary'), -300::int, 3::int),+(arrow_cast(NULL, 'Binary'), 3::int, 1::int),+(arrow_cast('Spark SQL', 'Binary'), NULL::int, 1::int),+(arrow_cast('Spark SQL', 'Binary'), 3::int, NULL::int);+----+5351 2+61 1+61726b2053514c 7+ 0+ 0+ 0+ 0+NULL NULL+NULL NULL+NULL NULL