Apache DataFusion SQL Query Engine
chore: Add helpers for computing `OffsetBuffer` span, length (#25481)
Neil Conway24a7f13f5123
diff --git a/datafusion/common/src/hash_utils.rs b/datafusion/common/src/hash_utils.rsindex 1e6d96ecfd..8da165e8fb 100644--- a/datafusion/common/src/hash_utils.rs+++ b/datafusion/common/src/hash_utils.rs@@ -89,6 +89,8 @@ use crate::cast::{};use crate::error::Result;use crate::error::{_internal_datafusion_err, _internal_err};+#[cfg(not(feature = "force_hash_collisions"))]+use crate::utils::offset_span;use std::cell::RefCell;mod build_hasher;@@ -689,9 +691,7 @@ fn hash_map_array(let offsets = array.offsets();// Create hashes for each entry in each row- let first_offset = offsets.first() as usize;- let last_offset = offsets.last() as usize;- let entries_len = last_offset - first_offset;+ let (first_offset, entries_len) = offset_span(offsets);// Only hash the entries that are actually referencedlet mut values_hashes = vec![0u64; entries_len];@@ -739,15 +739,11 @@ fn hash_list_array<OffsetSize>(whereOffsetSize: OffsetSizeTrait,{- // In case values is sliced, hash only the bytes used by the offsets of this ListArray- let first_offset = array.value_offsets().first().copied().unwrap_or_default();- let last_offset = array.value_offsets().last().copied().unwrap_or_default();- let value_bytes_len = (last_offset - first_offset).as_usize();- let mut values_hashes = vec![0u64; value_bytes_len];+ // Hash only the child values referenced by this ListArray's offsets.+ let (first_offset, values_len) = offset_span(array.offsets());+ let mut values_hashes = vec![0u64; values_len];child_hashing.create_hashes(- [array- .values()- .slice(first_offset.as_usize(), value_bytes_len)],+ [array.values().slice(first_offset, values_len)],&mut values_hashes,)?;@@ -756,8 +752,8 @@ where{if array.is_valid(i) {let hash = &mut hashes_buffer[i];- for values_hash in &values_hashes[(*start - first_offset).as_usize()- ..(*stop - first_offset).as_usize()]+ for values_hash in &values_hashes+ [start.as_usize() - first_offset..stop.as_usize() - first_offset]{*hash = combine_hashes(*hash, *values_hash);}@@ -771,7 +767,7 @@ where.zip(hashes_buffer.iter_mut()){for values_hash in &values_hashes- [(*start - first_offset).as_usize()..(*stop - first_offset).as_usize()]+ [start.as_usize() - first_offset..stop.as_usize() - first_offset]{*hash = combine_hashes(*hash, *values_hash);}diff --git a/datafusion/common/src/utils/mod.rs b/datafusion/common/src/utils/mod.rsindex fba76f949f..bf87b9a788 100644--- a/datafusion/common/src/utils/mod.rs+++ b/datafusion/common/src/utils/mod.rs@@ -1227,6 +1227,36 @@ pub fn take_function_args<const N: usize, T>(})}+/// Returns the number of values covered by an offset buffer.+///+/// Slicing a variable-length array narrows its offsets but retains its entire+/// values buffer. Use this span to size output buffers: it counts child elements+/// for lists and bytes for strings/binary arrays, including values in null rows.+/// An empty array still has one offset and therefore a span of zero.+#[inline]+pub fn offset_span_len<O: ArrowNativeType>(offsets: &OffsetBuffer<O>) -> usize {+ offset_span(offsets).1+}++/// Returns the start and length of the values covered by an offset buffer.+///+/// The returned pair can be passed to [`Array::slice`] to select the visible+/// child values of a sliced list or map. For strings/binary arrays, the start+/// and length are measured in bytes. Values in null rows are included.+/// An empty array has a length of zero but may have a nonzero start.+///+/// ```+/// # use arrow::buffer::OffsetBuffer;+/// # use datafusion_common::utils::offset_span;+/// let offsets = OffsetBuffer::new(vec![100_i32, 103, 108].into());+/// assert_eq!(offset_span(&offsets), (100, 8));+/// ```+#[inline]+pub fn offset_span<O: ArrowNativeType>(offsets: &OffsetBuffer<O>) -> (usize, usize) {+ let start = offsets.first().as_usize();+ (start, offsets.last().as_usize() - start)+}+/// Returns the inner values of a list, or an error otherwise/// For [`ListArray`] and [`LargeListArray`], if it's sliced, it returns a/// sliced array too. Therefore, too reconstruct a list using it,@@ -1244,13 +1274,10 @@ pub fn list_values(array: &dyn Array) -> Result<ArrayRef> {fn sliced_list_values<O: OffsetSizeTrait>(list: &GenericListArray<O>) -> ArrayRef {let values = list.values();- let offsets = list.offsets();-- let first = offsets.first().as_usize();- let last = offsets.last().as_usize();+ let (start, len) = offset_span(list.offsets());- if first != 0 || last != values.len() {- return values.slice(first, last - first);+ if start != 0 || len != values.len() {+ return values.slice(start, len);}Arc::clone(values)@@ -1300,9 +1327,9 @@ fn truncate_list_nulls<O: OffsetSizeTrait>(if valid_or_empty.has_false() {let array_data = list.values().to_data();let offsets = list.offsets();- let capacity = offsets[offsets.len() - 1] - offsets[0];+ let capacity = offset_span_len(offsets);let mut mutable_array_data =- MutableArrayData::new(vec![&array_data], false, capacity.as_usize());+ MutableArrayData::new(vec![&array_data], false, capacity);let (valid_or_empty, _nulls) = valid_or_empty.into_parts();@@ -1508,6 +1535,19 @@ mod tests {#[cfg(feature = "sql")]use sqlparser::ast::Ident;+ #[test]+ fn test_offset_span() {+ let offsets = OffsetBuffer::new(vec![0_i32, 5, 8, 8, 12].into());+ assert_eq!(offset_span(&offsets), (0, 12));+ assert_eq!(offset_span(&offsets.slice(1, 2)), (5, 3));+ assert_eq!(offset_span_len(&offsets.slice(1, 2)), 3);+ assert_eq!(offset_span(&offsets.slice(2, 1)), (8, 0));+ assert_eq!(offset_span(&offsets.slice(4, 0)), (12, 0));+ assert_eq!(offset_span(&OffsetBuffer::<i64>::new_empty()), (0, 0));+ let large = OffsetBuffer::new(vec![i64::MAX - 10, i64::MAX].into());+ assert_eq!(offset_span(&large), ((i64::MAX - 10) as usize, 10));+ }+#[test]fn test_usize_wire_conversions() {assert_eq!(usize_from_wire(42_u64, "SomeExec", "fetch").unwrap(), 42);diff --git a/datafusion/functions-aggregate/src/array_agg.rs b/datafusion/functions-aggregate/src/array_agg.rsindex cc00e8f45b..546fad9738 100644--- a/datafusion/functions-aggregate/src/array_agg.rs+++ b/datafusion/functions-aggregate/src/array_agg.rs@@ -35,8 +35,8 @@ use arrow_select::dictionary::garbage_collect_any_dictionary;use datafusion_common::cast::as_list_array;use datafusion_common::hash_utils::{RandomState, create_hashes};use datafusion_common::scalar::copy_array_data;-use datafusion_common::utils::SingleRowListArrayBuilder;use datafusion_common::utils::proxy::HashTableAllocExt;+use datafusion_common::utils::{SingleRowListArrayBuilder, offset_span};use datafusion_common::{Result, ScalarValue, assert_eq_or_internal_err, exec_err, internal_err,};@@ -284,19 +284,15 @@ impl ArrayAggAccumulator {/// This function will return the underlying list array values if all valid values are consecutive without gaps (i.e. no null value point to a non-empty list)/// If there are gaps but only in the end of the list array, the function will return the values without the null values in the endfn get_optional_values_to_merge_as_is(list_array: &ListArray) -> Option<ArrayRef> {- let offsets = list_array.value_offsets();- // Offsets always have at least 1 value- let initial_offset = offsets[0];+ let offsets = list_array.offsets();let null_count = list_array.null_count();// If no nulls than just use the fast path// This is ok as the state is a ListArray rather than a ListViewArray so all the values are consecutiveif null_count == 0 {// According to Arrow specification, the first offset can be non-zero- let list_values = list_array.values().slice(- initial_offset as usize,- (offsets[offsets.len() - 1] - initial_offset) as usize,- );+ let (start, len) = offset_span(offsets);+ let list_values = list_array.values().slice(start, len);return Some(list_values);}diff --git a/datafusion/functions-nested/src/position.rs b/datafusion/functions-nested/src/position.rsindex 9ef8142fa4..9ed9c2651e 100644--- a/datafusion/functions-nested/src/position.rs+++ b/datafusion/functions-nested/src/position.rs@@ -40,6 +40,7 @@ use arrow::array::{use datafusion_common::cast::{as_generic_list_array, as_int64_array, as_large_list_array, as_list_array,};+use datafusion_common::utils::offset_span;use datafusion_common::{Result, exec_datafusion_err, exec_err, utils::take_function_args,};@@ -255,11 +256,8 @@ fn array_position_scalar<O: OffsetSizeTrait>(// ListArrays, values() returns the full underlying array but only// elements between the first and last offset are referenced.let offsets = haystack.offsets();- let first_offset = offsets[0].as_usize();- let last_offset = offsets[haystack.len()].as_usize();- let visible_values = haystack- .values()- .slice(first_offset, last_offset - first_offset);+ let (first_offset, values_len) = offset_span(offsets);+ let visible_values = haystack.values().slice(first_offset, values_len);// `not_distinct` treats NULL=NULL as true, matching the semantics of// `array_position`.@@ -271,9 +269,9 @@ fn array_position_scalar<O: OffsetSizeTrait>(// Match positions are relative to visible_values (0-based), so// subtract first_offset from each offset when comparing.- for i in 0..haystack.len() {- let start = offsets[i].as_usize() - first_offset;- let end = offsets[i + 1].as_usize() - first_offset;+ for (i, window) in offsets.windows(2).enumerate() {+ let start = window[0].as_usize() - first_offset;+ let end = window[1].as_usize() - first_offset;if validity.is_some_and(|v| v.is_null(i)) {// Null row -> null output; advance past matches in range@@ -554,11 +552,8 @@ fn array_positions_scalar<O: OffsetSizeTrait>(// ListArrays, values() returns the full underlying array but only// elements between the first and last offset are referenced.let offsets = haystack.offsets();- let first_offset = offsets[0].as_usize();- let last_offset = offsets[num_rows].as_usize();- let visible_values = haystack- .values()- .slice(first_offset, last_offset - first_offset);+ let (first_offset, values_len) = offset_span(offsets);+ let visible_values = haystack.values().slice(first_offset, values_len);// `not_distinct` treats NULL=NULL as true, matching the semantics of// `array_positions`.@@ -573,9 +568,9 @@ fn array_positions_scalar<O: OffsetSizeTrait>(// Match positions are relative to visible_values (0-based), so// subtract first_offset from each offset when comparing.- for i in 0..num_rows {- let start = offsets[i].as_usize() - first_offset;- let end = offsets[i + 1].as_usize() - first_offset;+ for (i, window) in offsets.windows(2).enumerate() {+ let start = window[0].as_usize() - first_offset;+ let end = window[1].as_usize() - first_offset;if validity.is_some_and(|v| v.is_null(i)) {// Null row -> null output; advance past matches in range.diff --git a/datafusion/functions-nested/src/remove.rs b/datafusion/functions-nested/src/remove.rsindex fa4127c17c..7470c60b85 100644--- a/datafusion/functions-nested/src/remove.rs+++ b/datafusion/functions-nested/src/remove.rs@@ -25,7 +25,7 @@ use arrow::array::{use arrow::buffer::OffsetBuffer;use arrow::datatypes::{DataType, FieldRef};use datafusion_common::cast::as_int64_array;-use datafusion_common::utils::ListCoercion;+use datafusion_common::utils::{ListCoercion, offset_span};use datafusion_common::{Result, ScalarValue, exec_err, internal_err, utils::take_function_args,};@@ -576,9 +576,7 @@ fn general_remove_with_scalar<OffsetSize: OffsetSizeTrait>(};let list_offsets = list_array.offsets();- let first_offset = list_offsets[0].to_usize().unwrap();- let last_offset = list_offsets[list_offsets.len() - 1].to_usize().unwrap();- let values_range_len = last_offset - first_offset;+ let (first_offset, values_range_len) = offset_span(list_offsets);let values_slice = list_array.values().slice(first_offset, values_range_len);let original_data = values_slice.to_data();let mut offsets = Vec::<OffsetSize>::with_capacity(list_array.len() + 1);diff --git a/datafusion/functions-nested/src/replace.rs b/datafusion/functions-nested/src/replace.rsindex 4bfd0c0dbe..af45058801 100644--- a/datafusion/functions-nested/src/replace.rs+++ b/datafusion/functions-nested/src/replace.rs@@ -24,7 +24,7 @@ use arrow::array::{use arrow::buffer::OffsetBuffer;use arrow::datatypes::{DataType, Field, FieldRef};use datafusion_common::cast::as_int64_array;-use datafusion_common::utils::ListCoercion;+use datafusion_common::utils::{ListCoercion, offset_span};use datafusion_common::{Result, ScalarValue, exec_err, internal_err, utils::take_function_args,};@@ -582,11 +582,8 @@ fn general_replace_with_scalar<O: OffsetSizeTrait>()?));}- let first_offset = list_array.offsets()[0].to_usize().unwrap();- let last_offset = list_array.offsets()[list_array.len()].to_usize().unwrap();- let visible_values = list_array- .values()- .slice(first_offset, last_offset - first_offset);+ let (first_offset, values_len) = offset_span(list_array.offsets());+ let visible_values = list_array.values().slice(first_offset, values_len);let to_array = scalar_to.to_array_of_size(1)?;let original_data = visible_values.to_data();diff --git a/datafusion/functions-nested/src/set_ops.rs b/datafusion/functions-nested/src/set_ops.rsindex 5cd4819cae..66cce98b09 100644--- a/datafusion/functions-nested/src/set_ops.rs+++ b/datafusion/functions-nested/src/set_ops.rs@@ -28,7 +28,7 @@ use arrow::datatypes::DataType::{LargeList, List, Null};use arrow::datatypes::{DataType, Field, FieldRef};use arrow::row::{RowConverter, SortField};use datafusion_common::cast::{as_large_list_array, as_list_array};-use datafusion_common::utils::{ListCoercion, normalize_float_zero};+use datafusion_common::utils::{ListCoercion, normalize_float_zero, offset_span};use datafusion_common::{Result, assert_eq_or_internal_err, exec_err, internal_err, utils::take_function_args,};@@ -332,8 +332,7 @@ impl Display for SetOp {pub(crate) fn normalize_visible_values<OffsetSize: OffsetSizeTrait>(array: &GenericListArray<OffsetSize>,) -> ArrayRef {- let first = array.offsets()[0].as_usize();- let len = array.offsets()[array.len()].as_usize() - first;+ let (first, len) = offset_span(array.offsets());if first == 0 && len == array.values().len() {normalize_float_zero(array.values())} else {diff --git a/datafusion/functions-nested/src/sort.rs b/datafusion/functions-nested/src/sort.rsindex fadc5da4b4..bccea76867 100644--- a/datafusion/functions-nested/src/sort.rs+++ b/datafusion/functions-nested/src/sort.rs@@ -28,7 +28,7 @@ use arrow::datatypes::{ArrowNativeTypeOp, DataType, FieldRef};use arrow::row::{RowConverter, Rows, SortField};use arrow::{compute, compute::SortOptions, downcast_primitive_array};use datafusion_common::cast::{as_large_list_array, as_list_array, as_string_array};-use datafusion_common::utils::ListCoercion;+use datafusion_common::utils::{ListCoercion, offset_span};use datafusion_common::{Result, exec_err};use datafusion_expr::{ArrayFunctionArgument, ArrayFunctionSignature, ColumnarValue, Documentation,@@ -292,15 +292,14 @@ fn sort_row<N: ArrowNativeTypeOp>(row: &mut [N], descending: bool) {/// values into a single `Vec` and sorts each row's slice in-place.fn sort_rows_no_nulls<N: ArrowNativeTypeOp, O: OffsetSizeTrait>(src_values: &[N],- offsets: &[O],+ offsets: &OffsetBuffer<O>,list_nulls: Option<&NullBuffer>,descending: bool,) -> Vec<N> {- let values_start = offsets[0].as_usize();- let values_end = offsets[offsets.len() - 1].as_usize();+ let (values_start, total_values) = offset_span(offsets);// Copy all values into a mutable buffer- let mut values = src_values[values_start..values_end].to_vec();+ let mut values = src_values[values_start..values_start + total_values].to_vec();for (row_index, window) in offsets.windows(2).enumerate() {if list_nulls.is_some_and(|n| n.is_null(row_index)) {@@ -318,13 +317,12 @@ fn sort_rows_no_nulls<N: ArrowNativeTypeOp, O: OffsetSizeTrait>(fn sort_rows_with_nulls<N: ArrowNativeTypeOp, O: OffsetSizeTrait>(src_values: &[N],src_nulls: &NullBuffer,- offsets: &[O],+ offsets: &OffsetBuffer<O>,list_nulls: Option<&NullBuffer>,descending: bool,nulls_first: bool,) -> (Vec<N>, NullBuffer) {- let values_start = offsets[0].as_usize();- let total_values = offsets[offsets.len() - 1].as_usize() - values_start;+ let (values_start, total_values) = offset_span(offsets);let mut out_values: Vec<N> = vec![N::default(); total_values];let mut validity = BooleanBufferBuilder::new(total_values);@@ -385,8 +383,7 @@ fn array_sort_non_primitive<OffsetSize: OffsetSizeTrait>(let row_count = list_array.len();let values = list_array.values();let offsets = list_array.offsets();- let values_start = offsets[0].as_usize();- let total_values = offsets[row_count].as_usize() - values_start;+ let (values_start, total_values) = offset_span(offsets);let converter = RowConverter::new(vec![SortField::new_with_options(values.data_type().clone(),diff --git a/datafusion/functions/src/core/overlay.rs b/datafusion/functions/src/core/overlay.rsindex c1f3353a8f..ae5bc989e8 100644--- a/datafusion/functions/src/core/overlay.rs+++ b/datafusion/functions/src/core/overlay.rs@@ -16,8 +16,7 @@// under the License.use arrow::array::{- Array, ArrayRef, GenericStringArray, Int64Array, OffsetSizeTrait, StringArrayType,- StringViewArray,+ Array, ArrayRef, Int64Array, OffsetSizeTrait, StringArrayType, StringViewArray,};use arrow::buffer::NullBuffer;use arrow::datatypes::DataType;@@ -29,6 +28,7 @@ use crate::utils::{make_scalar_function, utf8_to_str_type};use datafusion_common::cast::{as_generic_string_array, as_int64_array, as_string_view_array,};+use datafusion_common::utils::offset_span_len;use datafusion_common::{Result, exec_err};use datafusion_expr::{ColumnarValue, Documentation, TypeSignature, Volatility};use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl, Signature};@@ -236,8 +236,8 @@ fn overlay<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {} else {let string_array = as_generic_string_array::<T>(&args[0])?;let characters_array = as_generic_string_array::<T>(&args[1])?;- let data_capacity = visible_offset_bytes(string_array)- .saturating_add(visible_offset_bytes(characters_array));+ let data_capacity = offset_span_len(string_array.offsets())+ .saturating_add(offset_span_len(characters_array.offsets()));let builder = GenericStringArrayBuilder::<T>::with_capacity(string_array.len(),data_capacity,@@ -311,16 +311,6 @@ fn visible_view_bytes(array: &StringViewArray) -> usize {array.lengths().map(|l| l as usize).sum()}-/// Bytes referenced by the visible window of `array`, derived from the offset-/// buffer.-fn visible_offset_bytes<T: OffsetSizeTrait>(array: &GenericStringArray<T>) -> usize {- let offsets = array.value_offsets();- // `value_offsets()` always has `array.len() + 1` entries (≥1).- let first = offsets.first().copied().unwrap_or_default();- let last = offsets.last().copied().unwrap_or_default();- last.as_usize() - first.as_usize()-}-#[cfg(test)]mod tests {use std::sync::Arc;diff --git a/datafusion/functions/src/encoding/inner.rs b/datafusion/functions/src/encoding/inner.rsindex 8b57033fa0..43d75f3061 100644--- a/datafusion/functions/src/encoding/inner.rs+++ b/datafusion/functions/src/encoding/inner.rs@@ -35,7 +35,7 @@ use datafusion_common::{types::{NativeType, logical_string},utils::{hex::{HexCase, encode_bytes, encode_bytes_to_slice},- take_function_args,+ offset_span_len, take_function_args,},};use datafusion_expr::{@@ -272,19 +272,13 @@ fn decode_scalar(value: &ScalarValue, encoding: Encoding) -> Result<ColumnarValu}}-/// Estimate how many bytes are actually represented by the array; in case the-/// the array slices it's internal buffer, this returns the byte size of that slice-/// but not the byte size of the entire buffer.+/// Estimates the number of input bytes covered by the array's visible rows.///-/// This is an estimation only as it can estimate higher if null slots are non-zero-/// sized.+/// Sliced arrays exclude bytes outside their visible offset span. This can+/// overestimate the bytes processed because null rows may cover nonempty byte+/// ranges.fn estimate_byte_data_size<O: OffsetSizeTrait>(array: &GenericBinaryArray<O>) -> usize {- let offsets = array.value_offsets();- // Unwraps are safe as should always have 1 element in offset buffer- let start = *offsets.first().unwrap();- let end = *offsets.last().unwrap();- let data_size = end - start;- data_size.as_usize()+ offset_span_len(array.offsets())}fn decode_array(array: &ArrayRef, encoding: Encoding) -> Result<ColumnarValue> {@@ -544,7 +538,7 @@ mod tests {let size = estimate_byte_data_size(&array);assert_eq!(size, 15);- // Offsets starting at 0, but don't count entire data buffer size+ // Nonzero starting offset, including bytes covered by null slotslet array = BinaryArray::new(OffsetBuffer::new(vec![50, 51, 51, 60, 80, 81].into()),vec![0; 100].into(),diff --git a/datafusion/functions/src/regex/regexpreplace.rs b/datafusion/functions/src/regex/regexpreplace.rsindex 6ad138de15..647abfed59 100644--- a/datafusion/functions/src/regex/regexpreplace.rs+++ b/datafusion/functions/src/regex/regexpreplace.rs@@ -32,6 +32,7 @@ use datafusion_common::cast::{};use datafusion_common::exec_err;use datafusion_common::plan_err;+use datafusion_common::utils::offset_span_len;use datafusion_common::{Result, cast::as_generic_string_array, internal_err};use datafusion_expr::ColumnarValue;use datafusion_expr::TypeSignature;@@ -554,12 +555,8 @@ fn regexp_replace_static_pattern_replace<T: OffsetSizeTrait>(// We are going to create the underlying string buffer from its parts// to be able to re-use the existing null buffer for sparse arrays.- let mut vals = BufferBuilder::<u8>::new({- let offsets = string_array.value_offsets();- (offsets[string_array.len()] - offsets[0])- .to_usize()- .unwrap()- });+ let mut vals =+ BufferBuilder::<u8>::new(offset_span_len(string_array.offsets()));let mut new_offsets = BufferBuilder::<T>::new(string_array.len() + 1);new_offsets.append(T::zero());diff --git a/datafusion/functions/src/string/common.rs b/datafusion/functions/src/string/common.rsindex c13f724648..0bc63afa6a 100644--- a/datafusion/functions/src/string/common.rs+++ b/datafusion/functions/src/string/common.rs@@ -31,6 +31,7 @@ use arrow::buffer::{Buffer, NullBuffer, OffsetBuffer, ScalarBuffer};use arrow::datatypes::DataType;use datafusion_common::Result;use datafusion_common::cast::{as_generic_string_array, as_string_view_array};+use datafusion_common::utils::{offset_span, offset_span_len};use datafusion_common::{ScalarValue, exec_err};use datafusion_expr::ColumnarValue;@@ -282,11 +283,7 @@ whereF: for<'a> FnMut(usize, &'a str) -> &'a str,{let len = string_array.len();- let input_offsets = string_array.value_offsets();- let start = input_offsets.first().unwrap().as_usize();- let end = input_offsets.last().unwrap().as_usize();-- let mut values: Vec<u8> = Vec::with_capacity(end - start);+ let mut values: Vec<u8> = Vec::with_capacity(offset_span_len(string_array.offsets()));let mut offsets: Vec<T> = Vec::with_capacity(len + 1);offsets.push(T::usize_as(0));@@ -540,10 +537,7 @@ fn case_conversion_array<O: OffsetSizeTrait>(// Values contain non-ASCII.let item_len = string_array.len();- let offsets = string_array.value_offsets();- let start = offsets.first().unwrap().as_usize();- let end = offsets.last().unwrap().as_usize();- let capacity = (end - start) + PRE_ALLOC_BYTES;+ let capacity = offset_span_len(string_array.offsets()) + PRE_ALLOC_BYTES;// Null-preserving: reuse the input null buffer as the output null buffer.let nulls = string_array.nulls().cloned();let mut builder = GenericStringArrayBuilder::<O>::with_capacity(item_len, capacity);@@ -705,10 +699,8 @@ fn case_conversion_ascii_array<O: OffsetSizeTrait>(string_array: &GenericStringArray<O>,lower: bool,) -> Result<ArrayRef> {- let value_offsets = string_array.value_offsets();- let start = value_offsets.first().unwrap().as_usize();- let end = value_offsets.last().unwrap().as_usize();- let relevant = &string_array.value_data()[start..end];+ let (start, len) = offset_span(string_array.offsets());+ let relevant = &string_array.value_data()[start..start + len];let converted: Vec<u8> = if lower {relevant.iter().map(u8::to_ascii_lowercase).collect()diff --git a/datafusion/functions/src/unicode/initcap.rs b/datafusion/functions/src/unicode/initcap.rsindex d18aaa945e..d521ca0411 100644--- a/datafusion/functions/src/unicode/initcap.rs+++ b/datafusion/functions/src/unicode/initcap.rs@@ -24,6 +24,7 @@ use arrow::datatypes::DataType;use crate::strings::{GenericStringArrayBuilder, StringViewArrayBuilder};use datafusion_common::cast::{as_generic_string_array, as_string_view_array};use datafusion_common::types::logical_string;+use datafusion_common::utils::offset_span_len;use datafusion_common::{Result, ScalarValue, exec_err};use datafusion_expr::{Coercion, ColumnarValue, Documentation, EncodingPreservation, ScalarFunctionArgs,@@ -204,12 +205,10 @@ fn initcap_ascii_array<T: OffsetSizeTrait>() -> ArrayRef {let offsets = string_array.offsets();let src = string_array.value_data();- let first_offset = offsets.first().as_usize();- let last_offset = offsets.last().as_usize();// For sliced arrays, only convert the visible bytes, not the entire input// buffer.- let mut out = Vec::with_capacity(last_offset - first_offset);+ let mut out = Vec::with_capacity(offset_span_len(offsets));for window in offsets.windows(2) {let start = window[0].as_usize();diff --git a/datafusion/functions/src/unicode/substrindex.rs b/datafusion/functions/src/unicode/substrindex.rsindex 51cd615b92..614e5db32b 100644--- a/datafusion/functions/src/unicode/substrindex.rs+++ b/datafusion/functions/src/unicode/substrindex.rs@@ -18,8 +18,8 @@use std::sync::Arc;use arrow::array::{- Array, ArrayRef, AsArray, ByteView, GenericStringArray, OffsetSizeTrait,- PrimitiveArray, StringArrayType, StringViewArray, make_view, new_null_array,+ Array, ArrayRef, AsArray, ByteView, OffsetSizeTrait, PrimitiveArray, StringArrayType,+ StringViewArray, make_view, new_null_array,};use arrow::buffer::ScalarBuffer;use arrow::datatypes::{DataType, Int64Type};@@ -27,6 +27,7 @@ use arrow_buffer::NullBuffer;use crate::strings::GenericStringArrayBuilder;use crate::utils::make_scalar_function;+use datafusion_common::utils::offset_span_len;use datafusion_common::{Result, ScalarValue, exec_datafusion_err, exec_err, utils::take_function_args,};@@ -153,7 +154,7 @@ fn substr_index(args: &[ArrayRef]) -> Result<ArrayRef> {count_array,GenericStringArrayBuilder::<i32>::with_capacity(string_array.len(),- visible_string_bytes(string_array),+ offset_span_len(string_array.offsets()),),)}@@ -167,7 +168,7 @@ fn substr_index(args: &[ArrayRef]) -> Result<ArrayRef> {count_array,GenericStringArrayBuilder::<i64>::with_capacity(string_array.len(),- visible_string_bytes(string_array),+ offset_span_len(string_array.offsets()),),)}@@ -231,7 +232,7 @@ fn substr_index_scalar(count,GenericStringArrayBuilder::<i32>::with_capacity(arr.len(),- visible_string_bytes(arr),+ offset_span_len(arr.offsets()),),)}@@ -243,7 +244,7 @@ fn substr_index_scalar(count,GenericStringArrayBuilder::<i64>::with_capacity(arr.len(),- visible_string_bytes(arr),+ offset_span_len(arr.offsets()),),)}@@ -253,14 +254,6 @@ fn substr_index_scalar(Ok(ColumnarValue::Array(result))}-#[inline]-fn visible_string_bytes<T: OffsetSizeTrait>(- string_array: &GenericStringArray<T>,-) -> usize {- let offsets = string_array.value_offsets();- offsets[offsets.len() - 1].as_usize() - offsets[0].as_usize()-}-fn substr_index_general<'a, S, O>(string_array: S,delimiter_array: S,diff --git a/datafusion/spark/src/function/string/quote.rs b/datafusion/spark/src/function/string/quote.rsindex b4fd5e3827..8fafc42dfd 100644--- a/datafusion/spark/src/function/string/quote.rs+++ b/datafusion/spark/src/function/string/quote.rs@@ -19,7 +19,7 @@ use arrow::array::{ArrayRef, GenericStringBuilder, OffsetSizeTrait};use arrow::datatypes::DataType;use datafusion_common::cast::{as_generic_string_array, as_string_view_array};use datafusion_common::types::{NativeType, logical_string};-use datafusion_common::utils::take_function_args;+use datafusion_common::utils::{offset_span_len, take_function_args};use datafusion_common::{Result, exec_err};use datafusion_expr::{Coercion, ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature,@@ -91,12 +91,7 @@ fn spark_quote_inner(arg: &[ArrayRef]) -> Result<ArrayRef> {fn quote_array<T: OffsetSizeTrait>(array: &ArrayRef) -> Result<ArrayRef> {let str_array = as_generic_string_array::<T>(array)?;- // Slicing an array keeps the whole value buffer and narrows only the- // offsets, so measure the data through the offsets rather than through- // `value_data()`. An offset buffer holds one more entry than the array has- // rows, so it is never empty.- let offsets = str_array.value_offsets();- let data_len = offsets.last().unwrap().as_usize() - offsets[0].as_usize();+ let data_len = offset_span_len(str_array.offsets());Ok(quote_impl::<T, _>(str_array.iter(), data_len))}