Apache DataFusion SQL Query Engine
chore: use `Vec` instead of `OffsetBuilder` (#23195)
Oleks V32e27acb9f64
diff --git a/datafusion/functions-nested/src/array_add.rs b/datafusion/functions-nested/src/array_add.rsindex c6edf67bf5..dd170911fb 100644--- a/datafusion/functions-nested/src/array_add.rs+++ b/datafusion/functions-nested/src/array_add.rs@@ -19,10 +19,9 @@use crate::utils::{coerce_array_math_arg_types, make_scalar_function};use arrow::array::{- Array, ArrayRef, Float64Array, GenericListArray, NullBufferBuilder,- OffsetBufferBuilder, OffsetSizeTrait,+ Array, ArrayRef, Float64Array, GenericListArray, NullBufferBuilder, OffsetSizeTrait,};-use arrow::buffer::NullBuffer;+use arrow::buffer::{NullBuffer, OffsetBuffer};use arrow::datatypes::{DataType,DataType::{LargeList, List},@@ -147,12 +146,13 @@ fn general_array_add<O: OffsetSizeTrait>(let mut out_values: Vec<f64> = Vec::with_capacity(lhs_values.len());let mut out_inner_nulls = NullBufferBuilder::new(lhs_values.len());- let mut out_offsets = OffsetBufferBuilder::<O>::new(lhs.len());+ let mut out_offsets = Vec::<O>::with_capacity(lhs.len() + 1);+ out_offsets.push(O::zero());for row in 0..lhs.len() {// Whole-row NULL on either side -> NULL output row, no elements.if row_nulls.as_ref().is_some_and(|nb| nb.is_null(row)) {- out_offsets.push_length(0);+ out_offsets.push(out_offsets[row]);continue;}@@ -185,7 +185,7 @@ fn general_array_add<O: OffsetSizeTrait>(None => out_inner_nulls.append_n_non_nulls(len1),}- out_offsets.push_length(len1);+ out_offsets.push(out_offsets[row] + O::usize_as(len1));}let values_array = Arc::new(Float64Array::new(@@ -196,7 +196,7 @@ fn general_array_add<O: OffsetSizeTrait>(Ok(Arc::new(GenericListArray::<O>::try_new(field,- out_offsets.finish(),+ OffsetBuffer::new(out_offsets.into()),values_array,row_nulls,)?))diff --git a/datafusion/functions-nested/src/array_filter.rs b/datafusion/functions-nested/src/array_filter.rsindex a1fa8268a3..7dd7230ae9 100644--- a/datafusion/functions-nested/src/array_filter.rs+++ b/datafusion/functions-nested/src/array_filter.rs@@ -20,7 +20,7 @@use arrow::{array::{Array, ArrayRef, AsArray, BooleanArray, LargeListArray, ListArray,- OffsetBufferBuilder, OffsetSizeTrait, new_empty_array,+ OffsetSizeTrait, new_empty_array,},buffer::{OffsetBuffer, ScalarBuffer},compute::{filter as arrow_filter, take_arrays},@@ -252,13 +252,11 @@ fn filter_list_values<O: OffsetSizeTrait>(offsets: &OffsetBuffer<O>,) -> Result<(ArrayRef, OffsetBuffer<O>)> {let num_sublists = offsets.len().saturating_sub(1);- let mut builder = OffsetBufferBuilder::<O>::new(num_sublists);-let has_nulls = predicate.null_count() > 0;- for i in 0..num_sublists {+ let new_offsets = OffsetBuffer::<O>::from_lengths((0..num_sublists).map(|i| {let start = offsets[i].as_usize();let end = offsets[i + 1].as_usize();- let count = if has_nulls {+ if has_nulls {(start..end).filter(|&j| predicate.is_valid(j) && predicate.value(j)).count()@@ -267,11 +265,8 @@ fn filter_list_values<O: OffsetSizeTrait>(.values().slice(start, end - start).count_set_bits()- };- builder.push_length(count);- }-- let new_offsets = builder.finish();+ }+ }));if new_offsets.last() == offsets.last() {return Ok((Arc::clone(values), offsets.clone()));diff --git a/datafusion/functions-nested/src/array_normalize.rs b/datafusion/functions-nested/src/array_normalize.rsindex 0ff7674032..f7da07e5f6 100644--- a/datafusion/functions-nested/src/array_normalize.rs+++ b/datafusion/functions-nested/src/array_normalize.rs@@ -19,9 +19,9 @@use crate::utils::make_scalar_function;use arrow::array::{- Array, ArrayRef, Float64Array, GenericListArray, NullBufferBuilder,- OffsetBufferBuilder, OffsetSizeTrait,+ Array, ArrayRef, Float64Array, GenericListArray, NullBufferBuilder, OffsetSizeTrait,};+use arrow::buffer::OffsetBuffer;use arrow::datatypes::{DataType,DataType::{FixedSizeList, LargeList, List, Null},@@ -144,13 +144,14 @@ fn general_array_normalize<O: OffsetSizeTrait>(arrays: &[ArrayRef]) -> Result<Arlet offsets = list_array.value_offsets();let mut new_values: Vec<f64> = Vec::with_capacity(values.len());- let mut new_offsets = OffsetBufferBuilder::<O>::new(list_array.len());+ let mut new_offsets = Vec::<O>::with_capacity(list_array.len() + 1);+ new_offsets.push(O::zero());let mut nulls = NullBufferBuilder::new(list_array.len());for row in 0..list_array.len() {if list_array.is_null(row) {nulls.append_null();- new_offsets.push_length(0);+ new_offsets.push(new_offsets[row]);continue;}@@ -161,7 +162,7 @@ fn general_array_normalize<O: OffsetSizeTrait>(arrays: &[ArrayRef]) -> Result<Arlet slice = values.slice(start, len);if slice.null_count() != 0 {nulls.append_null();- new_offsets.push_length(0);+ new_offsets.push(new_offsets[row]);continue;}@@ -170,7 +171,7 @@ fn general_array_normalize<O: OffsetSizeTrait>(arrays: &[ArrayRef]) -> Result<Ar// Empty array: return empty array (no normalization needed, no division by zero risk)if len == 0 {nulls.append_non_null();- new_offsets.push_length(0);+ new_offsets.push(new_offsets[row]);continue;}@@ -183,7 +184,7 @@ fn general_array_normalize<O: OffsetSizeTrait>(arrays: &[ArrayRef]) -> Result<Ar// Zero magnitude: undefined normalization. Emit NULL row.if sq_sum == 0.0 {nulls.append_null();- new_offsets.push_length(0);+ new_offsets.push(new_offsets[row]);continue;}@@ -192,7 +193,7 @@ fn general_array_normalize<O: OffsetSizeTrait>(arrays: &[ArrayRef]) -> Result<Arnew_values.push(vals[i] / mag);}nulls.append_non_null();- new_offsets.push_length(len);+ new_offsets.push(new_offsets[row] + O::usize_as(len));}let values_array = Arc::new(Float64Array::from(new_values));@@ -200,7 +201,7 @@ fn general_array_normalize<O: OffsetSizeTrait>(arrays: &[ArrayRef]) -> Result<ArOk(Arc::new(GenericListArray::<O>::try_new(field,- new_offsets.finish(),+ OffsetBuffer::new(new_offsets.into()),values_array,nulls.finish(),)?))diff --git a/datafusion/functions-nested/src/array_scale.rs b/datafusion/functions-nested/src/array_scale.rsindex 24750ade8a..a6be910d20 100644--- a/datafusion/functions-nested/src/array_scale.rs+++ b/datafusion/functions-nested/src/array_scale.rs@@ -18,10 +18,8 @@//! [`ScalarUDFImpl`] definitions for array_scale function.use crate::utils::make_scalar_function;-use arrow::array::{- Array, ArrayRef, Float64Array, GenericListArray, OffsetBufferBuilder, OffsetSizeTrait,-};-use arrow::buffer::NullBuffer;+use arrow::array::{Array, ArrayRef, Float64Array, GenericListArray, OffsetSizeTrait};+use arrow::buffer::{NullBuffer, OffsetBuffer};use arrow::datatypes::{DataType,DataType::{FixedSizeList, LargeList, List, Null},@@ -172,11 +170,12 @@ fn general_array_scale<O: OffsetSizeTrait>(let row_nulls = NullBuffer::union(list_array.nulls(), scalar_array.nulls());let mut value_builder = Float64Array::builder(values.len());- let mut new_offsets = OffsetBufferBuilder::<O>::new(list_array.len());+ let mut new_offsets = Vec::<O>::with_capacity(list_array.len() + 1);+ new_offsets.push(O::zero());for row in 0..list_array.len() {if row_nulls.as_ref().is_some_and(|nb| nb.is_null(row)) {- new_offsets.push_length(0);+ new_offsets.push(new_offsets[row]);continue;}@@ -196,7 +195,7 @@ fn general_array_scale<O: OffsetSizeTrait>(}}- new_offsets.push_length(len);+ new_offsets.push(new_offsets[row] + O::usize_as(len));}let values_array = Arc::new(value_builder.finish());@@ -213,7 +212,7 @@ fn general_array_scale<O: OffsetSizeTrait>(Ok(Arc::new(GenericListArray::<O>::try_new(field,- new_offsets.finish(),+ OffsetBuffer::new(new_offsets.into()),values_array,row_nulls,)?))diff --git a/datafusion/functions-nested/src/remove.rs b/datafusion/functions-nested/src/remove.rsindex 9d7dd4f44d..111147659a 100644--- a/datafusion/functions-nested/src/remove.rs+++ b/datafusion/functions-nested/src/remove.rs@@ -20,8 +20,7 @@use crate::utils;use arrow::array::{Array, ArrayRef, Capacities, GenericListArray, MutableArrayData, NullBufferBuilder,- OffsetBufferBuilder, OffsetSizeTrait, Scalar, cast::AsArray, make_array,- new_null_array,+ OffsetSizeTrait, Scalar, cast::AsArray, make_array, new_null_array,};use arrow::buffer::OffsetBuffer;use arrow::datatypes::{DataType, FieldRef};@@ -466,7 +465,7 @@ fn general_remove<OffsetSize: OffsetSizeTrait>(};let original_data = list_array.values().to_data();// Build up the offsets for the final output array- let mut offsets = Vec::<OffsetSize>::with_capacity(arr_n.len() + 1);+ let mut offsets = Vec::<OffsetSize>::with_capacity(list_array.len() + 1);offsets.push(OffsetSize::zero());let mut mutable = MutableArrayData::with_capacities(@@ -584,7 +583,8 @@ fn general_remove_with_scalar<OffsetSize: OffsetSizeTrait>(let values_range_len = last_offset - first_offset;let values_slice = list_array.values().slice(first_offset, values_range_len);let original_data = values_slice.to_data();- let mut offsets = OffsetBufferBuilder::<OffsetSize>::new(list_array.len());+ let mut offsets = Vec::<OffsetSize>::with_capacity(list_array.len() + 1);+ offsets.push(OffsetSize::zero());let mut mutable = MutableArrayData::with_capacities(vec![&original_data],@@ -598,7 +598,7 @@ fn general_remove_with_scalar<OffsetSize: OffsetSizeTrait>(for (row_index, offset_window) in list_offsets.windows(2).enumerate() {if nulls.as_ref().is_some_and(|nulls| nulls.is_null(row_index)) {- offsets.push_length(0);+ offsets.push(offsets[row_index]);continue;}@@ -611,7 +611,7 @@ fn general_remove_with_scalar<OffsetSize: OffsetSizeTrait>(if num_to_remove == 0 {mutable.extend(0, start, end);- offsets.push_length(row_len);+ offsets.push(offsets[row_index] + OffsetSize::usize_as(row_len));continue;}@@ -641,13 +641,13 @@ fn general_remove_with_scalar<OffsetSize: OffsetSizeTrait>(copied += end - prev_end;}- offsets.push_length(copied);+ offsets.push(offsets[row_index] + OffsetSize::usize_as(copied));}let new_values = make_array(mutable.freeze());Ok(Arc::new(GenericListArray::<OffsetSize>::try_new(Arc::clone(list_field),- offsets.finish(),+ OffsetBuffer::new(offsets.into()),new_values,nulls,)?))diff --git a/datafusion/functions-nested/src/replace.rs b/datafusion/functions-nested/src/replace.rsindex e8a95cff96..28808a05db 100644--- a/datafusion/functions-nested/src/replace.rs+++ b/datafusion/functions-nested/src/replace.rs@@ -19,7 +19,7 @@use arrow::array::{Array, ArrayRef, AsArray, Capacities, GenericListArray, MutableArrayData,- NullBufferBuilder, OffsetBufferBuilder, OffsetSizeTrait, Scalar, new_null_array,+ NullBufferBuilder, OffsetSizeTrait, Scalar, new_null_array,};use arrow::buffer::OffsetBuffer;use arrow::datatypes::{DataType, Field};@@ -391,7 +391,8 @@ fn general_replace<O: OffsetSizeTrait>(arr_n: &[Option<i64>],) -> Result<ArrayRef> {// Build up the offsets for the final output array- let mut offsets: Vec<O> = vec![O::usize_as(0)];+ let mut offsets: Vec<O> = Vec::with_capacity(list_array.len() + 1);+ offsets.push(O::usize_as(0));let values = list_array.values();let original_data = values.to_data();let to_data = to_array.to_data();@@ -538,7 +539,8 @@ fn general_replace_with_scalar<O: OffsetSizeTrait>(capacity,);- let mut offsets = OffsetBufferBuilder::<O>::new(list_array.len());+ let mut offsets = Vec::<O>::with_capacity(list_array.len() + 1);+ offsets.push(O::zero());// Single bulk comparison over the visible values only.let match_bitmap = arrow_ord::cmp::not_distinct(&visible_values, needle)?;@@ -551,7 +553,7 @@ fn general_replace_with_scalar<O: OffsetSizeTrait>(let row_len = end - start;if list_array.is_null(row_index) {- offsets.push_length(0);+ offsets.push(offsets[row_index]);continue;}@@ -563,7 +565,7 @@ fn general_replace_with_scalar<O: OffsetSizeTrait>(.peekable();if match_positions.peek().is_none() {mutable.extend(0, start, end);- offsets.push_length(row_len);+ offsets.push(offsets[row_index] + O::usize_as(row_len));continue;}@@ -586,14 +588,14 @@ fn general_replace_with_scalar<O: OffsetSizeTrait>(mutable.extend(0, start + prev_end, end);}- offsets.push_length(row_len);+ offsets.push(offsets[row_index] + O::usize_as(row_len));}let data = mutable.freeze();Ok(Arc::new(GenericListArray::<O>::try_new(Arc::new(Field::new_list_field(list_array.value_type(), true)),- offsets.finish(),+ OffsetBuffer::new(offsets.into()),arrow::array::make_array(data),list_array.nulls().cloned(),)?))diff --git a/datafusion/functions-nested/src/utils.rs b/datafusion/functions-nested/src/utils.rsindex 1b2bf428ff..8b413686ab 100644--- a/datafusion/functions-nested/src/utils.rs+++ b/datafusion/functions-nested/src/utils.rs@@ -23,7 +23,7 @@ use arrow::datatypes::{DataType, Field, Fields};use arrow::array::{Array, ArrayRef, BooleanArray, Float64Array, GenericListArray, NullBufferBuilder,- OffsetBufferBuilder, OffsetSizeTrait, Scalar,+ OffsetSizeTrait, Scalar,};use arrow::buffer::{NullBuffer, OffsetBuffer};use datafusion_common::cast::{@@ -361,11 +361,12 @@ wherelet mut out_values: Vec<f64> = Vec::with_capacity(lhs_values.len());let mut out_inner_nulls = NullBufferBuilder::new(lhs_values.len());- let mut out_offsets = OffsetBufferBuilder::<O>::new(lhs.len());+ let mut out_offsets = Vec::<O>::with_capacity(lhs.len() + 1);+ out_offsets.push(O::zero());for row in 0..lhs.len() {if row_nulls.as_ref().is_some_and(|nb| nb.is_null(row)) {- out_offsets.push_length(0);+ out_offsets.push(out_offsets[row]);continue;}@@ -395,7 +396,7 @@ whereNone => out_inner_nulls.append_n_non_nulls(len1),}- out_offsets.push_length(len1);+ out_offsets.push(out_offsets[row] + O::usize_as(len1));}let values_array = Arc::new(Float64Array::new(@@ -406,7 +407,7 @@ whereOk(Arc::new(GenericListArray::<O>::try_new(field,- out_offsets.finish(),+ OffsetBuffer::new(out_offsets.into()),values_array,row_nulls,)?))diff --git a/datafusion/functions/src/encoding/inner.rs b/datafusion/functions/src/encoding/inner.rsindex ad156f735b..877acbb529 100644--- a/datafusion/functions/src/encoding/inner.rs+++ b/datafusion/functions/src/encoding/inner.rs@@ -24,7 +24,7 @@ use arrow::{},datatypes::DataType,};-use arrow_buffer::{Buffer, OffsetBufferBuilder};+use arrow_buffer::{Buffer, OffsetBuffer};use base64::{Engine as _,engine::{DecodePaddingMode, GeneralPurpose, GeneralPurposeConfig},@@ -470,22 +470,21 @@ whereOutputOffset: OffsetSizeTrait,{let mut values = vec![0; conservative_upper_bound_size];- let mut offsets = OffsetBufferBuilder::new(input.len());+ let mut offsets = Vec::<OutputOffset>::with_capacity(input.len() + 1);+ offsets.push(OutputOffset::zero());let mut total_bytes_decoded = 0;for v in input.iter() {if let Some(v) = v {let cursor = &mut values[total_bytes_decoded..];let decoded = decode(v, cursor)?;total_bytes_decoded += decoded;- offsets.push_length(decoded);- } else {- offsets.push_length(0);}+ offsets.push(OutputOffset::usize_as(total_bytes_decoded));}// We reserved an upper bound size for the values buffer, but we only use the actual sizevalues.truncate(total_bytes_decoded);let binary_array = GenericBinaryArray::<OutputOffset>::try_new(- offsets.finish(),+ OffsetBuffer::new(offsets.into()),Buffer::from_vec(values),input.nulls().cloned(),)?;