Skip to content

Apache DataFusion SQL Query Engine

chore: use `Vec` instead of `OffsetBuilder` (#23195)

Oleks V32e27acb9f64
diff --git a/datafusion/functions-nested/src/array_add.rs b/datafusion/functions-nested/src/array_add.rs
index c6edf67bf5..dd170911fb 100644
--- a/datafusion/functions-nested/src/array_add.rs
+++ b/datafusion/functions-nested/src/array_add.rs
@@ -19,10 +19,9 @@
use crate::utils::{coerce_array_math_arg_types, make_scalar_function};
use arrow::array::{
- Array, ArrayRef, Float64Array, GenericListArray, NullBufferBuilder,
- OffsetBufferBuilder, OffsetSizeTrait,
+ Array, ArrayRef, Float64Array, GenericListArray, NullBufferBuilder, OffsetSizeTrait,
};
-use arrow::buffer::NullBuffer;
+use arrow::buffer::{NullBuffer, OffsetBuffer};
use arrow::datatypes::{
DataType,
DataType::{LargeList, List},
@@ -147,12 +146,13 @@ fn general_array_add<O: OffsetSizeTrait>(
let mut out_values: Vec<f64> = Vec::with_capacity(lhs_values.len());
let mut out_inner_nulls = NullBufferBuilder::new(lhs_values.len());
- let mut out_offsets = OffsetBufferBuilder::<O>::new(lhs.len());
+ let mut out_offsets = Vec::<O>::with_capacity(lhs.len() + 1);
+ out_offsets.push(O::zero());
for row in 0..lhs.len() {
// Whole-row NULL on either side -> NULL output row, no elements.
if row_nulls.as_ref().is_some_and(|nb| nb.is_null(row)) {
- out_offsets.push_length(0);
+ out_offsets.push(out_offsets[row]);
continue;
}
@@ -185,7 +185,7 @@ fn general_array_add<O: OffsetSizeTrait>(
None => out_inner_nulls.append_n_non_nulls(len1),
}
- out_offsets.push_length(len1);
+ out_offsets.push(out_offsets[row] + O::usize_as(len1));
}
let values_array = Arc::new(Float64Array::new(
@@ -196,7 +196,7 @@ fn general_array_add<O: OffsetSizeTrait>(
Ok(Arc::new(GenericListArray::<O>::try_new(
field,
- out_offsets.finish(),
+ OffsetBuffer::new(out_offsets.into()),
values_array,
row_nulls,
)?))
diff --git a/datafusion/functions-nested/src/array_filter.rs b/datafusion/functions-nested/src/array_filter.rs
index a1fa8268a3..7dd7230ae9 100644
--- a/datafusion/functions-nested/src/array_filter.rs
+++ b/datafusion/functions-nested/src/array_filter.rs
@@ -20,7 +20,7 @@
use arrow::{
array::{
Array, ArrayRef, AsArray, BooleanArray, LargeListArray, ListArray,
- OffsetBufferBuilder, OffsetSizeTrait, new_empty_array,
+ OffsetSizeTrait, new_empty_array,
},
buffer::{OffsetBuffer, ScalarBuffer},
compute::{filter as arrow_filter, take_arrays},
@@ -252,13 +252,11 @@ fn filter_list_values<O: OffsetSizeTrait>(
offsets: &OffsetBuffer<O>,
) -> Result<(ArrayRef, OffsetBuffer<O>)> {
let num_sublists = offsets.len().saturating_sub(1);
- let mut builder = OffsetBufferBuilder::<O>::new(num_sublists);
-
let has_nulls = predicate.null_count() > 0;
- for i in 0..num_sublists {
+ let new_offsets = OffsetBuffer::<O>::from_lengths((0..num_sublists).map(|i| {
let start = offsets[i].as_usize();
let end = offsets[i + 1].as_usize();
- let count = if has_nulls {
+ if has_nulls {
(start..end)
.filter(|&j| predicate.is_valid(j) && predicate.value(j))
.count()
@@ -267,11 +265,8 @@ fn filter_list_values<O: OffsetSizeTrait>(
.values()
.slice(start, end - start)
.count_set_bits()
- };
- builder.push_length(count);
- }
-
- let new_offsets = builder.finish();
+ }
+ }));
if new_offsets.last() == offsets.last() {
return Ok((Arc::clone(values), offsets.clone()));
diff --git a/datafusion/functions-nested/src/array_normalize.rs b/datafusion/functions-nested/src/array_normalize.rs
index 0ff7674032..f7da07e5f6 100644
--- a/datafusion/functions-nested/src/array_normalize.rs
+++ b/datafusion/functions-nested/src/array_normalize.rs
@@ -19,9 +19,9 @@
use crate::utils::make_scalar_function;
use arrow::array::{
- Array, ArrayRef, Float64Array, GenericListArray, NullBufferBuilder,
- OffsetBufferBuilder, OffsetSizeTrait,
+ Array, ArrayRef, Float64Array, GenericListArray, NullBufferBuilder, OffsetSizeTrait,
};
+use arrow::buffer::OffsetBuffer;
use arrow::datatypes::{
DataType,
DataType::{FixedSizeList, LargeList, List, Null},
@@ -144,13 +144,14 @@ fn general_array_normalize<O: OffsetSizeTrait>(arrays: &[ArrayRef]) -> Result<Ar
let offsets = list_array.value_offsets();
let mut new_values: Vec<f64> = Vec::with_capacity(values.len());
- let mut new_offsets = OffsetBufferBuilder::<O>::new(list_array.len());
+ let mut new_offsets = Vec::<O>::with_capacity(list_array.len() + 1);
+ new_offsets.push(O::zero());
let mut nulls = NullBufferBuilder::new(list_array.len());
for row in 0..list_array.len() {
if list_array.is_null(row) {
nulls.append_null();
- new_offsets.push_length(0);
+ new_offsets.push(new_offsets[row]);
continue;
}
@@ -161,7 +162,7 @@ fn general_array_normalize<O: OffsetSizeTrait>(arrays: &[ArrayRef]) -> Result<Ar
let slice = values.slice(start, len);
if slice.null_count() != 0 {
nulls.append_null();
- new_offsets.push_length(0);
+ new_offsets.push(new_offsets[row]);
continue;
}
@@ -170,7 +171,7 @@ fn general_array_normalize<O: OffsetSizeTrait>(arrays: &[ArrayRef]) -> Result<Ar
// Empty array: return empty array (no normalization needed, no division by zero risk)
if len == 0 {
nulls.append_non_null();
- new_offsets.push_length(0);
+ new_offsets.push(new_offsets[row]);
continue;
}
@@ -183,7 +184,7 @@ fn general_array_normalize<O: OffsetSizeTrait>(arrays: &[ArrayRef]) -> Result<Ar
// Zero magnitude: undefined normalization. Emit NULL row.
if sq_sum == 0.0 {
nulls.append_null();
- new_offsets.push_length(0);
+ new_offsets.push(new_offsets[row]);
continue;
}
@@ -192,7 +193,7 @@ fn general_array_normalize<O: OffsetSizeTrait>(arrays: &[ArrayRef]) -> Result<Ar
new_values.push(vals[i] / mag);
}
nulls.append_non_null();
- new_offsets.push_length(len);
+ new_offsets.push(new_offsets[row] + O::usize_as(len));
}
let values_array = Arc::new(Float64Array::from(new_values));
@@ -200,7 +201,7 @@ fn general_array_normalize<O: OffsetSizeTrait>(arrays: &[ArrayRef]) -> Result<Ar
Ok(Arc::new(GenericListArray::<O>::try_new(
field,
- new_offsets.finish(),
+ OffsetBuffer::new(new_offsets.into()),
values_array,
nulls.finish(),
)?))
diff --git a/datafusion/functions-nested/src/array_scale.rs b/datafusion/functions-nested/src/array_scale.rs
index 24750ade8a..a6be910d20 100644
--- a/datafusion/functions-nested/src/array_scale.rs
+++ b/datafusion/functions-nested/src/array_scale.rs
@@ -18,10 +18,8 @@
//! [`ScalarUDFImpl`] definitions for array_scale function.
use crate::utils::make_scalar_function;
-use arrow::array::{
- Array, ArrayRef, Float64Array, GenericListArray, OffsetBufferBuilder, OffsetSizeTrait,
-};
-use arrow::buffer::NullBuffer;
+use arrow::array::{Array, ArrayRef, Float64Array, GenericListArray, OffsetSizeTrait};
+use arrow::buffer::{NullBuffer, OffsetBuffer};
use arrow::datatypes::{
DataType,
DataType::{FixedSizeList, LargeList, List, Null},
@@ -172,11 +170,12 @@ fn general_array_scale<O: OffsetSizeTrait>(
let row_nulls = NullBuffer::union(list_array.nulls(), scalar_array.nulls());
let mut value_builder = Float64Array::builder(values.len());
- let mut new_offsets = OffsetBufferBuilder::<O>::new(list_array.len());
+ let mut new_offsets = Vec::<O>::with_capacity(list_array.len() + 1);
+ new_offsets.push(O::zero());
for row in 0..list_array.len() {
if row_nulls.as_ref().is_some_and(|nb| nb.is_null(row)) {
- new_offsets.push_length(0);
+ new_offsets.push(new_offsets[row]);
continue;
}
@@ -196,7 +195,7 @@ fn general_array_scale<O: OffsetSizeTrait>(
}
}
- new_offsets.push_length(len);
+ new_offsets.push(new_offsets[row] + O::usize_as(len));
}
let values_array = Arc::new(value_builder.finish());
@@ -213,7 +212,7 @@ fn general_array_scale<O: OffsetSizeTrait>(
Ok(Arc::new(GenericListArray::<O>::try_new(
field,
- new_offsets.finish(),
+ OffsetBuffer::new(new_offsets.into()),
values_array,
row_nulls,
)?))
diff --git a/datafusion/functions-nested/src/remove.rs b/datafusion/functions-nested/src/remove.rs
index 9d7dd4f44d..111147659a 100644
--- a/datafusion/functions-nested/src/remove.rs
+++ b/datafusion/functions-nested/src/remove.rs
@@ -20,8 +20,7 @@
use crate::utils;
use arrow::array::{
Array, ArrayRef, Capacities, GenericListArray, MutableArrayData, NullBufferBuilder,
- OffsetBufferBuilder, OffsetSizeTrait, Scalar, cast::AsArray, make_array,
- new_null_array,
+ OffsetSizeTrait, Scalar, cast::AsArray, make_array, new_null_array,
};
use arrow::buffer::OffsetBuffer;
use arrow::datatypes::{DataType, FieldRef};
@@ -466,7 +465,7 @@ fn general_remove<OffsetSize: OffsetSizeTrait>(
};
let original_data = list_array.values().to_data();
// Build up the offsets for the final output array
- let mut offsets = Vec::<OffsetSize>::with_capacity(arr_n.len() + 1);
+ let mut offsets = Vec::<OffsetSize>::with_capacity(list_array.len() + 1);
offsets.push(OffsetSize::zero());
let mut mutable = MutableArrayData::with_capacities(
@@ -584,7 +583,8 @@ fn general_remove_with_scalar<OffsetSize: OffsetSizeTrait>(
let values_range_len = last_offset - first_offset;
let values_slice = list_array.values().slice(first_offset, values_range_len);
let original_data = values_slice.to_data();
- let mut offsets = OffsetBufferBuilder::<OffsetSize>::new(list_array.len());
+ let mut offsets = Vec::<OffsetSize>::with_capacity(list_array.len() + 1);
+ offsets.push(OffsetSize::zero());
let mut mutable = MutableArrayData::with_capacities(
vec![&original_data],
@@ -598,7 +598,7 @@ fn general_remove_with_scalar<OffsetSize: OffsetSizeTrait>(
for (row_index, offset_window) in list_offsets.windows(2).enumerate() {
if nulls.as_ref().is_some_and(|nulls| nulls.is_null(row_index)) {
- offsets.push_length(0);
+ offsets.push(offsets[row_index]);
continue;
}
@@ -611,7 +611,7 @@ fn general_remove_with_scalar<OffsetSize: OffsetSizeTrait>(
if num_to_remove == 0 {
mutable.extend(0, start, end);
- offsets.push_length(row_len);
+ offsets.push(offsets[row_index] + OffsetSize::usize_as(row_len));
continue;
}
@@ -641,13 +641,13 @@ fn general_remove_with_scalar<OffsetSize: OffsetSizeTrait>(
copied += end - prev_end;
}
- offsets.push_length(copied);
+ offsets.push(offsets[row_index] + OffsetSize::usize_as(copied));
}
let new_values = make_array(mutable.freeze());
Ok(Arc::new(GenericListArray::<OffsetSize>::try_new(
Arc::clone(list_field),
- offsets.finish(),
+ OffsetBuffer::new(offsets.into()),
new_values,
nulls,
)?))
diff --git a/datafusion/functions-nested/src/replace.rs b/datafusion/functions-nested/src/replace.rs
index e8a95cff96..28808a05db 100644
--- a/datafusion/functions-nested/src/replace.rs
+++ b/datafusion/functions-nested/src/replace.rs
@@ -19,7 +19,7 @@
use arrow::array::{
Array, ArrayRef, AsArray, Capacities, GenericListArray, MutableArrayData,
- NullBufferBuilder, OffsetBufferBuilder, OffsetSizeTrait, Scalar, new_null_array,
+ NullBufferBuilder, OffsetSizeTrait, Scalar, new_null_array,
};
use arrow::buffer::OffsetBuffer;
use arrow::datatypes::{DataType, Field};
@@ -391,7 +391,8 @@ fn general_replace<O: OffsetSizeTrait>(
arr_n: &[Option<i64>],
) -> Result<ArrayRef> {
// Build up the offsets for the final output array
- let mut offsets: Vec<O> = vec![O::usize_as(0)];
+ let mut offsets: Vec<O> = Vec::with_capacity(list_array.len() + 1);
+ offsets.push(O::usize_as(0));
let values = list_array.values();
let original_data = values.to_data();
let to_data = to_array.to_data();
@@ -538,7 +539,8 @@ fn general_replace_with_scalar<O: OffsetSizeTrait>(
capacity,
);
- let mut offsets = OffsetBufferBuilder::<O>::new(list_array.len());
+ let mut offsets = Vec::<O>::with_capacity(list_array.len() + 1);
+ offsets.push(O::zero());
// Single bulk comparison over the visible values only.
let match_bitmap = arrow_ord::cmp::not_distinct(&visible_values, needle)?;
@@ -551,7 +553,7 @@ fn general_replace_with_scalar<O: OffsetSizeTrait>(
let row_len = end - start;
if list_array.is_null(row_index) {
- offsets.push_length(0);
+ offsets.push(offsets[row_index]);
continue;
}
@@ -563,7 +565,7 @@ fn general_replace_with_scalar<O: OffsetSizeTrait>(
.peekable();
if match_positions.peek().is_none() {
mutable.extend(0, start, end);
- offsets.push_length(row_len);
+ offsets.push(offsets[row_index] + O::usize_as(row_len));
continue;
}
@@ -586,14 +588,14 @@ fn general_replace_with_scalar<O: OffsetSizeTrait>(
mutable.extend(0, start + prev_end, end);
}
- offsets.push_length(row_len);
+ offsets.push(offsets[row_index] + O::usize_as(row_len));
}
let data = mutable.freeze();
Ok(Arc::new(GenericListArray::<O>::try_new(
Arc::new(Field::new_list_field(list_array.value_type(), true)),
- offsets.finish(),
+ OffsetBuffer::new(offsets.into()),
arrow::array::make_array(data),
list_array.nulls().cloned(),
)?))
diff --git a/datafusion/functions-nested/src/utils.rs b/datafusion/functions-nested/src/utils.rs
index 1b2bf428ff..8b413686ab 100644
--- a/datafusion/functions-nested/src/utils.rs
+++ b/datafusion/functions-nested/src/utils.rs
@@ -23,7 +23,7 @@ use arrow::datatypes::{DataType, Field, Fields};
use arrow::array::{
Array, ArrayRef, BooleanArray, Float64Array, GenericListArray, NullBufferBuilder,
- OffsetBufferBuilder, OffsetSizeTrait, Scalar,
+ OffsetSizeTrait, Scalar,
};
use arrow::buffer::{NullBuffer, OffsetBuffer};
use datafusion_common::cast::{
@@ -361,11 +361,12 @@ where
let mut out_values: Vec<f64> = Vec::with_capacity(lhs_values.len());
let mut out_inner_nulls = NullBufferBuilder::new(lhs_values.len());
- let mut out_offsets = OffsetBufferBuilder::<O>::new(lhs.len());
+ let mut out_offsets = Vec::<O>::with_capacity(lhs.len() + 1);
+ out_offsets.push(O::zero());
for row in 0..lhs.len() {
if row_nulls.as_ref().is_some_and(|nb| nb.is_null(row)) {
- out_offsets.push_length(0);
+ out_offsets.push(out_offsets[row]);
continue;
}
@@ -395,7 +396,7 @@ where
None => out_inner_nulls.append_n_non_nulls(len1),
}
- out_offsets.push_length(len1);
+ out_offsets.push(out_offsets[row] + O::usize_as(len1));
}
let values_array = Arc::new(Float64Array::new(
@@ -406,7 +407,7 @@ where
Ok(Arc::new(GenericListArray::<O>::try_new(
field,
- out_offsets.finish(),
+ OffsetBuffer::new(out_offsets.into()),
values_array,
row_nulls,
)?))
diff --git a/datafusion/functions/src/encoding/inner.rs b/datafusion/functions/src/encoding/inner.rs
index ad156f735b..877acbb529 100644
--- a/datafusion/functions/src/encoding/inner.rs
+++ b/datafusion/functions/src/encoding/inner.rs
@@ -24,7 +24,7 @@ use arrow::{
},
datatypes::DataType,
};
-use arrow_buffer::{Buffer, OffsetBufferBuilder};
+use arrow_buffer::{Buffer, OffsetBuffer};
use base64::{
Engine as _,
engine::{DecodePaddingMode, GeneralPurpose, GeneralPurposeConfig},
@@ -470,22 +470,21 @@ where
OutputOffset: OffsetSizeTrait,
{
let mut values = vec![0; conservative_upper_bound_size];
- let mut offsets = OffsetBufferBuilder::new(input.len());
+ let mut offsets = Vec::<OutputOffset>::with_capacity(input.len() + 1);
+ offsets.push(OutputOffset::zero());
let mut total_bytes_decoded = 0;
for v in input.iter() {
if let Some(v) = v {
let cursor = &mut values[total_bytes_decoded..];
let decoded = decode(v, cursor)?;
total_bytes_decoded += decoded;
- offsets.push_length(decoded);
- } else {
- offsets.push_length(0);
}
+ offsets.push(OutputOffset::usize_as(total_bytes_decoded));
}
// We reserved an upper bound size for the values buffer, but we only use the actual size
values.truncate(total_bytes_decoded);
let binary_array = GenericBinaryArray::<OutputOffset>::try_new(
- offsets.finish(),
+ OffsetBuffer::new(offsets.into()),
Buffer::from_vec(values),
input.nulls().cloned(),
)?;