Skip to content

Apache DataFusion SQL Query Engine

Optimize memory footprint of view arrays from `ScalarValue::to_array_of_size` (#19441)

Jeffrey Vo677c543aef6c
diff --git a/datafusion/common/src/scalar/mod.rs b/datafusion/common/src/scalar/mod.rs
index 2eb7e970df..fe0c92e0ec 100644
--- a/datafusion/common/src/scalar/mod.rs
+++ b/datafusion/common/src/scalar/mod.rs
@@ -57,19 +57,20 @@ use crate::utils::SingleRowListArrayBuilder;
use crate::{_internal_datafusion_err, arrow_datafusion_err};
use arrow::array::{
Array, ArrayData, ArrayRef, ArrowNativeTypeOp, ArrowPrimitiveType, AsArray,
- BinaryArray, BinaryViewArray, BooleanArray, Date32Array, Date64Array, Decimal32Array,
- Decimal64Array, Decimal128Array, Decimal256Array, DictionaryArray,
- DurationMicrosecondArray, DurationMillisecondArray, DurationNanosecondArray,
- DurationSecondArray, FixedSizeBinaryArray, FixedSizeBinaryBuilder,
- FixedSizeListArray, Float16Array, Float32Array, Float64Array, GenericListArray,
- Int8Array, Int16Array, Int32Array, Int64Array, IntervalDayTimeArray,
- IntervalMonthDayNanoArray, IntervalYearMonthArray, LargeBinaryArray, LargeListArray,
- LargeStringArray, ListArray, MapArray, MutableArrayData, OffsetSizeTrait,
- PrimitiveArray, Scalar, StringArray, StringViewArray, StructArray,
- Time32MillisecondArray, Time32SecondArray, Time64MicrosecondArray,
- Time64NanosecondArray, TimestampMicrosecondArray, TimestampMillisecondArray,
- TimestampNanosecondArray, TimestampSecondArray, UInt8Array, UInt16Array, UInt32Array,
- UInt64Array, UnionArray, new_empty_array, new_null_array,
+ BinaryArray, BinaryViewArray, BinaryViewBuilder, BooleanArray, Date32Array,
+ Date64Array, Decimal32Array, Decimal64Array, Decimal128Array, Decimal256Array,
+ DictionaryArray, DurationMicrosecondArray, DurationMillisecondArray,
+ DurationNanosecondArray, DurationSecondArray, FixedSizeBinaryArray,
+ FixedSizeBinaryBuilder, FixedSizeListArray, Float16Array, Float32Array, Float64Array,
+ GenericListArray, Int8Array, Int16Array, Int32Array, Int64Array,
+ IntervalDayTimeArray, IntervalMonthDayNanoArray, IntervalYearMonthArray,
+ LargeBinaryArray, LargeListArray, LargeStringArray, ListArray, MapArray,
+ MutableArrayData, OffsetSizeTrait, PrimitiveArray, Scalar, StringArray,
+ StringViewArray, StringViewBuilder, StructArray, Time32MillisecondArray,
+ Time32SecondArray, Time64MicrosecondArray, Time64NanosecondArray,
+ TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,
+ TimestampSecondArray, UInt8Array, UInt16Array, UInt32Array, UInt64Array, UnionArray,
+ new_empty_array, new_null_array,
};
use arrow::buffer::{BooleanBuffer, ScalarBuffer};
use arrow::compute::kernels::cast::{CastOptions, cast_with_options};
@@ -3027,7 +3028,13 @@ impl ScalarValue {
},
ScalarValue::Utf8View(e) => match e {
Some(value) => {
- Arc::new(StringViewArray::from_iter_values(repeat_n(value, size)))
+ let mut builder =
+ StringViewBuilder::with_capacity(size).with_deduplicate_strings();
+ for _ in 0..size {
+ builder.append_value(value);
+ }
+ let array = builder.finish();
+ Arc::new(array)
}
None => new_null_array(&DataType::Utf8View, size),
},
@@ -3042,9 +3049,15 @@ impl ScalarValue {
None => new_null_array(&DataType::Binary, size),
},
ScalarValue::BinaryView(e) => match e {
- Some(value) => Arc::new(
- repeat_n(Some(value.as_slice()), size).collect::<BinaryViewArray>(),
- ),
+ Some(value) => {
+ let mut builder =
+ BinaryViewBuilder::with_capacity(size).with_deduplicate_strings();
+ for _ in 0..size {
+ builder.append_value(value);
+ }
+ let array = builder.finish();
+ Arc::new(array)
+ }
None => new_null_array(&DataType::BinaryView, size),
},
ScalarValue::FixedSizeBinary(s, e) => match e {
@@ -9232,6 +9245,27 @@ mod tests {
}
}
+ #[test]
+ fn test_views_minimize_memory() {
+ let value = "this string is longer than 12 bytes".to_string();
+
+ let scalar = ScalarValue::Utf8View(Some(value.clone()));
+ let array = scalar.to_array_of_size(10).unwrap();
+ let array = array.as_string_view();
+ let buffers = array.data_buffers();
+ assert_eq!(1, buffers.len());
+ // Ensure we only have a single copy of the value string
+ assert_eq!(value.len(), buffers[0].len());
+
+ // Same but for BinaryView
+ let scalar = ScalarValue::BinaryView(Some(value.bytes().collect()));
+ let array = scalar.to_array_of_size(10).unwrap();
+ let array = array.as_binary_view();
+ let buffers = array.data_buffers();
+ assert_eq!(1, buffers.len());
+ assert_eq!(value.len(), buffers[0].len());
+ }
+
#[test]
fn test_convert_array_to_scalar_vec() {
// 1: Regular ListArray