Apache DataFusion SQL Query Engine
feat: support `ListView` and `LargeListView` in `ScalarValue` (#21669)
Jeffrey Vo8a45d0250f30
diff --git a/datafusion/common/src/scalar/mod.rs b/datafusion/common/src/scalar/mod.rsindex ab58bf4a61..d726b5c940 100644--- a/datafusion/common/src/scalar/mod.rs+++ b/datafusion/common/src/scalar/mod.rs@@ -44,12 +44,12 @@ use crate::cast::{as_float16_array, as_float32_array, as_float64_array, as_int8_array, as_int16_array,as_int32_array, as_int64_array, as_interval_dt_array, as_interval_mdn_array,as_interval_ym_array, as_large_binary_array, as_large_list_array,- as_large_string_array, as_run_array, as_string_array, as_string_view_array,- as_time32_millisecond_array, as_time32_second_array, as_time64_microsecond_array,- as_time64_nanosecond_array, as_timestamp_microsecond_array,- as_timestamp_millisecond_array, as_timestamp_nanosecond_array,- as_timestamp_second_array, as_uint8_array, as_uint16_array, as_uint32_array,- as_uint64_array, as_union_array,+ as_large_list_view_array, as_large_string_array, as_list_view_array, as_run_array,+ as_string_array, as_string_view_array, as_time32_millisecond_array,+ as_time32_second_array, as_time64_microsecond_array, as_time64_nanosecond_array,+ as_timestamp_microsecond_array, as_timestamp_millisecond_array,+ as_timestamp_nanosecond_array, as_timestamp_second_array, as_uint8_array,+ as_uint16_array, as_uint32_array, as_uint64_array, as_union_array,};use crate::error::{_exec_err, _internal_err, _not_impl_err, DataFusionError, Result};use crate::format::DEFAULT_CAST_OPTIONS;@@ -63,14 +63,15 @@ use arrow::array::{DictionaryArray, DurationMicrosecondArray, DurationMillisecondArray,DurationNanosecondArray, DurationSecondArray, FixedSizeBinaryArray,FixedSizeListArray, Float16Array, Float32Array, Float64Array, GenericListArray,- Int8Array, Int16Array, Int32Array, Int64Array, IntervalDayTimeArray,- IntervalMonthDayNanoArray, IntervalYearMonthArray, LargeBinaryArray, LargeListArray,- LargeStringArray, ListArray, MapArray, MutableArrayData, OffsetSizeTrait,- PrimitiveArray, RunArray, Scalar, StringArray, StringViewArray, StringViewBuilder,- StructArray, Time32MillisecondArray, Time32SecondArray, Time64MicrosecondArray,- Time64NanosecondArray, TimestampMicrosecondArray, TimestampMillisecondArray,- TimestampNanosecondArray, TimestampSecondArray, UInt8Array, UInt16Array, UInt32Array,- UInt64Array, UnionArray, downcast_run_array, new_empty_array, new_null_array,+ GenericListViewArray, Int8Array, Int16Array, Int32Array, Int64Array,+ IntervalDayTimeArray, IntervalMonthDayNanoArray, IntervalYearMonthArray,+ LargeBinaryArray, LargeListArray, LargeListViewArray, LargeStringArray, ListArray,+ ListViewArray, MapArray, MutableArrayData, PrimitiveArray, RunArray, Scalar,+ StringArray, StringViewArray, StringViewBuilder, StructArray, Time32MillisecondArray,+ Time32SecondArray, Time64MicrosecondArray, Time64NanosecondArray,+ TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,+ TimestampSecondArray, UInt8Array, UInt16Array, UInt32Array, UInt64Array, UnionArray,+ downcast_run_array, new_empty_array, new_null_array,};use arrow::buffer::{BooleanBuffer, ScalarBuffer};use arrow::compute::kernels::cast::{CastOptions, cast_with_options};@@ -257,8 +258,8 @@ pub(crate) fn format_timestamp_type_for_error(target_type: &DataType) -> String////// # Nested Types///-/// `List` / `LargeList` / `FixedSizeList` / `Struct` / `Map` are represented as a-/// single element array of the corresponding type.+/// `List` / `LargeList` / `FixedSizeList` / `ListView` / `LargeListView` / `Struct` / `Map`+/// are represented as a single element array of the corresponding type.////// ## Example: Creating [`ScalarValue::Struct`] using [`ScalarStructBuilder`]/// ```@@ -381,6 +382,14 @@ pub enum ScalarValue {List(Arc<ListArray>),/// The array must be a LargeListArray with length 1.LargeList(Arc<LargeListArray>),+ /// Represents a single element of a [`ListViewArray`] as an [`ArrayRef`]+ ///+ /// The array must be a ListViewArray with length 1.+ ListView(Arc<ListViewArray>),+ /// Represents a single element of a [`LargeListViewArray`] as an [`ArrayRef`]+ ///+ /// The array must be a LargeListViewArray with length 1.+ LargeListView(Arc<LargeListViewArray>),/// Represents a single element [`StructArray`] as an [`ArrayRef`]. See/// [`ScalarValue`] for examples of how to create instances of this type.Struct(Arc<StructArray>),@@ -517,6 +526,10 @@ impl PartialEq for ScalarValue {(List(_), _) => false,(LargeList(v1), LargeList(v2)) => v1.eq(v2),(LargeList(_), _) => false,+ (ListView(v1), ListView(v2)) => v1.eq(v2),+ (ListView(_), _) => false,+ (LargeListView(v1), LargeListView(v2)) => v1.eq(v2),+ (LargeListView(_), _) => false,(Struct(v1), Struct(v2)) => v1.eq(v2),(Struct(_), _) => false,(Map(v1), Map(v2)) => v1.eq(v2),@@ -662,7 +675,8 @@ impl PartialOrd for ScalarValue {(FixedSizeBinary(_, _), _) => None,(LargeBinary(v1), LargeBinary(v2)) => v1.partial_cmp(v2),(LargeBinary(_), _) => None,- // ScalarValue::List / ScalarValue::FixedSizeList / ScalarValue::LargeList are ensure to have length 1+ // ScalarValue::List / ScalarValue::FixedSizeList / ScalarValue::LargeList / ScalarValue::ListView / ScalarValue::LargeListView+ // are guaranteed to have length 1(List(arr1), List(arr2)) => partial_cmp_list(arr1.as_ref(), arr2.as_ref()),(FixedSizeList(arr1), FixedSizeList(arr2)) => {partial_cmp_list(arr1.as_ref(), arr2.as_ref())@@ -670,7 +684,17 @@ impl PartialOrd for ScalarValue {(LargeList(arr1), LargeList(arr2)) => {partial_cmp_list(arr1.as_ref(), arr2.as_ref())}- (List(_), _) | (LargeList(_), _) | (FixedSizeList(_), _) => None,+ (ListView(arr1), ListView(arr2)) => {+ partial_cmp_list(arr1.as_ref(), arr2.as_ref())+ }+ (LargeListView(arr1), LargeListView(arr2)) => {+ partial_cmp_list(arr1.as_ref(), arr2.as_ref())+ }+ (List(_), _)+ | (LargeList(_), _)+ | (FixedSizeList(_), _)+ | (ListView(_), _)+ | (LargeListView(_), _) => None,(Struct(struct_arr1), Struct(struct_arr2)) => {partial_cmp_struct(struct_arr1.as_ref(), struct_arr2.as_ref())}@@ -745,7 +769,7 @@ impl PartialOrd for ScalarValue {}}-/// List/LargeList/FixedSizeList scalars always have a single element+/// List/LargeList/FixedSizeList/ListView/LargeListView scalars always have a single element/// array. This function returns that arrayfn first_array_for_list(arr: &dyn Array) -> ArrayRef {assert_eq!(arr.len(), 1);@@ -755,14 +779,18 @@ fn first_array_for_list(arr: &dyn Array) -> ArrayRef {arr.value(0)} else if let Some(arr) = arr.as_fixed_size_list_opt() {arr.value(0)+ } else if let Some(arr) = arr.as_list_view_opt::<i32>() {+ arr.value(0)+ } else if let Some(arr) = arr.as_list_view_opt::<i64>() {+ arr.value(0)} else {unreachable!(- "Since only List / LargeList / FixedSizeList are supported, this should never happen"+ "Since only List / LargeList / FixedSizeList / ListView / LargeListView are supported, this should never happen")}}-/// Compares two List/LargeList/FixedSizeList scalars+/// Compares two List/LargeList/FixedSizeList/ListView/LargeListView scalarsfn partial_cmp_list(arr1: &dyn Array, arr2: &dyn Array) -> Option<Ordering> {if arr1.data_type() != arr2.data_type() {return None;@@ -950,6 +978,12 @@ impl Hash for ScalarValue {FixedSizeList(arr) => {hash_nested_array(arr.to_owned() as ArrayRef, state);}+ ListView(arr) => {+ hash_nested_array(arr.to_owned() as ArrayRef, state);+ }+ LargeListView(arr) => {+ hash_nested_array(arr.to_owned() as ArrayRef, state);+ }Struct(arr) => {hash_nested_array(arr.to_owned() as ArrayRef, state);}@@ -1288,6 +1322,12 @@ impl ScalarValue {1,)))}+ DataType::ListView(field_ref) => ScalarValue::ListView(Arc::new(+ GenericListViewArray::new_null(Arc::clone(field_ref), 1),+ )),+ DataType::LargeListView(field_ref) => ScalarValue::LargeListView(Arc::new(+ GenericListViewArray::new_null(Arc::clone(field_ref), 1),+ )),DataType::Struct(fields) => ScalarValue::Struct(new_null_array(&DataType::Struct(fields.to_owned()), 1).as_struct()@@ -1640,7 +1680,7 @@ impl ScalarValue {let empty_arr = new_empty_array(field.data_type());let values = Arc::new(SingleRowListArrayBuilder::new(empty_arr)- .with_nullable(field.is_nullable())+ .with_field(field).build_fixed_size_list_array(0),);Ok(ScalarValue::FixedSizeList(values))@@ -1649,6 +1689,24 @@ impl ScalarValue {let list = ScalarValue::new_large_list(&[], field.data_type());Ok(ScalarValue::LargeList(list))}+ DataType::ListView(field) => {+ let empty_arr = new_empty_array(field.data_type());+ let values = Arc::new(+ SingleRowListArrayBuilder::new(empty_arr)+ .with_field(field)+ .build_list_view_array(),+ );+ Ok(ScalarValue::ListView(values))+ }+ DataType::LargeListView(field) => {+ let empty_arr = new_empty_array(field.data_type());+ let values = Arc::new(+ SingleRowListArrayBuilder::new(empty_arr)+ .with_field(field)+ .build_large_list_view_array(),+ );+ Ok(ScalarValue::LargeListView(values))+ }// Struct typesDataType::Struct(fields) => {@@ -1698,12 +1756,6 @@ impl ScalarValue {_internal_err!("Union type must have at least one field")}}-- DataType::ListView(_) | DataType::LargeListView(_) => {- _not_impl_err!(- "Default value for data_type \"{datatype}\" is not implemented yet"- )- }}}@@ -1961,6 +2013,8 @@ impl ScalarValue {ScalarValue::List(arr) => arr.data_type().to_owned(),ScalarValue::LargeList(arr) => arr.data_type().to_owned(),ScalarValue::FixedSizeList(arr) => arr.data_type().to_owned(),+ ScalarValue::ListView(arr) => arr.data_type().to_owned(),+ ScalarValue::LargeListView(arr) => arr.data_type().to_owned(),ScalarValue::Struct(arr) => arr.data_type().to_owned(),ScalarValue::Map(arr) => arr.data_type().to_owned(),ScalarValue::Date32(_) => DataType::Date32,@@ -2471,6 +2525,8 @@ impl ScalarValue {ScalarValue::List(arr) => arr.len() == arr.null_count(),ScalarValue::LargeList(arr) => arr.len() == arr.null_count(),ScalarValue::FixedSizeList(arr) => arr.len() == arr.null_count(),+ ScalarValue::ListView(arr) => arr.len() == arr.null_count(),+ ScalarValue::LargeListView(arr) => arr.len() == arr.null_count(),ScalarValue::Struct(arr) => arr.len() == arr.null_count(),ScalarValue::Map(arr) => arr.len() == arr.null_count(),ScalarValue::Date32(v) => v.is_none(),@@ -2841,6 +2897,8 @@ impl ScalarValue {}DataType::List(_)| DataType::LargeList(_)+ | DataType::ListView(_)+ | DataType::LargeListView(_)| DataType::Map(_, _)| DataType::Struct(_)| DataType::Union(_, _) => {@@ -2997,9 +3055,7 @@ impl ScalarValue {DataType::Time32(TimeUnit::Microsecond)| DataType::Time32(TimeUnit::Nanosecond)| DataType::Time64(TimeUnit::Second)- | DataType::Time64(TimeUnit::Millisecond)- | DataType::ListView(_)- | DataType::LargeListView(_) => {+ | DataType::Time64(TimeUnit::Millisecond) => {return _not_impl_err!("Unsupported creation of {:?} array from ScalarValue {:?}",data_type,@@ -3420,6 +3476,18 @@ impl ScalarValue {}Self::list_to_array_of_size(arr.as_ref() as &dyn Array, size)?}+ ScalarValue::ListView(arr) => {+ if size == 1 {+ return Ok(Arc::clone(arr) as Arc<dyn Array>);+ }+ Self::list_to_array_of_size(arr.as_ref() as &dyn Array, size)?+ }+ ScalarValue::LargeListView(arr) => {+ if size == 1 {+ return Ok(Arc::clone(arr) as Arc<dyn Array>);+ }+ Self::list_to_array_of_size(arr.as_ref() as &dyn Array, size)?+ }ScalarValue::Struct(arr) => {if size == 1 {return Ok(Arc::clone(arr) as Arc<dyn Array>);@@ -3797,29 +3865,35 @@ impl ScalarValue {pub fn convert_array_to_scalar_vec(array: &dyn Array,) -> Result<Vec<Option<Vec<Self>>>> {- fn generic_collect<OffsetSize: OffsetSizeTrait>(- array: &dyn Array,- ) -> Result<Vec<Option<Vec<ScalarValue>>>> {- array- .as_list::<OffsetSize>()- .iter()- .map(|nested_array| {- nested_array- .map(|array| {- (0..array.len())- .map(|i| ScalarValue::try_from_array(&array, i))- .collect::<Result<Vec<_>>>()- })- .transpose()+ fn map_element(+ nested_array: Option<ArrayRef>,+ ) -> Result<Option<Vec<ScalarValue>>> {+ nested_array+ .map(|array| {+ (0..array.len())+ .map(|i| ScalarValue::try_from_array(&array, i))+ .collect::<Result<Vec<_>>>()})- .collect()+ .transpose()}match array.data_type() {- DataType::List(_) => generic_collect::<i32>(array),- DataType::LargeList(_) => generic_collect::<i64>(array),+ DataType::List(_) => array.as_list::<i32>().iter().map(map_element).collect(),+ DataType::LargeList(_) => {+ array.as_list::<i64>().iter().map(map_element).collect()+ }+ DataType::ListView(_) => array+ .as_list_view::<i32>()+ .iter()+ .map(map_element)+ .collect(),+ DataType::LargeListView(_) => array+ .as_list_view::<i64>()+ .iter()+ .map(map_element)+ .collect(),_ => _internal_err!(- "ScalarValue::convert_array_to_scalar_vec input must be a List/LargeList type"+ "ScalarValue::convert_array_to_scalar_vec input must be a List/LargeList/ListView/LargeListView type"),}}@@ -3917,20 +3991,20 @@ impl ScalarValue {.build_fixed_size_list_scalar(list_size)}DataType::ListView(field) => {- let list_array = array.as_list_view::<i32>();+ let list_array = as_list_view_array(array)?;let nested_array = list_array.value(index);- // Store as List scalar since ScalarValue has no ListView variant.+ // Produces a single element `ListViewArray` with the value at `index`.SingleRowListArrayBuilder::new(nested_array).with_field(field)- .build_list_scalar()+ .build_list_view_scalar()}DataType::LargeListView(field) => {- let list_array = array.as_list_view::<i64>();+ let list_array = as_large_list_view_array(array)?;let nested_array = list_array.value(index);- // Store as LargeList scalar since ScalarValue has no LargeListView variant.+ // Produces a single element `LargeListViewArray` with the value at `index`.SingleRowListArrayBuilder::new(nested_array).with_field(field)- .build_large_list_scalar()+ .build_large_list_view_scalar()}DataType::Date32 => typed_cast!(array, index, as_date32_array, Date32)?,DataType::Date64 => typed_cast!(array, index, as_date64_array, Date64)?,@@ -4387,6 +4461,12 @@ impl ScalarValue {ScalarValue::FixedSizeList(arr) => {Self::eq_array_list(&(arr.to_owned() as ArrayRef), array, index)}+ ScalarValue::ListView(arr) => {+ Self::eq_array_list(&(arr.to_owned() as ArrayRef), array, index)+ }+ ScalarValue::LargeListView(arr) => {+ Self::eq_array_list(&(arr.to_owned() as ArrayRef), array, index)+ }ScalarValue::Struct(arr) => {Self::eq_array_list(&(arr.to_owned() as ArrayRef), array, index)}@@ -4574,6 +4654,8 @@ impl ScalarValue {ScalarValue::List(arr) => arr.get_array_memory_size(),ScalarValue::LargeList(arr) => arr.get_array_memory_size(),ScalarValue::FixedSizeList(arr) => arr.get_array_memory_size(),+ ScalarValue::ListView(arr) => arr.get_array_memory_size(),+ ScalarValue::LargeListView(arr) => arr.get_array_memory_size(),ScalarValue::Struct(arr) => arr.get_array_memory_size(),ScalarValue::Map(arr) => arr.get_array_memory_size(),ScalarValue::Union(vals, fields, _mode) => {@@ -4690,6 +4772,14 @@ impl ScalarValue {let array = copy_array_data(&arr.to_data());*Arc::make_mut(arr) = LargeListArray::from(array)}+ ScalarValue::ListView(arr) => {+ let array = copy_array_data(&arr.to_data());+ *Arc::make_mut(arr) = ListViewArray::from(array);+ }+ ScalarValue::LargeListView(arr) => {+ let array = copy_array_data(&arr.to_data());+ *Arc::make_mut(arr) = LargeListViewArray::from(array)+ }ScalarValue::Struct(arr) => {let array = copy_array_data(&arr.to_data());*Arc::make_mut(arr) = StructArray::from(array);@@ -5227,6 +5317,8 @@ impl fmt::Display for ScalarValue {ScalarValue::List(arr) => fmt_list(arr.as_ref(), f)?,ScalarValue::LargeList(arr) => fmt_list(arr.as_ref(), f)?,ScalarValue::FixedSizeList(arr) => fmt_list(arr.as_ref(), f)?,+ ScalarValue::ListView(arr) => fmt_list(arr.as_ref(), f)?,+ ScalarValue::LargeListView(arr) => fmt_list(arr.as_ref(), f)?,ScalarValue::Date32(e) => format_option!(f,e.map(|v| {@@ -5350,7 +5442,7 @@ impl fmt::Display for ScalarValue {}fn fmt_list(arr: &dyn Array, f: &mut fmt::Formatter) -> fmt::Result {- // ScalarValue List, LargeList, FixedSizeList should always have a single element+ // ScalarValue List, LargeList, FixedSizeList, ListView, LargeListView should always have a single elementassert_eq!(arr.len(), 1);let options = FormatOptions::default().with_display_error(true);let formatter = ArrayFormatter::try_new(arr, &options).unwrap();@@ -5436,6 +5528,8 @@ impl fmt::Debug for ScalarValue {ScalarValue::FixedSizeList(_) => write!(f, "FixedSizeList({self})"),ScalarValue::List(_) => write!(f, "List({self})"),ScalarValue::LargeList(_) => write!(f, "LargeList({self})"),+ ScalarValue::ListView(_) => write!(f, "ListView({self})"),+ ScalarValue::LargeListView(_) => write!(f, "LargeListView({self})"),ScalarValue::Struct(struct_arr) => {// ScalarValue Struct should always have a single elementassert_eq!(struct_arr.len(), 1);@@ -5575,12 +5669,15 @@ impl ScalarType<i32> for Date32Type {mod tests {use super::*;- use crate::cast::{as_list_array, as_map_array, as_struct_array};+ use crate::cast::{+ as_large_list_view_array, as_list_array, as_map_array, as_struct_array,+ };use crate::test_util::batches_to_string;use arrow::array::{- FixedSizeListBuilder, Int32Builder, LargeListBuilder, ListBuilder, MapBuilder,- NullArray, NullBufferBuilder, OffsetSizeTrait, PrimitiveBuilder, RecordBatch,- StringBuilder, StringDictionaryBuilder, StructBuilder, UnionBuilder,+ FixedSizeListBuilder, Int32Builder, LargeListBuilder, LargeListViewBuilder,+ ListBuilder, ListViewBuilder, MapBuilder, NullArray, NullBufferBuilder,+ OffsetSizeTrait, PrimitiveBuilder, RecordBatch, StringBuilder,+ StringDictionaryBuilder, StructBuilder, UnionBuilder,};use arrow::buffer::{Buffer, NullBuffer, OffsetBuffer};use arrow::compute::{is_null, kernels};@@ -5766,6 +5863,27 @@ mod tests {]);assert_eq!(&arr, actual_list_arr);++ // ListView+ let arr =+ ListViewArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(vec![+ Some(1),+ None,+ Some(2),+ ])]);++ let sv = ScalarValue::ListView(Arc::new(arr));+ let actual_arr = sv+ .to_array_of_size(2)+ .expect("Failed to convert to array of size");+ let actual_list_arr = actual_arr.as_list_view::<i32>();++ let arr = ListViewArray::from_iter_primitive::<Int32Type, _, _>(vec![+ Some(vec![Some(1), None, Some(2)]),+ Some(vec![Some(1), None, Some(2)]),+ ]);++ assert_eq!(&arr, actual_list_arr);}#[test]@@ -5905,29 +6023,11 @@ mod tests {values.into_iter().map(|v| {- let arr = if v.is_some() {- Arc::new(- GenericListArray::<O>::from_iter_primitive::<Int64Type, _, _>(- vec![v],- ),- )- } else if O::IS_LARGE {- new_null_array(- &DataType::LargeList(Arc::new(Field::new_list_field(- DataType::Int64,- true,- ))),- 1,- )- } else {- new_null_array(- &DataType::List(Arc::new(Field::new_list_field(- DataType::Int64,- true,- ))),- 1,- )- };+ let arr = Arc::new(GenericListArray::<O>::from_iter_primitive::<+ Int64Type,+ _,+ _,+ >(vec![v])) as ArrayRef;if O::IS_LARGE {ScalarValue::LargeList(arr.as_list::<i64>().to_owned().into())@@ -5938,6 +6038,29 @@ mod tests {.collect()}+ fn build_list_view<O: OffsetSizeTrait>(+ values: Vec<Option<Vec<Option<i64>>>>,+ ) -> Vec<ScalarValue> {+ values+ .into_iter()+ .map(|v| {+ let arr = Arc::new(GenericListViewArray::<O>::from_iter_primitive::<+ Int64Type,+ _,+ _,+ >(vec![v])) as ArrayRef;++ if O::IS_LARGE {+ ScalarValue::LargeListView(+ arr.as_list_view::<i64>().to_owned().into(),+ )+ } else {+ ScalarValue::ListView(arr.as_list_view::<i32>().to_owned().into())+ }+ })+ .collect()+ }+#[test]fn test_iter_to_array_fixed_size_list() {let field = Arc::new(Field::new_list_field(DataType::Int32, true));@@ -6066,13 +6189,13 @@ mod tests {#[test]fn iter_to_array_primitive_test() {+ // List// List[[1,2,3]], List[null], List[[4,5]]let scalars = build_list::<i32>(vec![Some(vec![Some(1), Some(2), Some(3)]),None,Some(vec![Some(4), Some(5)]),]);-let array = ScalarValue::iter_to_array(scalars).unwrap();let list_array = as_list_array(&array).unwrap();// List[[1,2,3], null, [4,5]]@@ -6083,20 +6206,57 @@ mod tests {]);assert_eq!(list_array, &expected);+ // LargeList+ // List[[1,2,3]], List[null], List[[4,5]]let scalars = build_list::<i64>(vec![Some(vec![Some(1), Some(2), Some(3)]),None,Some(vec![Some(4), Some(5)]),]);-let array = ScalarValue::iter_to_array(scalars).unwrap();- let list_array = as_large_list_array(&array).unwrap();+ let large_list_array = as_large_list_array(&array).unwrap();let expected = LargeListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![Some(1), Some(2), Some(3)]),None,Some(vec![Some(4), Some(5)]),]);- assert_eq!(list_array, &expected);+ assert_eq!(large_list_array, &expected);++ // ListView+ // ListView[[1,2,3]], ListView[null], ListView[[4,5]]+ let scalars = build_list_view::<i32>(vec![+ Some(vec![Some(1), Some(2), Some(3)]),+ None,+ Some(vec![Some(4), Some(5)]),+ ]);++ let array = ScalarValue::iter_to_array(scalars).unwrap();+ let list_view_array = as_list_view_array(&array).unwrap();+ // ListView[[1,2,3], null, [4,5]]+ let expected = ListViewArray::from_iter_primitive::<Int64Type, _, _>(vec![+ Some(vec![Some(1), Some(2), Some(3)]),+ None,+ Some(vec![Some(4), Some(5)]),+ ]);+ assert_eq!(list_view_array, &expected);++ // LargeListView+ // LargeListView[[1,2,3]], LargeListView[null], LargeListView[[4,5]]+ let scalars = build_list_view::<i64>(vec![+ Some(vec![Some(1), Some(2), Some(3)]),+ None,+ Some(vec![Some(4), Some(5)]),+ ]);++ let array = ScalarValue::iter_to_array(scalars).unwrap();+ let large_list_view_array = as_large_list_view_array(&array).unwrap();+ // LargeListView[[1,2,3], null, [4,5]]+ let expected = LargeListViewArray::from_iter_primitive::<Int64Type, _, _>(vec![+ Some(vec![Some(1), Some(2), Some(3)]),+ None,+ Some(vec![Some(4), Some(5)]),+ ]);+ assert_eq!(large_list_view_array, &expected);}#[test]@@ -6139,16 +6299,36 @@ mod tests {]));let fsl_array: ArrayRef =- Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(vec![+ Arc::new(FixedSizeListArray::from_iter_primitive::<Int32Type, _, _>(+ vec![+ Some(vec![Some(0), Some(1), Some(2)]),+ None,+ Some(vec![Some(3), None, Some(5)]),+ ],+ 3,+ ));+ let list_view_array: ArrayRef =+ Arc::new(ListViewArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(vec![Some(0), Some(1), Some(2)]),None,- Some(vec![Some(3), None, Some(5)]),+ Some(vec![None, Some(5)]),]));- for arr in [list_array, fsl_array] {+ for arr in [list_array, fsl_array, list_view_array] {for i in 0..arr.len() {- let scalar =- ScalarValue::List(arr.slice(i, 1).as_list::<i32>().to_owned().into());+ let slice = arr.slice(i, 1);+ let scalar = match arr.data_type() {+ DataType::List(_) => {+ ScalarValue::List(slice.as_list::<i32>().to_owned().into())+ }+ DataType::FixedSizeList(_, _) => ScalarValue::FixedSizeList(+ slice.as_fixed_size_list().to_owned().into(),+ ),+ DataType::ListView(_) => ScalarValue::ListView(+ slice.as_list_view::<i32>().to_owned().into(),+ ),+ _ => unreachable!(),+ };assert!(scalar.eq_array(&arr, i).unwrap());}}@@ -6847,6 +7027,40 @@ mod tests {),));assert_eq!(a.partial_cmp(&b), Some(Ordering::Greater));++ let a = ScalarValue::ListView(Arc::new(ListViewArray::from_iter_primitive::<+ Int64Type,+ _,+ _,+ >(vec![Some(vec![+ None,+ Some(2),+ Some(3),+ ])])));+ let b = ScalarValue::ListView(Arc::new(ListViewArray::from_iter_primitive::<+ Int64Type,+ _,+ _,+ >(vec![Some(vec![+ Some(1),+ Some(2),+ Some(3),+ ])])));+ assert_eq!(a.partial_cmp(&b), Some(Ordering::Greater));++ let a =+ ScalarValue::LargeListView(Arc::new(+ LargeListViewArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(+ vec![None, Some(2), Some(3)],+ )]),+ ));+ let b =+ ScalarValue::LargeListView(Arc::new(+ LargeListViewArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(+ vec![Some(1), Some(2), Some(3)],+ )]),+ ));+ assert_eq!(a.partial_cmp(&b), Some(Ordering::Greater));}#[test]@@ -7194,6 +7408,30 @@ mod tests {);assert_eq!(expected, scalar);assert!(expected.is_null());++ // Test for ListView+ let data_type = &DataType::ListView(Arc::clone(&inner_field));+ let scalar: ScalarValue = data_type.try_into().unwrap();+ let expected = ScalarValue::ListView(+ new_null_array(data_type, 1)+ .as_list_view::<i32>()+ .to_owned()+ .into(),+ );+ assert_eq!(expected, scalar);+ assert!(expected.is_null());++ // Test for LargeListView+ let data_type = &DataType::LargeListView(Arc::clone(&inner_field));+ let scalar: ScalarValue = data_type.try_into().unwrap();+ let expected = ScalarValue::LargeListView(+ new_null_array(data_type, 1)+ .as_list_view::<i64>()+ .to_owned()+ .into(),+ );+ assert_eq!(expected, scalar);+ assert!(expected.is_null());}#[test]@@ -7908,6 +8146,34 @@ mod tests {builder.append(true);Arc::new(builder.finish())},+ // list view array+ {+ let values_builder = StringBuilder::new();+ let mut builder = ListViewBuilder::new(values_builder);+ // [A, B]+ builder.values().append_value("A");+ builder.values().append_value("B");+ builder.append(true);+ // [ ] (empty list)+ builder.append(true);+ // Null+ builder.append(false);+ Arc::new(builder.finish())+ },+ // large list view array+ {+ let values_builder = StringBuilder::new();+ let mut builder = LargeListViewBuilder::new(values_builder);+ // [A, B]+ builder.values().append_value("A");+ builder.values().append_value("B");+ builder.append(true);+ // [ ] (empty list)+ builder.append(true);+ // Null+ builder.append(false);+ Arc::new(builder.finish())+ },// map{let string_builder = StringBuilder::new();@@ -8430,6 +8696,38 @@ mod tests {},DataType::LargeList(Arc::new(Field::new("element", DataType::Int64, true))),);+ check_scalar_cast(+ {+ let element_field =+ Arc::new(Field::new("element", DataType::Int32, true));++ let mut builder =+ ListViewBuilder::new(Int32Builder::new()).with_field(element_field);+ builder.append_value([Some(1)]);+ builder.append(true);++ ScalarValue::ListView(Arc::new(builder.finish()))+ },+ DataType::ListView(Arc::new(Field::new("element", DataType::Int64, true))),+ );+ check_scalar_cast(+ {+ let element_field =+ Arc::new(Field::new("element", DataType::Int32, true));++ let mut builder = LargeListViewBuilder::new(Int32Builder::new())+ .with_field(element_field);+ builder.append_value([Some(1)]);+ builder.append(true);++ ScalarValue::LargeListView(Arc::new(builder.finish()))+ },+ DataType::LargeListView(Arc::new(Field::new(+ "element",+ DataType::Int64,+ true,+ ))),+ );}// mimics how casting work on scalar values by `casting` `scalar` to `desired_type`@@ -9089,7 +9387,7 @@ mod tests {)))),];- let check_array = |array| {+ let check_array = |array: Arc<dyn Array>| {let is_null = is_null(&array).unwrap();assert_eq!(is_null, BooleanArray::from(vec![true, false, false]));@@ -9154,6 +9452,21 @@ mod tests {");}+ #[test]+ fn test_list_view_display() {+ let s = ScalarValue::ListView(+ ListViewArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![+ Some(1),+ None,+ Some(3),+ ])])+ .into(),+ );++ assert_eq!(s.to_string(), "[1, , 3]");+ assert_eq!(format!("{s:?}"), "ListView([1, , 3])");+ }+#[test]fn test_null_bug() {let field_a = Field::new("a", DataType::Int32, true);@@ -9612,6 +9925,10 @@ mod tests {42,)).unwrap(),+ ScalarValue::try_new_null(&DataType::ListView(Arc::clone(&field_ref)))+ .unwrap(),+ ScalarValue::try_new_null(&DataType::LargeListView(Arc::clone(&field_ref)))+ .unwrap(),ScalarValue::try_new_null(&DataType::Struct(vec![Arc::clone(&field_ref)].into(),))@@ -9704,6 +10021,41 @@ mod tests {_ => panic!("Expected List"),}+ let list_field = Field::new_list_field(DataType::Int32, true);+ let list_result =+ ScalarValue::new_default(&DataType::LargeList(Arc::new(list_field.clone())))+ .unwrap();+ match list_result {+ ScalarValue::LargeList(arr) => {+ assert_eq!(arr.len(), 1);+ assert_eq!(arr.value_length(0), 0); // empty list+ }+ _ => panic!("Expected LargeList"),+ }++ let list_result =+ ScalarValue::new_default(&DataType::ListView(Arc::new(list_field.clone())))+ .unwrap();+ match list_result {+ ScalarValue::ListView(arr) => {+ assert_eq!(arr.len(), 1);+ assert_eq!(arr.value_size(0), 0); // empty list+ }+ _ => panic!("Expected ListView"),+ }++ let list_result = ScalarValue::new_default(&DataType::LargeListView(Arc::new(+ list_field.clone(),+ )))+ .unwrap();+ match list_result {+ ScalarValue::LargeListView(arr) => {+ assert_eq!(arr.len(), 1);+ assert_eq!(arr.value_size(0), 0); // empty list+ }+ _ => panic!("Expected LargeListView"),+ }+// Test struct typelet struct_fields = Fields::from(vec![Field::new("a", DataType::Int32, false),@@ -9813,6 +10165,30 @@ mod tests {)))),None);+ assert_eq!(+ ScalarValue::min(&DataType::LargeList(Arc::new(Field::new(+ "item",+ DataType::Int32,+ true+ )))),+ None+ );+ assert_eq!(+ ScalarValue::min(&DataType::ListView(Arc::new(Field::new(+ "item",+ DataType::Int32,+ true+ )))),+ None+ );+ assert_eq!(+ ScalarValue::min(&DataType::LargeListView(Arc::new(Field::new(+ "item",+ DataType::Int32,+ true+ )))),+ None+ );}#[test]@@ -9889,6 +10265,22 @@ mod tests {)]))),None);+ assert_eq!(+ ScalarValue::max(&DataType::ListView(Arc::new(Field::new(+ "item",+ DataType::Int32,+ true+ )))),+ None+ );+ assert_eq!(+ ScalarValue::max(&DataType::LargeListView(Arc::new(Field::new(+ "item",+ DataType::Int32,+ true+ )))),+ None+ );}#[test]@@ -10267,5 +10659,52 @@ mod tests {]),]);++ // 6: Regular ListViewArray+ let list = ListViewArray::from_iter_primitive::<Int64Type, _, _>(vec![+ Some(vec![Some(1), Some(2)]),+ None,+ Some(vec![Some(3), None, Some(4)]),+ ]);+ let converted = ScalarValue::convert_array_to_scalar_vec(&list).unwrap();+ assert_eq!(+ converted,+ vec![+ Some(vec![+ ScalarValue::Int64(Some(1)),+ ScalarValue::Int64(Some(2))+ ]),+ None,+ Some(vec![+ ScalarValue::Int64(Some(3)),+ ScalarValue::Int64(None),+ ScalarValue::Int64(Some(4))+ ]),+ ]+ );++ // 7: Regular LargeListViewArray+ let large_list =+ LargeListViewArray::from_iter_primitive::<Int64Type, _, _>(vec![+ Some(vec![Some(1), Some(2)]),+ None,+ Some(vec![Some(3), None, Some(4)]),+ ]);+ let converted = ScalarValue::convert_array_to_scalar_vec(&large_list).unwrap();+ assert_eq!(+ converted,+ vec![+ Some(vec![+ ScalarValue::Int64(Some(1)),+ ScalarValue::Int64(Some(2))+ ]),+ None,+ Some(vec![+ ScalarValue::Int64(Some(3)),+ ScalarValue::Int64(None),+ ScalarValue::Int64(Some(4))+ ]),+ ]+ );}}diff --git a/datafusion/common/src/utils/mod.rs b/datafusion/common/src/utils/mod.rsindex 9752574002..2f01859f45 100644--- a/datafusion/common/src/utils/mod.rs+++ b/datafusion/common/src/utils/mod.rs@@ -30,7 +30,8 @@ use arrow::array::{Array, ArrayRef, FixedSizeListArray, LargeListArray, ListArray, OffsetSizeTrait,cast::AsArray,};-use arrow::buffer::OffsetBuffer;+use arrow::array::{LargeListViewArray, ListViewArray};+use arrow::buffer::{OffsetBuffer, ScalarBuffer};use arrow::compute::{SortColumn, SortOptions, partition};use arrow::datatypes::{DataType, Field, SchemaRef};#[cfg(feature = "sql")]@@ -480,6 +481,34 @@ impl SingleRowListArrayBuilder {ScalarValue::FixedSizeList(Arc::new(self.build_fixed_size_list_array(list_size)))}+ /// Build a single element [`ListViewArray`]+ pub fn build_list_view_array(self) -> ListViewArray {+ let (field, arr) = self.into_field_and_arr();+ let offsets = ScalarBuffer::from(vec![0]);+ let sizes = ScalarBuffer::from(vec![i32::try_from(arr.len()).expect(+ "Trying to construct a ListView where element length exceeds i32::MAX",+ )]);+ ListViewArray::new(field, offsets, sizes, arr, None)+ }++ /// Build a single element [`ListViewArray`] and wrap as [`ScalarValue::ListView`]+ pub fn build_list_view_scalar(self) -> ScalarValue {+ ScalarValue::ListView(Arc::new(self.build_list_view_array()))+ }++ /// Build a single element [`LargeListViewArray`]+ pub fn build_large_list_view_array(self) -> LargeListViewArray {+ let (field, arr) = self.into_field_and_arr();+ let offsets = ScalarBuffer::from(vec![0]);+ let sizes = ScalarBuffer::from(vec![arr.len() as i64]);+ LargeListViewArray::new(field, offsets, sizes, arr, None)+ }++ /// Build a single element [`LargeListViewArray`] and wrap as [`ScalarValue::LargeListView`]+ pub fn build_large_list_view_scalar(self) -> ScalarValue {+ ScalarValue::LargeListView(Arc::new(self.build_large_list_view_array()))+ }+/// Helper function: convert this builder into a tuple of field and arrayfn into_field_and_arr(self) -> (Arc<Field>, ArrayRef) {let Self {diff --git a/datafusion/proto-common/proto/datafusion_common.proto b/datafusion/proto-common/proto/datafusion_common.protoindex 31ece63577..53850e3146 100644--- a/datafusion/proto-common/proto/datafusion_common.proto+++ b/datafusion/proto-common/proto/datafusion_common.proto@@ -199,7 +199,7 @@ message Union{repeated int32 type_ids = 3;}-// Used for List/FixedSizeList/LargeList/Struct/Map+// Used for List/FixedSizeList/LargeList/ListView/LargeListView/Struct/Mapmessage ScalarNestedValue {message Dictionary {bytes ipc_message = 1;@@ -306,6 +306,8 @@ message ScalarValue{ScalarNestedValue large_list_value = 16;ScalarNestedValue list_value = 17;ScalarNestedValue fixed_size_list_value = 18;+ ScalarNestedValue list_view_value = 46;+ ScalarNestedValue large_list_view_value = 47;ScalarNestedValue struct_value = 32;ScalarNestedValue map_value = 41;@@ -398,6 +400,8 @@ message ArrowType{List LIST = 25;List LARGE_LIST = 26;FixedSizeList FIXED_SIZE_LIST = 27;+ List LIST_VIEW = 43;+ List LARGE_LIST_VIEW = 44;Struct STRUCT = 28;Union UNION = 29;Dictionary DICTIONARY = 30;diff --git a/datafusion/proto-common/src/from_proto/mod.rs b/datafusion/proto-common/src/from_proto/mod.rsindex 4b7a91f38c..5913a2c8f3 100644--- a/datafusion/proto-common/src/from_proto/mod.rs+++ b/datafusion/proto-common/src/from_proto/mod.rs@@ -296,6 +296,16 @@ impl TryFrom<&protobuf::arrow_type::ArrowTypeEnum> for DataType {let list_size = list.list_size;DataType::FixedSizeList(Arc::new(list_type), list_size)}+ arrow_type::ArrowTypeEnum::ListView(list) => {+ let list_type =+ list.as_ref().field_type.as_deref().required("field_type")?;+ DataType::ListView(Arc::new(list_type))+ }+ arrow_type::ArrowTypeEnum::LargeListView(list) => {+ let list_type =+ list.as_ref().field_type.as_deref().required("field_type")?;+ DataType::LargeListView(Arc::new(list_type))+ }arrow_type::ArrowTypeEnum::Struct(strct) => DataType::Struct(parse_proto_fields_to_fields(&strct.sub_field_types)?.into(),),@@ -405,6 +415,8 @@ impl TryFrom<&protobuf::ScalarValue> for ScalarValue {Value::ListValue(v)| Value::FixedSizeListValue(v)| Value::LargeListValue(v)+ | Value::ListViewValue(v)+ | Value::LargeListViewValue(v)| Value::StructValue(v)| Value::MapValue(v) => {let protobuf::ScalarNestedValue {@@ -517,6 +529,12 @@ impl TryFrom<&protobuf::ScalarValue> for ScalarValue {Value::FixedSizeListValue(_) => {Self::FixedSizeList(arr.as_fixed_size_list().to_owned().into())}+ Value::ListViewValue(_) => {+ Self::ListView(arr.as_list_view::<i32>().to_owned().into())+ }+ Value::LargeListViewValue(_) => {+ Self::LargeListView(arr.as_list_view::<i64>().to_owned().into())+ }Value::StructValue(_) => {Self::Struct(arr.as_struct().to_owned().into())}diff --git a/datafusion/proto-common/src/generated/pbjson.rs b/datafusion/proto-common/src/generated/pbjson.rsindex 77a3b71488..950b5a6f63 100644--- a/datafusion/proto-common/src/generated/pbjson.rs+++ b/datafusion/proto-common/src/generated/pbjson.rs@@ -264,6 +264,12 @@ impl serde::Serialize for ArrowType {arrow_type::ArrowTypeEnum::FixedSizeList(v) => {struct_ser.serialize_field("FIXEDSIZELIST", v)?;}+ arrow_type::ArrowTypeEnum::ListView(v) => {+ struct_ser.serialize_field("LISTVIEW", v)?;+ }+ arrow_type::ArrowTypeEnum::LargeListView(v) => {+ struct_ser.serialize_field("LARGELISTVIEW", v)?;+ }arrow_type::ArrowTypeEnum::Struct(v) => {struct_ser.serialize_field("STRUCT", v)?;}@@ -332,6 +338,10 @@ impl<'de> serde::Deserialize<'de> for ArrowType {"LARGELIST","FIXED_SIZE_LIST","FIXEDSIZELIST",+ "LIST_VIEW",+ "LISTVIEW",+ "LARGE_LIST_VIEW",+ "LARGELISTVIEW","STRUCT","UNION","DICTIONARY",@@ -376,6 +386,8 @@ impl<'de> serde::Deserialize<'de> for ArrowType {List,LargeList,FixedSizeList,+ ListView,+ LargeListView,Struct,Union,Dictionary,@@ -436,6 +448,8 @@ impl<'de> serde::Deserialize<'de> for ArrowType {"LIST" => Ok(GeneratedField::List),"LARGELIST" | "LARGE_LIST" => Ok(GeneratedField::LargeList),"FIXEDSIZELIST" | "FIXED_SIZE_LIST" => Ok(GeneratedField::FixedSizeList),+ "LISTVIEW" | "LIST_VIEW" => Ok(GeneratedField::ListView),+ "LARGELISTVIEW" | "LARGE_LIST_VIEW" => Ok(GeneratedField::LargeListView),"STRUCT" => Ok(GeneratedField::Struct),"UNION" => Ok(GeneratedField::Union),"DICTIONARY" => Ok(GeneratedField::Dictionary),@@ -694,6 +708,20 @@ impl<'de> serde::Deserialize<'de> for ArrowType {return Err(serde::de::Error::duplicate_field("FIXEDSIZELIST"));}arrow_type_enum__ = map_.next_value::<::std::option::Option<_>>()?.map(arrow_type::ArrowTypeEnum::FixedSizeList)+;+ }+ GeneratedField::ListView => {+ if arrow_type_enum__.is_some() {+ return Err(serde::de::Error::duplicate_field("LISTVIEW"));+ }+ arrow_type_enum__ = map_.next_value::<::std::option::Option<_>>()?.map(arrow_type::ArrowTypeEnum::ListView)+;+ }+ GeneratedField::LargeListView => {+ if arrow_type_enum__.is_some() {+ return Err(serde::de::Error::duplicate_field("LARGELISTVIEW"));+ }+ arrow_type_enum__ = map_.next_value::<::std::option::Option<_>>()?.map(arrow_type::ArrowTypeEnum::LargeListView);}GeneratedField::Struct => {@@ -7978,6 +8006,12 @@ impl serde::Serialize for ScalarValue {scalar_value::Value::FixedSizeListValue(v) => {struct_ser.serialize_field("fixedSizeListValue", v)?;}+ scalar_value::Value::ListViewValue(v) => {+ struct_ser.serialize_field("listViewValue", v)?;+ }+ scalar_value::Value::LargeListViewValue(v) => {+ struct_ser.serialize_field("largeListViewValue", v)?;+ }scalar_value::Value::StructValue(v) => {struct_ser.serialize_field("structValue", v)?;}@@ -8115,6 +8149,10 @@ impl<'de> serde::Deserialize<'de> for ScalarValue {"listValue","fixed_size_list_value","fixedSizeListValue",+ "list_view_value",+ "listViewValue",+ "large_list_view_value",+ "largeListViewValue","struct_value","structValue","map_value",@@ -8185,6 +8223,8 @@ impl<'de> serde::Deserialize<'de> for ScalarValue {LargeListValue,ListValue,FixedSizeListValue,+ ListViewValue,+ LargeListViewValue,StructValue,MapValue,Decimal32Value,@@ -8249,6 +8289,8 @@ impl<'de> serde::Deserialize<'de> for ScalarValue {"largeListValue" | "large_list_value" => Ok(GeneratedField::LargeListValue),"listValue" | "list_value" => Ok(GeneratedField::ListValue),"fixedSizeListValue" | "fixed_size_list_value" => Ok(GeneratedField::FixedSizeListValue),+ "listViewValue" | "list_view_value" => Ok(GeneratedField::ListViewValue),+ "largeListViewValue" | "large_list_view_value" => Ok(GeneratedField::LargeListViewValue),"structValue" | "struct_value" => Ok(GeneratedField::StructValue),"mapValue" | "map_value" => Ok(GeneratedField::MapValue),"decimal32Value" | "decimal32_value" => Ok(GeneratedField::Decimal32Value),@@ -8417,6 +8459,20 @@ impl<'de> serde::Deserialize<'de> for ScalarValue {return Err(serde::de::Error::duplicate_field("fixedSizeListValue"));}value__ = map_.next_value::<::std::option::Option<_>>()?.map(scalar_value::Value::FixedSizeListValue)+;+ }+ GeneratedField::ListViewValue => {+ if value__.is_some() {+ return Err(serde::de::Error::duplicate_field("listViewValue"));+ }+ value__ = map_.next_value::<::std::option::Option<_>>()?.map(scalar_value::Value::ListViewValue)+;+ }+ GeneratedField::LargeListViewValue => {+ if value__.is_some() {+ return Err(serde::de::Error::duplicate_field("largeListViewValue"));+ }+ value__ = map_.next_value::<::std::option::Option<_>>()?.map(scalar_value::Value::LargeListViewValue);}GeneratedField::StructValue => {diff --git a/datafusion/proto-common/src/generated/prost.rs b/datafusion/proto-common/src/generated/prost.rsindex 1251a51ab0..aac131e57e 100644--- a/datafusion/proto-common/src/generated/prost.rs+++ b/datafusion/proto-common/src/generated/prost.rs@@ -191,7 +191,7 @@ pub struct Union {#[prost(int32, repeated, tag = "3")]pub type_ids: ::prost::alloc::vec::Vec<i32>,}-/// Used for List/FixedSizeList/LargeList/Struct/Map+/// Used for List/FixedSizeList/LargeList/ListView/LargeListView/Struct/Map#[derive(Clone, PartialEq, ::prost::Message)]pub struct ScalarNestedValue {#[prost(bytes = "vec", tag = "1")]@@ -327,7 +327,7 @@ pub struct ScalarFixedSizeBinary {pub struct ScalarValue {#[prost(oneof = "scalar_value::Value",- tags = "33, 1, 2, 3, 23, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 32, 41, 43, 44, 20, 39, 21, 24, 35, 36, 37, 38, 26, 27, 28, 29, 22, 30, 25, 31, 34, 42, 45"+ tags = "33, 1, 2, 3, 23, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 46, 47, 32, 41, 43, 44, 20, 39, 21, 24, 35, 36, 37, 38, 26, 27, 28, 29, 22, 30, 25, 31, 34, 42, 45")]pub value: ::core::option::Option<scalar_value::Value>,}@@ -378,6 +378,10 @@ pub mod scalar_value {ListValue(super::ScalarNestedValue),#[prost(message, tag = "18")]FixedSizeListValue(super::ScalarNestedValue),+ #[prost(message, tag = "46")]+ ListViewValue(super::ScalarNestedValue),+ #[prost(message, tag = "47")]+ LargeListViewValue(super::ScalarNestedValue),#[prost(message, tag = "32")]StructValue(super::ScalarNestedValue),#[prost(message, tag = "41")]@@ -467,7 +471,7 @@ pub struct Decimal256 {pub struct ArrowType {#[prost(oneof = "arrow_type::ArrowTypeEnum",- tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 35, 32, 15, 34, 16, 31, 17, 18, 19, 20, 21, 22, 23, 40, 41, 24, 36, 25, 26, 27, 28, 29, 30, 33, 42"+ tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 35, 32, 15, 34, 16, 31, 17, 18, 19, 20, 21, 22, 23, 40, 41, 24, 36, 25, 26, 27, 43, 44, 28, 29, 30, 33, 42")]pub arrow_type_enum: ::core::option::Option<arrow_type::ArrowTypeEnum>,}@@ -548,6 +552,10 @@ pub mod arrow_type {LargeList(::prost::alloc::boxed::Box<super::List>),#[prost(message, tag = "27")]FixedSizeList(::prost::alloc::boxed::Box<super::FixedSizeList>),+ #[prost(message, tag = "43")]+ ListView(::prost::alloc::boxed::Box<super::List>),+ #[prost(message, tag = "44")]+ LargeListView(::prost::alloc::boxed::Box<super::List>),#[prost(message, tag = "28")]Struct(super::Struct),#[prost(message, tag = "29")]diff --git a/datafusion/proto-common/src/to_proto/mod.rs b/datafusion/proto-common/src/to_proto/mod.rsindex 65089f029b..a2da9b7799 100644--- a/datafusion/proto-common/src/to_proto/mod.rs+++ b/datafusion/proto-common/src/to_proto/mod.rs@@ -171,6 +171,14 @@ impl TryFrom<&DataType> for protobuf::arrow_type::ArrowTypeEnum {DataType::LargeList(item_type) => Self::LargeList(Box::new(protobuf::List {field_type: Some(Box::new(item_type.as_ref().try_into()?)),})),+ DataType::ListView(item_type) => Self::ListView(Box::new(protobuf::List {+ field_type: Some(Box::new(item_type.as_ref().try_into()?)),+ })),+ DataType::LargeListView(item_type) => {+ Self::LargeListView(Box::new(protobuf::List {+ field_type: Some(Box::new(item_type.as_ref().try_into()?)),+ }))+ }DataType::Struct(struct_fields) => Self::Struct(protobuf::Struct {sub_field_types: convert_arc_fields_to_proto_fields(struct_fields)?,}),@@ -227,11 +235,6 @@ impl TryFrom<&DataType> for protobuf::arrow_type::ArrowTypeEnum {values_field: Some(Box::new(values_field.as_ref().try_into()?)),}))}- DataType::ListView(_) | DataType::LargeListView(_) => {- return Err(Error::General(format!(- "Proto serialization error: {val} not yet supported"- )));- }};Ok(res)@@ -383,6 +386,12 @@ impl TryFrom<&ScalarValue> for protobuf::ScalarValue {ScalarValue::FixedSizeList(arr) => {encode_scalar_nested_value(arr.to_owned() as ArrayRef, val)}+ ScalarValue::ListView(arr) => {+ encode_scalar_nested_value(arr.to_owned() as ArrayRef, val)+ }+ ScalarValue::LargeListView(arr) => {+ encode_scalar_nested_value(arr.to_owned() as ArrayRef, val)+ }ScalarValue::Struct(arr) => {encode_scalar_nested_value(arr.to_owned() as ArrayRef, val)}@@ -1042,8 +1051,8 @@ fn create_proto_scalar<I, T: FnOnce(&I) -> protobuf::scalar_value::Value>(Ok(protobuf::ScalarValue { value: Some(value) })}-// Nested ScalarValue types (List / FixedSizeList / LargeList / Struct / Map) are serialized using-// Arrow IPC messages as a single column RecordBatch+// Nested ScalarValue types (List / FixedSizeList / LargeList / ListView / LargeListView / Struct / Map)+// are serialized using Arrow IPC messages as a single column RecordBatchfn encode_scalar_nested_value(arr: ArrayRef,val: &ScalarValue,@@ -1105,6 +1114,16 @@ fn encode_scalar_nested_value(scalar_list_value,)),}),+ ScalarValue::ListView(_) => Ok(protobuf::ScalarValue {+ value: Some(protobuf::scalar_value::Value::ListViewValue(+ scalar_list_value,+ )),+ }),+ ScalarValue::LargeListView(_) => Ok(protobuf::ScalarValue {+ value: Some(protobuf::scalar_value::Value::LargeListViewValue(+ scalar_list_value,+ )),+ }),ScalarValue::Struct(_) => Ok(protobuf::ScalarValue {value: Some(protobuf::scalar_value::Value::StructValue(scalar_list_value,diff --git a/datafusion/proto/src/generated/datafusion_proto_common.rs b/datafusion/proto/src/generated/datafusion_proto_common.rsindex 1251a51ab0..aac131e57e 100644--- a/datafusion/proto/src/generated/datafusion_proto_common.rs+++ b/datafusion/proto/src/generated/datafusion_proto_common.rs@@ -191,7 +191,7 @@ pub struct Union {#[prost(int32, repeated, tag = "3")]pub type_ids: ::prost::alloc::vec::Vec<i32>,}-/// Used for List/FixedSizeList/LargeList/Struct/Map+/// Used for List/FixedSizeList/LargeList/ListView/LargeListView/Struct/Map#[derive(Clone, PartialEq, ::prost::Message)]pub struct ScalarNestedValue {#[prost(bytes = "vec", tag = "1")]@@ -327,7 +327,7 @@ pub struct ScalarFixedSizeBinary {pub struct ScalarValue {#[prost(oneof = "scalar_value::Value",- tags = "33, 1, 2, 3, 23, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 32, 41, 43, 44, 20, 39, 21, 24, 35, 36, 37, 38, 26, 27, 28, 29, 22, 30, 25, 31, 34, 42, 45"+ tags = "33, 1, 2, 3, 23, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 46, 47, 32, 41, 43, 44, 20, 39, 21, 24, 35, 36, 37, 38, 26, 27, 28, 29, 22, 30, 25, 31, 34, 42, 45")]pub value: ::core::option::Option<scalar_value::Value>,}@@ -378,6 +378,10 @@ pub mod scalar_value {ListValue(super::ScalarNestedValue),#[prost(message, tag = "18")]FixedSizeListValue(super::ScalarNestedValue),+ #[prost(message, tag = "46")]+ ListViewValue(super::ScalarNestedValue),+ #[prost(message, tag = "47")]+ LargeListViewValue(super::ScalarNestedValue),#[prost(message, tag = "32")]StructValue(super::ScalarNestedValue),#[prost(message, tag = "41")]@@ -467,7 +471,7 @@ pub struct Decimal256 {pub struct ArrowType {#[prost(oneof = "arrow_type::ArrowTypeEnum",- tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 35, 32, 15, 34, 16, 31, 17, 18, 19, 20, 21, 22, 23, 40, 41, 24, 36, 25, 26, 27, 28, 29, 30, 33, 42"+ tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 35, 32, 15, 34, 16, 31, 17, 18, 19, 20, 21, 22, 23, 40, 41, 24, 36, 25, 26, 27, 43, 44, 28, 29, 30, 33, 42")]pub arrow_type_enum: ::core::option::Option<arrow_type::ArrowTypeEnum>,}@@ -548,6 +552,10 @@ pub mod arrow_type {LargeList(::prost::alloc::boxed::Box<super::List>),#[prost(message, tag = "27")]FixedSizeList(::prost::alloc::boxed::Box<super::FixedSizeList>),+ #[prost(message, tag = "43")]+ ListView(::prost::alloc::boxed::Box<super::List>),+ #[prost(message, tag = "44")]+ LargeListView(::prost::alloc::boxed::Box<super::List>),#[prost(message, tag = "28")]Struct(super::Struct),#[prost(message, tag = "29")]diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rsindex bb91cb029e..6de9dd4caa 100644--- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs@@ -16,7 +16,8 @@// under the License.use arrow::array::{- ArrayRef, FixedSizeListArray, Int32Builder, MapArray, MapBuilder, StringBuilder,+ ArrayRef, FixedSizeListArray, Int32Builder, LargeListViewArray, ListViewArray,+ MapArray, MapBuilder, StringBuilder,};use arrow::datatypes::{DECIMAL256_MAX_PRECISION, DataType, Field, FieldRef, Fields, Int32Type,@@ -1534,263 +1535,277 @@ impl LogicalExtensionCodec for UDFExtensionCodec {#[test]fn round_trip_scalar_values_and_data_types() {- let should_pass: Vec<ScalarValue> = vec![- ScalarValue::Boolean(None),- ScalarValue::Float32(None),- ScalarValue::Float64(None),- ScalarValue::Int8(None),- ScalarValue::Int16(None),- ScalarValue::Int32(None),- ScalarValue::Int64(None),- ScalarValue::UInt8(None),- ScalarValue::UInt16(None),- ScalarValue::UInt32(None),- ScalarValue::UInt64(None),- ScalarValue::Utf8(None),- ScalarValue::LargeUtf8(None),- ScalarValue::List(ScalarValue::new_list_nullable(&[], &DataType::Boolean)),- ScalarValue::LargeList(ScalarValue::new_large_list(&[], &DataType::Boolean)),- ScalarValue::Date32(None),- ScalarValue::Boolean(Some(true)),- ScalarValue::Boolean(Some(false)),- ScalarValue::Float32(Some(1.0)),- ScalarValue::Float32(Some(f32::MAX)),- ScalarValue::Float32(Some(f32::MIN)),- ScalarValue::Float32(Some(-2000.0)),- ScalarValue::Float64(Some(1.0)),- ScalarValue::Float64(Some(f64::MAX)),- ScalarValue::Float64(Some(f64::MIN)),- ScalarValue::Float64(Some(-2000.0)),- ScalarValue::Int8(Some(i8::MIN)),- ScalarValue::Int8(Some(i8::MAX)),- ScalarValue::Int8(Some(0)),- ScalarValue::Int8(Some(-15)),- ScalarValue::Int16(Some(i16::MIN)),- ScalarValue::Int16(Some(i16::MAX)),- ScalarValue::Int16(Some(0)),- ScalarValue::Int16(Some(-15)),- ScalarValue::Int32(Some(i32::MIN)),- ScalarValue::Int32(Some(i32::MAX)),- ScalarValue::Int32(Some(0)),- ScalarValue::Int32(Some(-15)),- ScalarValue::Int64(Some(i64::MIN)),- ScalarValue::Int64(Some(i64::MAX)),- ScalarValue::Int64(Some(0)),- ScalarValue::Int64(Some(-15)),- ScalarValue::UInt8(Some(u8::MAX)),- ScalarValue::UInt8(Some(0)),- ScalarValue::UInt16(Some(u16::MAX)),- ScalarValue::UInt16(Some(0)),- ScalarValue::UInt32(Some(u32::MAX)),- ScalarValue::UInt32(Some(0)),- ScalarValue::UInt64(Some(u64::MAX)),- ScalarValue::UInt64(Some(0)),- ScalarValue::Utf8(Some(String::from("Test string "))),- ScalarValue::LargeUtf8(Some(String::from("Test Large utf8"))),- ScalarValue::Utf8View(Some(String::from("Test stringview"))),- ScalarValue::BinaryView(Some(b"binaryview".to_vec())),- ScalarValue::Date32(Some(0)),- ScalarValue::Date32(Some(i32::MAX)),- ScalarValue::Date32(None),- ScalarValue::Date64(Some(0)),- ScalarValue::Date64(Some(i64::MAX)),- ScalarValue::Date64(None),- ScalarValue::Time32Second(Some(0)),- ScalarValue::Time32Second(Some(i32::MAX)),- ScalarValue::Time32Second(None),- ScalarValue::Time32Millisecond(Some(0)),- ScalarValue::Time32Millisecond(Some(i32::MAX)),- ScalarValue::Time32Millisecond(None),- ScalarValue::Time64Microsecond(Some(0)),- ScalarValue::Time64Microsecond(Some(i64::MAX)),- ScalarValue::Time64Microsecond(None),- ScalarValue::Time64Nanosecond(Some(0)),- ScalarValue::Time64Nanosecond(Some(i64::MAX)),- ScalarValue::Time64Nanosecond(None),- ScalarValue::TimestampNanosecond(Some(0), None),- ScalarValue::TimestampNanosecond(Some(i64::MAX), None),- ScalarValue::TimestampNanosecond(Some(0), Some("UTC".into())),- ScalarValue::TimestampNanosecond(None, None),- ScalarValue::TimestampMicrosecond(Some(0), None),- ScalarValue::TimestampMicrosecond(Some(i64::MAX), None),- ScalarValue::TimestampMicrosecond(Some(0), Some("UTC".into())),- ScalarValue::TimestampMicrosecond(None, None),- ScalarValue::TimestampMillisecond(Some(0), None),- ScalarValue::TimestampMillisecond(Some(i64::MAX), None),- ScalarValue::TimestampMillisecond(Some(0), Some("UTC".into())),- ScalarValue::TimestampMillisecond(None, None),- ScalarValue::TimestampSecond(Some(0), None),- ScalarValue::TimestampSecond(Some(i64::MAX), None),- ScalarValue::TimestampSecond(Some(0), Some("UTC".into())),- ScalarValue::TimestampSecond(None, None),- ScalarValue::IntervalDayTime(Some(IntervalDayTimeType::make_value(0, 0))),- ScalarValue::IntervalDayTime(Some(IntervalDayTimeType::make_value(1, 2))),- ScalarValue::IntervalDayTime(Some(IntervalDayTimeType::make_value(- i32::MAX,- i32::MAX,- ))),- ScalarValue::IntervalDayTime(None),- ScalarValue::IntervalMonthDayNano(Some(IntervalMonthDayNanoType::make_value(- 0, 0, 0,- ))),- ScalarValue::IntervalMonthDayNano(Some(IntervalMonthDayNanoType::make_value(- 1, 2, 3,- ))),- ScalarValue::IntervalMonthDayNano(Some(IntervalMonthDayNanoType::make_value(- i32::MAX,- i32::MAX,- i64::MAX,- ))),- ScalarValue::IntervalMonthDayNano(None),- ScalarValue::List(ScalarValue::new_list_nullable(- &[- ScalarValue::Float32(Some(-213.1)),- ScalarValue::Float32(None),- ScalarValue::Float32(Some(5.5)),- ScalarValue::Float32(Some(2.0)),- ScalarValue::Float32(Some(1.0)),- ],- &DataType::Float32,- )),- ScalarValue::LargeList(ScalarValue::new_large_list(- &[- ScalarValue::Float32(Some(-213.1)),- ScalarValue::Float32(None),- ScalarValue::Float32(Some(5.5)),- ScalarValue::Float32(Some(2.0)),- ScalarValue::Float32(Some(1.0)),- ],- &DataType::Float32,- )),- ScalarValue::List(ScalarValue::new_list_nullable(- &[- ScalarValue::List(ScalarValue::new_list_nullable(- &[],- &DataType::Float32,- )),- ScalarValue::List(ScalarValue::new_list_nullable(- &[- ScalarValue::Float32(Some(-213.1)),- ScalarValue::Float32(None),- ScalarValue::Float32(Some(5.5)),- ScalarValue::Float32(Some(2.0)),- ScalarValue::Float32(Some(1.0)),- ],- &DataType::Float32,- )),- ],- &DataType::List(new_arc_field("item", DataType::Float32, true)),- )),- ScalarValue::LargeList(ScalarValue::new_large_list(- &[- ScalarValue::LargeList(ScalarValue::new_large_list(- &[],- &DataType::Float32,- )),- ScalarValue::LargeList(ScalarValue::new_large_list(- &[- ScalarValue::Float32(Some(-213.1)),- ScalarValue::Float32(None),- ScalarValue::Float32(Some(5.5)),- ScalarValue::Float32(Some(2.0)),- ScalarValue::Float32(Some(1.0)),- ],- &DataType::Float32,- )),- ],- &DataType::LargeList(new_arc_field("item", DataType::Float32, true)),- )),- ScalarValue::FixedSizeList(Arc::new(FixedSizeListArray::from_iter_primitive::<- Int32Type,- _,- _,- >(- vec![Some(vec![Some(1), Some(2), Some(3)])],- 3,- ))),- ScalarValue::Dictionary(- Box::new(DataType::Int32),- Box::new(ScalarValue::from("foo")),- ),- ScalarValue::Dictionary(- Box::new(DataType::Int32),- Box::new(ScalarValue::Utf8(None)),- ),- ScalarValue::RunEndEncoded(- Field::new("run_ends", DataType::Int32, false).into(),- Field::new("values", DataType::Utf8, true).into(),- Box::new(ScalarValue::from("foo")),- ),- ScalarValue::RunEndEncoded(- Field::new("run_ends", DataType::Int32, false).into(),- Field::new("values", DataType::Utf8, true).into(),- Box::new(ScalarValue::Utf8(None)),- ),- ScalarValue::Binary(Some(b"bar".to_vec())),- ScalarValue::Binary(None),- ScalarValue::LargeBinary(Some(b"bar".to_vec())),- ScalarValue::LargeBinary(None),- ScalarStructBuilder::new()- .with_scalar(+ let should_pass: Vec<ScalarValue> =+ vec![+ ScalarValue::Boolean(None),+ ScalarValue::Float32(None),+ ScalarValue::Float64(None),+ ScalarValue::Int8(None),+ ScalarValue::Int16(None),+ ScalarValue::Int32(None),+ ScalarValue::Int64(None),+ ScalarValue::UInt8(None),+ ScalarValue::UInt16(None),+ ScalarValue::UInt32(None),+ ScalarValue::UInt64(None),+ ScalarValue::Utf8(None),+ ScalarValue::LargeUtf8(None),+ ScalarValue::List(ScalarValue::new_list_nullable(&[], &DataType::Boolean)),+ ScalarValue::LargeList(ScalarValue::new_large_list(&[], &DataType::Boolean)),+ ScalarValue::Date32(None),+ ScalarValue::Boolean(Some(true)),+ ScalarValue::Boolean(Some(false)),+ ScalarValue::Float32(Some(1.0)),+ ScalarValue::Float32(Some(f32::MAX)),+ ScalarValue::Float32(Some(f32::MIN)),+ ScalarValue::Float32(Some(-2000.0)),+ ScalarValue::Float64(Some(1.0)),+ ScalarValue::Float64(Some(f64::MAX)),+ ScalarValue::Float64(Some(f64::MIN)),+ ScalarValue::Float64(Some(-2000.0)),+ ScalarValue::Int8(Some(i8::MIN)),+ ScalarValue::Int8(Some(i8::MAX)),+ ScalarValue::Int8(Some(0)),+ ScalarValue::Int8(Some(-15)),+ ScalarValue::Int16(Some(i16::MIN)),+ ScalarValue::Int16(Some(i16::MAX)),+ ScalarValue::Int16(Some(0)),+ ScalarValue::Int16(Some(-15)),+ ScalarValue::Int32(Some(i32::MIN)),+ ScalarValue::Int32(Some(i32::MAX)),+ ScalarValue::Int32(Some(0)),+ ScalarValue::Int32(Some(-15)),+ ScalarValue::Int64(Some(i64::MIN)),+ ScalarValue::Int64(Some(i64::MAX)),+ ScalarValue::Int64(Some(0)),+ ScalarValue::Int64(Some(-15)),+ ScalarValue::UInt8(Some(u8::MAX)),+ ScalarValue::UInt8(Some(0)),+ ScalarValue::UInt16(Some(u16::MAX)),+ ScalarValue::UInt16(Some(0)),+ ScalarValue::UInt32(Some(u32::MAX)),+ ScalarValue::UInt32(Some(0)),+ ScalarValue::UInt64(Some(u64::MAX)),+ ScalarValue::UInt64(Some(0)),+ ScalarValue::Utf8(Some(String::from("Test string "))),+ ScalarValue::LargeUtf8(Some(String::from("Test Large utf8"))),+ ScalarValue::Utf8View(Some(String::from("Test stringview"))),+ ScalarValue::BinaryView(Some(b"binaryview".to_vec())),+ ScalarValue::Date32(Some(0)),+ ScalarValue::Date32(Some(i32::MAX)),+ ScalarValue::Date32(None),+ ScalarValue::Date64(Some(0)),+ ScalarValue::Date64(Some(i64::MAX)),+ ScalarValue::Date64(None),+ ScalarValue::Time32Second(Some(0)),+ ScalarValue::Time32Second(Some(i32::MAX)),+ ScalarValue::Time32Second(None),+ ScalarValue::Time32Millisecond(Some(0)),+ ScalarValue::Time32Millisecond(Some(i32::MAX)),+ ScalarValue::Time32Millisecond(None),+ ScalarValue::Time64Microsecond(Some(0)),+ ScalarValue::Time64Microsecond(Some(i64::MAX)),+ ScalarValue::Time64Microsecond(None),+ ScalarValue::Time64Nanosecond(Some(0)),+ ScalarValue::Time64Nanosecond(Some(i64::MAX)),+ ScalarValue::Time64Nanosecond(None),+ ScalarValue::TimestampNanosecond(Some(0), None),+ ScalarValue::TimestampNanosecond(Some(i64::MAX), None),+ ScalarValue::TimestampNanosecond(Some(0), Some("UTC".into())),+ ScalarValue::TimestampNanosecond(None, None),+ ScalarValue::TimestampMicrosecond(Some(0), None),+ ScalarValue::TimestampMicrosecond(Some(i64::MAX), None),+ ScalarValue::TimestampMicrosecond(Some(0), Some("UTC".into())),+ ScalarValue::TimestampMicrosecond(None, None),+ ScalarValue::TimestampMillisecond(Some(0), None),+ ScalarValue::TimestampMillisecond(Some(i64::MAX), None),+ ScalarValue::TimestampMillisecond(Some(0), Some("UTC".into())),+ ScalarValue::TimestampMillisecond(None, None),+ ScalarValue::TimestampSecond(Some(0), None),+ ScalarValue::TimestampSecond(Some(i64::MAX), None),+ ScalarValue::TimestampSecond(Some(0), Some("UTC".into())),+ ScalarValue::TimestampSecond(None, None),+ ScalarValue::IntervalDayTime(Some(IntervalDayTimeType::make_value(0, 0))),+ ScalarValue::IntervalDayTime(Some(IntervalDayTimeType::make_value(1, 2))),+ ScalarValue::IntervalDayTime(Some(IntervalDayTimeType::make_value(+ i32::MAX,+ i32::MAX,+ ))),+ ScalarValue::IntervalDayTime(None),+ ScalarValue::IntervalMonthDayNano(Some(+ IntervalMonthDayNanoType::make_value(0, 0, 0),+ )),+ ScalarValue::IntervalMonthDayNano(Some(+ IntervalMonthDayNanoType::make_value(1, 2, 3),+ )),+ ScalarValue::IntervalMonthDayNano(Some(+ IntervalMonthDayNanoType::make_value(i32::MAX, i32::MAX, i64::MAX),+ )),+ ScalarValue::IntervalMonthDayNano(None),+ ScalarValue::List(ScalarValue::new_list_nullable(+ &[+ ScalarValue::Float32(Some(-213.1)),+ ScalarValue::Float32(None),+ ScalarValue::Float32(Some(5.5)),+ ScalarValue::Float32(Some(2.0)),+ ScalarValue::Float32(Some(1.0)),+ ],+ &DataType::Float32,+ )),+ ScalarValue::LargeList(ScalarValue::new_large_list(+ &[+ ScalarValue::Float32(Some(-213.1)),+ ScalarValue::Float32(None),+ ScalarValue::Float32(Some(5.5)),+ ScalarValue::Float32(Some(2.0)),+ ScalarValue::Float32(Some(1.0)),+ ],+ &DataType::Float32,+ )),+ ScalarValue::List(ScalarValue::new_list_nullable(+ &[+ ScalarValue::List(ScalarValue::new_list_nullable(+ &[],+ &DataType::Float32,+ )),+ ScalarValue::List(ScalarValue::new_list_nullable(+ &[+ ScalarValue::Float32(Some(-213.1)),+ ScalarValue::Float32(None),+ ScalarValue::Float32(Some(5.5)),+ ScalarValue::Float32(Some(2.0)),+ ScalarValue::Float32(Some(1.0)),+ ],+ &DataType::Float32,+ )),+ ],+ &DataType::List(new_arc_field("item", DataType::Float32, true)),+ )),+ ScalarValue::LargeList(ScalarValue::new_large_list(+ &[+ ScalarValue::LargeList(ScalarValue::new_large_list(+ &[],+ &DataType::Float32,+ )),+ ScalarValue::LargeList(ScalarValue::new_large_list(+ &[+ ScalarValue::Float32(Some(-213.1)),+ ScalarValue::Float32(None),+ ScalarValue::Float32(Some(5.5)),+ ScalarValue::Float32(Some(2.0)),+ ScalarValue::Float32(Some(1.0)),+ ],+ &DataType::Float32,+ )),+ ],+ &DataType::LargeList(new_arc_field("item", DataType::Float32, true)),+ )),+ ScalarValue::FixedSizeList(Arc::new(+ FixedSizeListArray::from_iter_primitive::<Int32Type, _, _>(+ vec![Some(vec![Some(1), Some(2), Some(3)])],+ 3,+ ),+ )),+ ScalarValue::ListView(Arc::new(ListViewArray::from_iter_primitive::<+ Int32Type,+ _,+ _,+ >(vec![Some(vec![+ Some(1),+ None,+ Some(3),+ ])]))),+ ScalarValue::LargeListView(Arc::new(+ LargeListViewArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(+ vec![Some(1), None, Some(3)],+ )]),+ )),+ ScalarValue::Dictionary(+ Box::new(DataType::Int32),+ Box::new(ScalarValue::from("foo")),+ ),+ ScalarValue::Dictionary(+ Box::new(DataType::Int32),+ Box::new(ScalarValue::Utf8(None)),+ ),+ ScalarValue::RunEndEncoded(+ Field::new("run_ends", DataType::Int32, false).into(),+ Field::new("values", DataType::Utf8, true).into(),+ Box::new(ScalarValue::from("foo")),+ ),+ ScalarValue::RunEndEncoded(+ Field::new("run_ends", DataType::Int32, false).into(),+ Field::new("values", DataType::Utf8, true).into(),+ Box::new(ScalarValue::Utf8(None)),+ ),+ ScalarValue::Binary(Some(b"bar".to_vec())),+ ScalarValue::Binary(None),+ ScalarValue::LargeBinary(Some(b"bar".to_vec())),+ ScalarValue::LargeBinary(None),+ ScalarStructBuilder::new()+ .with_scalar(+ Field::new("a", DataType::Int32, true),+ ScalarValue::from(23i32),+ )+ .with_scalar(+ Field::new("b", DataType::Boolean, false),+ ScalarValue::from(false),+ )+ .build()+ .unwrap(),+ ScalarStructBuilder::new()+ .with_scalar(+ Field::new("a", DataType::Int32, true),+ ScalarValue::from(23i32),+ )+ .with_scalar(+ Field::new("b", DataType::Boolean, false),+ ScalarValue::from(false),+ )+ .build()+ .unwrap(),+ ScalarValue::try_from(&DataType::Struct(Fields::from(vec![Field::new("a", DataType::Int32, true),- ScalarValue::from(23i32),- )- .with_scalar(Field::new("b", DataType::Boolean, false),- ScalarValue::from(false),- )- .build()+ ]))).unwrap(),- ScalarStructBuilder::new()- .with_scalar(+ ScalarValue::try_from(&DataType::Struct(Fields::from(vec![Field::new("a", DataType::Int32, true),- ScalarValue::from(23i32),- )- .with_scalar(Field::new("b", DataType::Boolean, false),- ScalarValue::from(false),- )- .build()+ ]))).unwrap(),- ScalarValue::try_from(&DataType::Struct(Fields::from(vec![- Field::new("a", DataType::Int32, true),- Field::new("b", DataType::Boolean, false),- ])))- .unwrap(),- ScalarValue::try_from(&DataType::Struct(Fields::from(vec![- Field::new("a", DataType::Int32, true),- Field::new("b", DataType::Boolean, false),- ])))- .unwrap(),- ScalarValue::try_from(&DataType::Map(- Arc::new(Field::new(- "entries",- DataType::Struct(Fields::from(vec![- Field::new("key", DataType::Int32, true),- Field::new("value", DataType::Utf8, false),- ])),- false,- )),- false,- ))- .unwrap(),- ScalarValue::try_from(&DataType::Map(- Arc::new(Field::new(- "entries",- DataType::Struct(Fields::from(vec![- Field::new("key", DataType::Int32, true),- Field::new("value", DataType::Utf8, true),- ])),+ ScalarValue::try_from(&DataType::Map(+ Arc::new(Field::new(+ "entries",+ DataType::Struct(Fields::from(vec![+ Field::new("key", DataType::Int32, true),+ Field::new("value", DataType::Utf8, false),+ ])),+ false,+ )),false,- )),- true,- ))- .unwrap(),- ScalarValue::Map(Arc::new(create_map_array_test_case())),- ScalarValue::FixedSizeBinary(b"bar".to_vec().len() as i32, Some(b"bar".to_vec())),- ScalarValue::FixedSizeBinary(0, None),- ScalarValue::FixedSizeBinary(5, None),- ];+ ))+ .unwrap(),+ ScalarValue::try_from(&DataType::Map(+ Arc::new(Field::new(+ "entries",+ DataType::Struct(Fields::from(vec![+ Field::new("key", DataType::Int32, true),+ Field::new("value", DataType::Utf8, true),+ ])),+ false,+ )),+ true,+ ))+ .unwrap(),+ ScalarValue::Map(Arc::new(create_map_array_test_case())),+ ScalarValue::FixedSizeBinary(+ b"bar".to_vec().len() as i32,+ Some(b"bar".to_vec()),+ ),+ ScalarValue::FixedSizeBinary(0, None),+ ScalarValue::FixedSizeBinary(5, None),+ ];// ScalarValue directlyfor test_case in should_pass.iter() {diff --git a/datafusion/sql/src/unparser/expr.rs b/datafusion/sql/src/unparser/expr.rsindex 4885dde077..ca51af2769 100644--- a/datafusion/sql/src/unparser/expr.rs+++ b/datafusion/sql/src/unparser/expr.rs@@ -1324,6 +1324,8 @@ impl Unparser<'_> {ScalarValue::FixedSizeList(a) => self.scalar_value_list_to_sql(a.values()),ScalarValue::List(a) => self.scalar_value_list_to_sql(a.values()),ScalarValue::LargeList(a) => self.scalar_value_list_to_sql(a.values()),+ ScalarValue::ListView(a) => self.scalar_value_list_to_sql(a.values()),+ ScalarValue::LargeListView(a) => self.scalar_value_list_to_sql(a.values()),ScalarValue::Date32(Some(_)) => {let date = v.to_array()?@@ -1839,7 +1841,7 @@ mod tests {use std::{sync::Arc, vec};use crate::unparser::dialect::SqliteDialect;- use arrow::array::{LargeListArray, ListArray};+ use arrow::array::{LargeListArray, LargeListViewArray, ListArray, ListViewArray};use arrow::datatypes::{DataType::Int8, Field, Int32Type, Schema, TimeUnit};use ast::ObjectName;use datafusion_common::datatype::DataTypeExt;@@ -2353,6 +2355,28 @@ mod tests {),"[1, 2, 3]",),+ (+ Expr::Literal(+ ScalarValue::ListView(Arc::new(+ ListViewArray::from_iter_primitive::<Int32Type, _, _>(vec![+ Some(vec![Some(1), Some(2), Some(3)]),+ ]),+ )),+ None,+ ),+ "[1, 2, 3]",+ ),+ (+ Expr::Literal(+ ScalarValue::LargeListView(Arc::new(+ LargeListViewArray::from_iter_primitive::<Int32Type, _, _>(vec![+ Some(vec![Some(1), Some(2), Some(3)]),+ ]),+ )),+ None,+ ),+ "[1, 2, 3]",+ ),(Expr::BinaryExpr(BinaryExpr {left: Box::new(col("a")),