Skip to content

Apache DataFusion SQL Query Engine

chore: merge make_array and spark array (#19006)

yqrz2217c2b836d8
diff --git a/Cargo.lock b/Cargo.lock
index 5e4aff24c0..8b5b51de92 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2663,6 +2663,7 @@ dependencies = [
"datafusion-execution",
"datafusion-expr",
"datafusion-functions",
+ "datafusion-functions-nested",
"log",
"rand 0.9.2",
"sha1",
diff --git a/datafusion/functions-nested/src/make_array.rs b/datafusion/functions-nested/src/make_array.rs
index 97d64c70cd..6ca1341650 100644
--- a/datafusion/functions-nested/src/make_array.rs
+++ b/datafusion/functions-nested/src/make_array.rs
@@ -125,19 +125,7 @@ impl ScalarUDFImpl for MakeArray {
}
fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
- if let Ok(unified) = try_type_union_resolution_with_struct(arg_types) {
- return Ok(unified);
- }
-
- if let Some(unified) = type_union_resolution(arg_types) {
- Ok(vec![unified; arg_types.len()])
- } else {
- plan_err!(
- "Failed to unify argument types of {}: [{}]",
- self.name(),
- arg_types.iter().join(", ")
- )
- }
+ coerce_types_inner(arg_types, self.name())
}
fn documentation(&self) -> Option<&Documentation> {
@@ -163,7 +151,7 @@ pub(crate) fn make_array_inner(arrays: &[ArrayRef]) -> Result<ArrayRef> {
SingleRowListArrayBuilder::new(array).build_list_array(),
))
} else {
- array_array::<i32>(arrays, data_type.clone())
+ array_array::<i32>(arrays, data_type.clone(), Field::LIST_FIELD_DEFAULT_NAME)
}
}
@@ -207,9 +195,10 @@ pub(crate) fn make_array_inner(arrays: &[ArrayRef]) -> Result<ArrayRef> {
/// └──────────────┘ └──────────────┘ └─────────────────────────────┘
/// col1 col2 output
/// ```
-fn array_array<O: OffsetSizeTrait>(
+pub fn array_array<O: OffsetSizeTrait>(
args: &[ArrayRef],
data_type: DataType,
+ field_name: &str,
) -> Result<ArrayRef> {
// do not accept 0 arguments.
if args.is_empty() {
@@ -252,9 +241,25 @@ fn array_array<O: OffsetSizeTrait>(
let data = mutable.freeze();
Ok(Arc::new(GenericListArray::<O>::try_new(
- Arc::new(Field::new_list_field(data_type, true)),
+ Arc::new(Field::new(field_name, data_type, true)),
OffsetBuffer::new(offsets.into()),
arrow::array::make_array(data),
None,
)?))
}
+
+pub fn coerce_types_inner(arg_types: &[DataType], name: &str) -> Result<Vec<DataType>> {
+ if let Ok(unified) = try_type_union_resolution_with_struct(arg_types) {
+ return Ok(unified);
+ }
+
+ if let Some(unified) = type_union_resolution(arg_types) {
+ Ok(vec![unified; arg_types.len()])
+ } else {
+ plan_err!(
+ "Failed to unify argument types of {}: [{}]",
+ name,
+ arg_types.iter().join(", ")
+ )
+ }
+}
diff --git a/datafusion/spark/Cargo.toml b/datafusion/spark/Cargo.toml
index 279c88b525..f39b6992c3 100644
--- a/datafusion/spark/Cargo.toml
+++ b/datafusion/spark/Cargo.toml
@@ -48,6 +48,7 @@ datafusion-common = { workspace = true }
datafusion-execution = { workspace = true }
datafusion-expr = { workspace = true }
datafusion-functions = { workspace = true, features = ["crypto_expressions"] }
+datafusion-functions-nested = { workspace = true }
log = { workspace = true }
rand = { workspace = true }
sha1 = "0.10"
diff --git a/datafusion/spark/src/function/array/spark_array.rs b/datafusion/spark/src/function/array/spark_array.rs
index bb9665613d..67a307bbe9 100644
--- a/datafusion/spark/src/function/array/spark_array.rs
+++ b/datafusion/spark/src/function/array/spark_array.rs
@@ -17,19 +17,15 @@
use std::{any::Any, sync::Arc};
-use arrow::array::{
- make_array, new_null_array, Array, ArrayData, ArrayRef, Capacities, GenericListArray,
- MutableArrayData, NullArray, OffsetSizeTrait,
-};
-use arrow::buffer::OffsetBuffer;
+use arrow::array::{new_null_array, Array, ArrayRef};
use arrow::datatypes::{DataType, Field, FieldRef};
use datafusion_common::utils::SingleRowListArrayBuilder;
-use datafusion_common::{internal_err, plan_datafusion_err, plan_err, Result};
-use datafusion_expr::type_coercion::binary::comparison_coercion;
+use datafusion_common::{internal_err, Result};
use datafusion_expr::{
ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature,
TypeSignature, Volatility,
};
+use datafusion_functions_nested::make_array::{array_array, coerce_types_inner};
use crate::function::functions_nested_utils::make_scalar_function;
@@ -38,7 +34,6 @@ const ARRAY_FIELD_DEFAULT_NAME: &str = "element";
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct SparkArray {
signature: Signature,
- aliases: Vec<String>,
}
impl Default for SparkArray {
@@ -54,7 +49,6 @@ impl SparkArray {
vec![TypeSignature::UserDefined, TypeSignature::Nullary],
Volatility::Immutable,
),
- aliases: vec![String::from("spark_make_array")],
}
}
}
@@ -92,10 +86,6 @@ impl ScalarUDFImpl for SparkArray {
}
}
- if expr_type.is_null() {
- expr_type = DataType::Int32;
- }
-
let return_type = DataType::List(Arc::new(Field::new(
ARRAY_FIELD_DEFAULT_NAME,
expr_type,
@@ -115,30 +105,11 @@ impl ScalarUDFImpl for SparkArray {
}
fn aliases(&self) -> &[String] {
- &self.aliases
+ &[]
}
fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
- let first_type = arg_types.first().ok_or_else(|| {
- plan_datafusion_err!("Spark array function requires at least one argument")
- })?;
- let new_type =
- arg_types
- .iter()
- .skip(1)
- .try_fold(first_type.clone(), |acc, x| {
- // The coerced types found by `comparison_coercion` are not guaranteed to be
- // coercible for the arguments. `comparison_coercion` returns more loose
- // types that can be coerced to both `acc` and `x` for comparison purpose.
- // See `maybe_data_types` for the actual coercion.
- let coerced_type = comparison_coercion(&acc, x);
- if let Some(coerced_type) = coerced_type {
- Ok(coerced_type)
- } else {
- plan_err!("Coercion from {acc} to {x} failed.")
- }
- })?;
- Ok(vec![new_type; arg_types.len()])
+ coerce_types_inner(arg_types, self.name())
}
}
@@ -160,7 +131,7 @@ pub fn make_array_inner(arrays: &[ArrayRef]) -> Result<ArrayRef> {
DataType::Null => {
let length = arrays.iter().map(|a| a.len()).sum();
// By default Int32
- let array = new_null_array(&DataType::Int32, length);
+ let array = new_null_array(&DataType::Null, length);
Ok(Arc::new(
SingleRowListArrayBuilder::new(array)
.with_nullable(true)
@@ -168,98 +139,6 @@ pub fn make_array_inner(arrays: &[ArrayRef]) -> Result<ArrayRef> {
.build_list_array(),
))
}
- _ => array_array::<i32>(arrays, data_type),
- }
-}
-
-/// Convert one or more [`ArrayRef`] of the same type into a
-/// `ListArray` or 'LargeListArray' depending on the offset size.
-///
-/// # Example (non nested)
-///
-/// Calling `array(col1, col2)` where col1 and col2 are non nested
-/// would return a single new `ListArray`, where each row was a list
-/// of 2 elements:
-///
-/// ```text
-/// ┌─────────┐ ┌─────────┐ ┌──────────────┐
-/// │ ┌─────┐ │ │ ┌─────┐ │ │ ┌──────────┐ │
-/// │ │ A │ │ │ │ X │ │ │ │ [A, X] │ │
-/// │ ├─────┤ │ │ ├─────┤ │ │ ├──────────┤ │
-/// │ │NULL │ │ │ │ Y │ │──────────▶│ │[NULL, Y] │ │
-/// │ ├─────┤ │ │ ├─────┤ │ │ ├──────────┤ │
-/// │ │ C │ │ │ │ Z │ │ │ │ [C, Z] │ │
-/// │ └─────┘ │ │ └─────┘ │ │ └──────────┘ │
-/// └─────────┘ └─────────┘ └──────────────┘
-/// col1 col2 output
-/// ```
-///
-/// # Example (nested)
-///
-/// Calling `array(col1, col2)` where col1 and col2 are lists
-/// would return a single new `ListArray`, where each row was a list
-/// of the corresponding elements of col1 and col2.
-///
-/// ``` text
-/// ┌──────────────┐ ┌──────────────┐ ┌─────────────────────────────┐
-/// │ ┌──────────┐ │ │ ┌──────────┐ │ │ ┌────────────────────────┐ │
-/// │ │ [A, X] │ │ │ │ [] │ │ │ │ [[A, X], []] │ │
-/// │ ├──────────┤ │ │ ├──────────┤ │ │ ├────────────────────────┤ │
-/// │ │[NULL, Y] │ │ │ │[Q, R, S] │ │───────▶│ │ [[NULL, Y], [Q, R, S]] │ │
-/// │ ├──────────┤ │ │ ├──────────┤ │ │ ├────────────────────────│ │
-/// │ │ [C, Z] │ │ │ │ NULL │ │ │ │ [[C, Z], NULL] │ │
-/// │ └──────────┘ │ │ └──────────┘ │ │ └────────────────────────┘ │
-/// └──────────────┘ └──────────────┘ └─────────────────────────────┘
-/// col1 col2 output
-/// ```
-fn array_array<O: OffsetSizeTrait>(
- args: &[ArrayRef],
- data_type: DataType,
-) -> Result<ArrayRef> {
- // do not accept 0 arguments.
- if args.is_empty() {
- return plan_err!("Array requires at least one argument");
- }
-
- let mut data = vec![];
- let mut total_len = 0;
- for arg in args {
- let arg_data = if arg.as_any().is::<NullArray>() {
- ArrayData::new_empty(&data_type)
- } else {
- arg.to_data()
- };
- total_len += arg_data.len();
- data.push(arg_data);
+ _ => array_array::<i32>(arrays, data_type, ARRAY_FIELD_DEFAULT_NAME),
}
-
- let mut offsets: Vec<O> = Vec::with_capacity(total_len);
- offsets.push(O::usize_as(0));
-
- let capacity = Capacities::Array(total_len);
- let data_ref = data.iter().collect::<Vec<_>>();
- let mut mutable = MutableArrayData::with_capacities(data_ref, true, capacity);
-
- let num_rows = args[0].len();
- for row_idx in 0..num_rows {
- for (arr_idx, arg) in args.iter().enumerate() {
- if !arg.as_any().is::<NullArray>()
- && !arg.is_null(row_idx)
- && arg.is_valid(row_idx)
- {
- mutable.extend(arr_idx, row_idx, row_idx + 1);
- } else {
- mutable.extend_nulls(1);
- }
- }
- offsets.push(O::usize_as(mutable.len()));
- }
- let data = mutable.freeze();
-
- Ok(Arc::new(GenericListArray::<O>::try_new(
- Arc::new(Field::new(ARRAY_FIELD_DEFAULT_NAME, data_type, true)),
- OffsetBuffer::new(offsets.into()),
- make_array(data),
- None,
- )?))
}