Skip to content

Apache DataFusion SQL Query Engine

feat: add inner_product scalar function (#21861)

crm269a29e3301046
diff --git a/datafusion/functions-nested/src/inner_product.rs b/datafusion/functions-nested/src/inner_product.rs
new file mode 100644
index 0000000000..8d8c565663
--- /dev/null
+++ b/datafusion/functions-nested/src/inner_product.rs
@@ -0,0 +1,214 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! [`ScalarUDFImpl`] definitions for inner_product function.
+
+use crate::utils::make_scalar_function;
+use arrow::array::{Array, ArrayRef, Float64Array, OffsetSizeTrait};
+use arrow::datatypes::{
+ DataType,
+ DataType::{FixedSizeList, LargeList, List, Null},
+ Field,
+};
+use datafusion_common::cast::{as_float64_array, as_generic_list_array};
+use datafusion_common::utils::{ListCoercion, coerced_type_with_base_type_only};
+use datafusion_common::{
+ Result, exec_err, internal_err, plan_err, utils::take_function_args,
+};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ Volatility,
+};
+use datafusion_macros::user_doc;
+use std::sync::Arc;
+
+make_udf_expr_and_func!(
+ InnerProduct,
+ inner_product,
+ array1 array2,
+ "returns the inner product (dot product) of two numeric arrays.",
+ inner_product_udf
+);
+
+#[user_doc(
+ doc_section(label = "Array Functions"),
+ description = "Returns the inner product (dot product) of two input arrays of equal length, computed as `sum(array1[i] * array2[i])`. Returns NULL if either array is NULL or contains NULL elements. Returns 0.0 for two empty arrays.",
+ syntax_example = "inner_product(array1, array2)",
+ sql_example = r#"```sql
+> select inner_product([1.0, 2.0, 3.0], [4.0, 5.0, 6.0]);
++-------------------------------------------------------+
+| inner_product(List([1.0,2.0,3.0]),List([4.0,5.0,6.0])) |
++-------------------------------------------------------+
+| 32.0 |
++-------------------------------------------------------+
+```"#,
+ argument(
+ name = "array1",
+ description = "Array expression. Can be a constant, column, or function, and any combination of array operators."
+ ),
+ argument(
+ name = "array2",
+ description = "Array expression. Can be a constant, column, or function, and any combination of array operators."
+ )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct InnerProduct {
+ signature: Signature,
+ aliases: Vec<String>,
+}
+
+impl Default for InnerProduct {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl InnerProduct {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Immutable),
+ aliases: vec!["dot_product".to_string()],
+ }
+ }
+}
+
+impl ScalarUDFImpl for InnerProduct {
+ fn name(&self) -> &str {
+ "inner_product"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ Ok(DataType::Float64)
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ let [_, _] = take_function_args(self.name(), arg_types)?;
+ let coercion = Some(&ListCoercion::FixedSizedListToList);
+
+ for arg_type in arg_types {
+ if !matches!(arg_type, Null | List(_) | LargeList(_) | FixedSizeList(..)) {
+ return plan_err!("{} does not support type {arg_type}", self.name());
+ }
+ }
+
+ // If any input is `LargeList`, both sides must be widened to `LargeList`
+ // so the runtime dispatch in `inner_product_inner` sees a homogeneous
+ // pair. Follows the pattern in `ArrayConcat::coerce_types`.
+ let any_large_list = arg_types.iter().any(|t| matches!(t, LargeList(_)));
+
+ let coerced = arg_types
+ .iter()
+ .map(|arg_type| {
+ if matches!(arg_type, Null) {
+ let field = Arc::new(Field::new_list_field(DataType::Float64, true));
+ return if any_large_list {
+ LargeList(field)
+ } else {
+ List(field)
+ };
+ }
+ let coerced = coerced_type_with_base_type_only(
+ arg_type,
+ &DataType::Float64,
+ coercion,
+ );
+ match coerced {
+ List(field) if any_large_list => LargeList(field),
+ other => other,
+ }
+ })
+ .collect();
+
+ Ok(coerced)
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
+ make_scalar_function(inner_product_inner)(&args.args)
+ }
+
+ fn aliases(&self) -> &[String] {
+ &self.aliases
+ }
+
+ fn documentation(&self) -> Option<&Documentation> {
+ self.doc()
+ }
+}
+
+fn inner_product_inner(args: &[ArrayRef]) -> Result<ArrayRef> {
+ let [array1, array2] = take_function_args("inner_product", args)?;
+ match (array1.data_type(), array2.data_type()) {
+ (List(_), List(_)) => general_inner_product::<i32>(args),
+ (LargeList(_), LargeList(_)) => general_inner_product::<i64>(args),
+ (arg_type1, arg_type2) => internal_err!(
+ "inner_product received unexpected types after coercion: {arg_type1} and {arg_type2}"
+ ),
+ }
+}
+
+fn general_inner_product<O: OffsetSizeTrait>(arrays: &[ArrayRef]) -> Result<ArrayRef> {
+ let list_array1 = as_generic_list_array::<O>(&arrays[0])?;
+ let list_array2 = as_generic_list_array::<O>(&arrays[1])?;
+
+ let values1 = as_float64_array(list_array1.values())?;
+ let values2 = as_float64_array(list_array2.values())?;
+ let offsets1 = list_array1.value_offsets();
+ let offsets2 = list_array2.value_offsets();
+
+ let mut builder = Float64Array::builder(list_array1.len());
+ for row in 0..list_array1.len() {
+ if list_array1.is_null(row) || list_array2.is_null(row) {
+ builder.append_null();
+ continue;
+ }
+
+ let start1 = offsets1[row].as_usize();
+ let end1 = offsets1[row + 1].as_usize();
+ let start2 = offsets2[row].as_usize();
+ let end2 = offsets2[row + 1].as_usize();
+ let len1 = end1 - start1;
+ let len2 = end2 - start2;
+
+ if len1 != len2 {
+ return exec_err!(
+ "inner_product requires both list inputs to have the same length, got {len1} and {len2}"
+ );
+ }
+
+ let slice1 = values1.slice(start1, len1);
+ let slice2 = values2.slice(start2, len2);
+ if slice1.null_count() != 0 || slice2.null_count() != 0 {
+ builder.append_null();
+ continue;
+ }
+
+ let vals1 = slice1.values();
+ let vals2 = slice2.values();
+
+ let mut dot = 0.0;
+ for i in 0..len1 {
+ dot += vals1[i] * vals2[i];
+ }
+ builder.append_value(dot);
+ }
+
+ Ok(Arc::new(builder.finish()) as ArrayRef)
+}
diff --git a/datafusion/functions-nested/src/lib.rs b/datafusion/functions-nested/src/lib.rs
index f94b3921a5..e4d08a9a5e 100644
--- a/datafusion/functions-nested/src/lib.rs
+++ b/datafusion/functions-nested/src/lib.rs
@@ -55,6 +55,7 @@ pub mod except;
pub mod expr_ext;
pub mod extract;
pub mod flatten;
+pub mod inner_product;
pub mod length;
pub mod make_array;
pub mod map;
@@ -107,6 +108,7 @@ pub mod expr_fn {
pub use super::extract::array_pop_front;
pub use super::extract::array_slice;
pub use super::flatten::flatten;
+ pub use super::inner_product::inner_product;
pub use super::length::array_length;
pub use super::make_array::make_array;
pub use super::map_entries::map_entries;
@@ -163,6 +165,7 @@ pub fn all_default_nested_functions() -> Vec<Arc<ScalarUDF>> {
empty::array_empty_udf(),
length::array_length_udf(),
cosine_distance::cosine_distance_udf(),
+ inner_product::inner_product_udf(),
distance::array_distance_udf(),
flatten::flatten_udf(),
min_max::array_max_udf(),
diff --git a/datafusion/sqllogictest/test_files/inner_product.slt b/datafusion/sqllogictest/test_files/inner_product.slt
new file mode 100644
index 0000000000..81ada918d5
--- /dev/null
+++ b/datafusion/sqllogictest/test_files/inner_product.slt
@@ -0,0 +1,189 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+## inner_product
+
+# Orthogonal vectors: dot product = 0
+query R
+select inner_product([1.0, 0.0], [0.0, 1.0]);
+----
+0
+
+# General case: 1*4 + 2*5 + 3*6 = 32
+query R
+select inner_product([1.0, 2.0, 3.0], [4.0, 5.0, 6.0]);
+----
+32
+
+# Self dot product: [1,2,3]·[1,2,3] = 14
+query R
+select inner_product([1.0, 2.0, 3.0], [1.0, 2.0, 3.0]);
+----
+14
+
+# Opposite vectors: negative
+query R
+select inner_product([1.0, 0.0], [-1.0, 0.0]);
+----
+-1
+
+# Zero vector returns 0 (not NULL — inner product with zero is well-defined)
+query R
+select inner_product([0.0, 0.0], [1.0, 2.0]);
+----
+0
+
+# Both zero vectors: 0
+query R
+select inner_product([0.0, 0.0], [0.0, 0.0]);
+----
+0
+
+# Bare NULL input returns NULL
+query R
+select inner_product(NULL, [1.0, 2.0]);
+----
+NULL
+
+# NULL in second position returns NULL
+query R
+select inner_product([1.0, 2.0], NULL);
+----
+NULL
+
+# Both NULL returns NULL
+query R
+select inner_product(NULL, NULL);
+----
+NULL
+
+# Mismatched lengths error
+query error inner_product requires both list inputs to have the same length
+select inner_product([1.0, 2.0], [1.0]);
+
+# NULL element inside a list returns NULL for that row
+query R
+select inner_product([1.0, 2.0, NULL], [1.0, 2.0, 3.0]);
+----
+NULL
+
+# LargeList support
+query R
+select inner_product(
+ arrow_cast([1.0, 2.0, 3.0], 'LargeList(Float64)'),
+ arrow_cast([4.0, 5.0, 6.0], 'LargeList(Float64)')
+);
+----
+32
+
+# Mixed List + LargeList: widens to LargeList
+query R
+select inner_product([1.0, 2.0, 3.0], arrow_cast([4.0, 5.0, 6.0], 'LargeList(Float64)'));
+----
+32
+
+# Reverse order: LargeList + List also widens
+query R
+select inner_product(arrow_cast([1.0, 2.0, 3.0], 'LargeList(Float64)'), [4.0, 5.0, 6.0]);
+----
+32
+
+# FixedSizeList inputs (coerced to List)
+query R
+select inner_product(
+ arrow_cast([1.0, 2.0, 3.0], 'FixedSizeList(3, Float64)'),
+ arrow_cast([4.0, 5.0, 6.0], 'FixedSizeList(3, Float64)')
+);
+----
+32
+
+# FixedSizeList + LargeList: widens to LargeList
+query R
+select inner_product(
+ arrow_cast([1.0, 2.0, 3.0], 'FixedSizeList(3, Float64)'),
+ arrow_cast([4.0, 5.0, 6.0], 'LargeList(Float64)')
+);
+----
+32
+
+# Float32 inner type (coerced to Float64)
+query R
+select inner_product(arrow_cast([1.0, 2.0, 3.0], 'List(Float32)'), [4.0, 5.0, 6.0]);
+----
+32
+
+# Int64 inner type (coerced to Float64)
+query R
+select inner_product(arrow_cast([1, 2, 3], 'List(Int64)'), arrow_cast([4, 5, 6], 'List(Int64)'));
+----
+32
+
+# Integer literals (coerced to Float64)
+query R
+select inner_product([1, 2, 3], [4, 5, 6]);
+----
+32
+
+# Unsupported non-list input (plan error)
+query error inner_product does not support type
+select inner_product(1, 2);
+
+# Multi-row query with NULL row propagation
+query R
+select inner_product(column1, column2) from (values
+ (make_array(1.0, 0.0), make_array(0.0, 1.0)),
+ (make_array(1.0, 2.0, 3.0), make_array(4.0, 5.0, 6.0)),
+ (make_array(1.0, 0.0), make_array(-1.0, 0.0)),
+ (make_array(1.0, 0.0), NULL)
+) as t(column1, column2);
+----
+0
+32
+-1
+NULL
+
+# Empty arrays: inner product of empty vectors = 0
+query R
+select inner_product(arrow_cast(make_array(), 'List(Float64)'), arrow_cast(make_array(), 'List(Float64)'));
+----
+0
+
+# No arguments error
+query error inner_product function requires 2 arguments, got 0
+select inner_product();
+
+# Return type is Float64
+query RT
+select inner_product([1.0, 2.0], [3.0, 4.0]), arrow_typeof(inner_product([1.0, 2.0], [3.0, 4.0]));
+----
+11 Float64
+
+# dot_product alias produces the same result
+query R
+select dot_product([1.0, 2.0, 3.0], [4.0, 5.0, 6.0]);
+----
+32
+
+# dot_product alias with NULL row propagates correctly
+query R
+select dot_product(column1, column2) from (values
+ (make_array(1.0, 0.0), make_array(0.0, 1.0)),
+ (make_array(1.0, 0.0), NULL)
+) as t(column1, column2);
+----
+0
+NULL
diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.md
index c107a91730..260f69f737 100644
--- a/docs/source/user-guide/sql/scalar_functions.md
+++ b/docs/source/user-guide/sql/scalar_functions.md
@@ -3284,9 +3284,11 @@ _Alias of [current_date](#current_date)._
- [arrays_zip](#arrays_zip)
- [cardinality](#cardinality)
- [cosine_distance](#cosine_distance)
+- [dot_product](#dot_product)
- [empty](#empty)
- [flatten](#flatten)
- [generate_series](#generate_series)
+- [inner_product](#inner_product)
- [list_any_match](#list_any_match)
- [list_any_value](#list_any_value)
- [list_append](#list_append)
@@ -4561,6 +4563,10 @@ cosine_distance(array1, array2)
+-----------------------------------------------+
```
+### `dot_product`
+
+_Alias of [inner_product](#inner_product)._
+
### `empty`
Returns 1 for an empty array or 0 for a non-empty array.
@@ -4643,6 +4649,34 @@ generate_series(start, stop[, step])
+------------------------------------+
```
+### `inner_product`
+
+Returns the inner product (dot product) of two input arrays of equal length, computed as `sum(array1[i] * array2[i])`. Returns NULL if either array is NULL or contains NULL elements. Returns 0.0 for two empty arrays.
+
+```sql
+inner_product(array1, array2)
+```
+
+#### Arguments
+
+- **array1**: Array expression. Can be a constant, column, or function, and any combination of array operators.
+- **array2**: Array expression. Can be a constant, column, or function, and any combination of array operators.
+
+#### Example
+
+```sql
+> select inner_product([1.0, 2.0, 3.0], [4.0, 5.0, 6.0]);
++-------------------------------------------------------+
+| inner_product(List([1.0,2.0,3.0]),List([4.0,5.0,6.0])) |
++-------------------------------------------------------+
+| 32.0 |
++-------------------------------------------------------+
+```
+
+#### Aliases
+
+- dot_product
+
### `list_any_match`
_Alias of [array_any_match](#array_any_match)._