Apache DataFusion SQL Query Engine
Add lambda substrait support (#21193)
gstvg9a6f67e20242
diff --git a/datafusion/substrait/src/logical_plan/consumer/expr/field_reference.rs b/datafusion/substrait/src/logical_plan/consumer/expr/field_reference.rsindex dae6c625ef..be084f3603 100644--- a/datafusion/substrait/src/logical_plan/consumer/expr/field_reference.rs+++ b/datafusion/substrait/src/logical_plan/consumer/expr/field_reference.rs@@ -21,7 +21,7 @@ use datafusion::logical_expr::Expr;use std::sync::Arc;use substrait::proto::expression::FieldReference;use substrait::proto::expression::field_reference::ReferenceType::DirectReference;-use substrait::proto::expression::field_reference::RootType;+use substrait::proto::expression::field_reference::{LambdaParameterReference, RootType};use substrait::proto::expression::reference_segment::ReferenceType::StructField;pub async fn from_field_reference(@@ -56,9 +56,9 @@ pub(crate) fn from_substrait_field_reference(Some(RootType::Expression(_)) => not_impl_err!("Expression root type in field reference is not supported"),- Some(RootType::LambdaParameterReference(_)) => not_impl_err!(- "Lambda parameter reference in field reference is not yet supported"- ),+ Some(RootType::LambdaParameterReference(+ LambdaParameterReference { steps_out },+ )) => consumer.lambda_variable(*steps_out as usize, field_idx),}}_ => not_impl_err!(@@ -85,3 +85,85 @@ fn resolve_outer_reference(let col = Column::from((qualifier, field));Ok(Expr::OuterReferenceColumn(Arc::clone(field), col))}++#[cfg(test)]+mod tests {+ use datafusion::{+ common::{DFSchema, assert_contains},+ prelude::SessionContext,+ };+ use substrait::proto::{+ Type,+ expression::{+ FieldReference, ReferenceSegment,+ field_reference::{self, LambdaParameterReference, RootType},+ reference_segment::{ReferenceType, StructField},+ },+ r#type::{I64, Kind},+ };++ use crate::{+ extensions::Extensions,+ logical_plan::consumer::{+ DefaultSubstraitConsumer, SubstraitConsumer, from_field_reference,+ },+ };++ #[tokio::test]+ async fn test_lambda_variable_invalid_steps_out() {+ let lambda_field_ref = lambda_field_ref(0, 99);++ let extensions = Extensions::default();+ let session_state = SessionContext::new().state();+ let consumer = DefaultSubstraitConsumer::new(&extensions, &session_state);++ let err =+ from_field_reference(&consumer, &lambda_field_ref, DFSchema::empty_ref())+ .await+ .unwrap_err();++ assert_contains!(err.to_string(), "No lambda at 99 steps out, got only 0");+ }++ #[tokio::test]+ async fn test_lambda_variable_invalid_field_idx() {+ let lambda_field_ref = lambda_field_ref(1, 0);++ let extensions = Extensions::default();+ let session_state = SessionContext::new().state();+ let consumer = DefaultSubstraitConsumer::new(&extensions, &session_state);+ let _names = consumer+ .push_lambda_parameters(+ &[Type {+ kind: Some(Kind::I64(I64::default())),+ }],+ DFSchema::empty_ref(),+ )+ .unwrap();++ let err =+ from_field_reference(&consumer, &lambda_field_ref, DFSchema::empty_ref())+ .await+ .unwrap_err();++ assert_contains!(+ err.to_string(),+ "At lambda 0 steps out, no field at index 1, got only 1"+ );+ }++ fn lambda_field_ref(field: i32, steps_out: u32) -> FieldReference {+ FieldReference {+ reference_type: Some(field_reference::ReferenceType::DirectReference(+ ReferenceSegment {+ reference_type: Some(ReferenceType::StructField(Box::new(+ StructField { field, child: None },+ ))),+ },+ )),+ root_type: Some(RootType::LambdaParameterReference(+ LambdaParameterReference { steps_out },+ )),+ }+ }+}diff --git a/datafusion/substrait/src/logical_plan/consumer/expr/lambda.rs b/datafusion/substrait/src/logical_plan/consumer/expr/lambda.rsnew file mode 100644index 0000000000..c4554dea87--- /dev/null+++ b/datafusion/substrait/src/logical_plan/consumer/expr/lambda.rs@@ -0,0 +1,102 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use datafusion::{+ common::{DFSchema, substrait_err},+ prelude::{Expr, lambda},+};+use substrait::proto;++use crate::logical_plan::consumer::SubstraitConsumer;++pub async fn from_lambda(+ consumer: &impl SubstraitConsumer,+ expr: &proto::expression::Lambda,+ input_schema: &DFSchema,+) -> datafusion::common::Result<Expr> {+ let Some(parameters) = expr.parameters.as_ref() else {+ return substrait_err!("Lambda expression without parameters is not allowed");+ };++ let names = consumer.push_lambda_parameters(¶meters.types, input_schema)?;++ let Some(body) = expr.body.as_ref() else {+ return substrait_err!("Lambda expression without body is not allowed");+ };++ let body = consumer.consume_expression(body, input_schema).await?;++ consumer.pop_lambda_parameters();++ Ok(lambda(names, body))+}++#[cfg(test)]+mod tests {+ use datafusion::{+ common::{DFSchema, assert_contains},+ prelude::SessionContext,+ };+ use substrait::proto::{self, Expression, r#type::Struct};++ use crate::{+ extensions::Extensions,+ logical_plan::consumer::{DefaultSubstraitConsumer, from_lambda},+ };++ #[tokio::test]+ async fn test_lambda_without_body() {+ let lambda = proto::expression::Lambda {+ parameters: Some(Struct::default()),+ body: None,+ };++ let extensions = Extensions::default();+ let session_state = SessionContext::new().state();+ let consumer = DefaultSubstraitConsumer::new(&extensions, &session_state);++ let err = from_lambda(&consumer, &lambda, DFSchema::empty_ref())+ .await+ .unwrap_err();++ assert_contains!(+ err.to_string(),+ "Lambda expression without body is not allowed"+ );+ }++ #[tokio::test]+ async fn test_lambda_without_parameters() {+ let lambda = proto::expression::Lambda {+ parameters: None,+ body: Some(Box::new(Expression::default())),+ };++ let extensions = Extensions::default();+ let session_state = SessionContext::new().state();+ let consumer = DefaultSubstraitConsumer::new(&extensions, &session_state);++ let err = from_lambda(&consumer, &lambda, DFSchema::empty_ref())+ .await+ .unwrap_err();++ assert_contains!(+ err.to_string(),+ "Lambda expression without parameters is not allowed"+ );+ }+}diff --git a/datafusion/substrait/src/logical_plan/consumer/expr/mod.rs b/datafusion/substrait/src/logical_plan/consumer/expr/mod.rsindex 295456e95f..2fcc11f4e4 100644--- a/datafusion/substrait/src/logical_plan/consumer/expr/mod.rs+++ b/datafusion/substrait/src/logical_plan/consumer/expr/mod.rs@@ -20,6 +20,7 @@ mod cast;mod field_reference;mod function_arguments;mod if_then;+mod lambda;mod literal;mod nested;mod scalar_function;@@ -32,6 +33,7 @@ pub use cast::*;pub use field_reference::*;pub use function_arguments::*;pub use if_then::*;+pub use lambda::*;pub use literal::*;pub use nested::*;pub use scalar_function::*;@@ -95,8 +97,11 @@ pub async fn from_substrait_rex(RexType::DynamicParameter(expr) => {consumer.consume_dynamic_parameter(expr, input_schema).await}- RexType::Lambda(_) | RexType::LambdaInvocation(_) => {- not_impl_err!("Lambda expressions are not yet supported")+ RexType::Lambda(lambda) => {+ consumer.consume_lambda(lambda.as_ref(), input_schema).await+ }+ RexType::LambdaInvocation(_) => {+ not_impl_err!("Lambda invocations are not supported")}},None => substrait_err!("Expression must set rex_type: {expression:?}"),diff --git a/datafusion/substrait/src/logical_plan/consumer/expr/scalar_function.rs b/datafusion/substrait/src/logical_plan/consumer/expr/scalar_function.rsindex 1a0fb3f55f..4cd856fc56 100644--- a/datafusion/substrait/src/logical_plan/consumer/expr/scalar_function.rs+++ b/datafusion/substrait/src/logical_plan/consumer/expr/scalar_function.rs@@ -30,7 +30,6 @@ pub async fn from_scalar_function(f: &ScalarFunction,input_schema: &DFSchema,) -> Result<Expr> {- //TODO: handle higher order functions, as they are also encoded as scalar functionslet Some(fn_signature) = consumer.get_extensions().functions@@ -45,6 +44,20 @@ pub async fn from_scalar_function(let fn_name = substrait_fun_name(fn_signature);let args = from_substrait_func_args(consumer, &f.arguments, input_schema).await?;+ let higher_order_func = consumer+ .get_function_registry()+ .higher_order_function(fn_name)+ .or_else(|e| {+ if let Some(alt_name) = substrait_to_df_name(fn_name) {+ consumer+ .get_function_registry()+ .higher_order_function(alt_name)+ .or(Err(e))+ } else {+ Err(e)+ }+ });+let udf_func = consumer.get_function_registry().udf(fn_name).or_else(|e| {if let Some(alt_name) = substrait_to_df_name(fn_name) {consumer.get_function_registry().udf(alt_name).or(Err(e))@@ -53,9 +66,14 @@ pub async fn from_scalar_function(}});- // try to first match the requested function into registered udfs, then built-in ops+ // try to first match the requested function into registered higher-order functions, then udfs, built-in ops// and finally built-in expressions- if let Ok(func) = udf_func {+ if let Ok(func) = higher_order_func {+ Ok(Expr::HigherOrderFunction(expr::HigherOrderFunction::new(+ func.to_owned(),+ args,+ )))+ } else if let Ok(func) = udf_func {Ok(Expr::ScalarFunction(expr::ScalarFunction::new_udf(func.to_owned(),args,diff --git a/datafusion/substrait/src/logical_plan/consumer/substrait_consumer.rs b/datafusion/substrait/src/logical_plan/consumer/substrait_consumer.rsindex 65bc53ce08..bbd80b4cff 100644--- a/datafusion/substrait/src/logical_plan/consumer/substrait_consumer.rs+++ b/datafusion/substrait/src/logical_plan/consumer/substrait_consumer.rs@@ -23,21 +23,27 @@ use super::{from_substrait_rex, from_window_function,};use crate::extensions::Extensions;+use crate::logical_plan::consumer::{+ field_from_substrait_type_without_names, from_lambda,+};use async_trait::async_trait;-use datafusion::arrow::datatypes::DataType;+use datafusion::arrow::datatypes::{DataType, FieldRef};use datafusion::catalog::TableProvider;+use datafusion::common::datatype::FieldExt;use datafusion::common::{DFSchema, ScalarValue, TableReference, not_impl_err, substrait_err,};use datafusion::execution::{FunctionRegistry, SessionState};+use datafusion::logical_expr::expr::LambdaVariable;use datafusion::logical_expr::{Expr, Extension, LogicalPlan};+use std::collections::VecDeque;use std::sync::{Arc, RwLock};-use substrait::proto;use substrait::proto::expression as substrait_expression;use substrait::proto::expression::{Enum, FieldReference, IfThen, Literal, MultiOrList, Nested, ScalarFunction,SingularOrList, SwitchExpression, WindowFunction,};+use substrait::proto::{self, Type};use substrait::proto::{AggregateRel, ConsistentPartitionWindowRel, CrossRel, DynamicParameter, ExchangeRel,Expression, ExtensionLeafRel, ExtensionMultiRel, ExtensionSingleRel, FetchRel,@@ -62,17 +68,19 @@ use substrait::proto::{/// # use datafusion::logical_expr::{Expr, LogicalPlan, LogicalPlanBuilder};/// # use std::sync::Arc;/// # use substrait::proto;-/// # use substrait::proto::{ExtensionLeafRel, FilterRel, ProjectRel};+/// # use substrait::proto::{ExtensionLeafRel, FilterRel, ProjectRel, Type};/// # use datafusion::arrow::datatypes::DataType;/// # use datafusion::logical_expr::expr::ScalarFunction;/// # use datafusion_substrait::extensions::Extensions;/// # use datafusion_substrait::logical_plan::consumer::{-/// # from_project_rel, from_substrait_rel, from_substrait_rex, SubstraitConsumer+/// # from_project_rel, from_substrait_rel, from_substrait_rex, SubstraitConsumer, DefaultSubstraitLambdaConsumer/// # };////// struct CustomSubstraitConsumer {/// extensions: Arc<Extensions>,/// state: Arc<SessionState>,+/// // You can reuse existing consumer code related to lambdas+/// lambda_consumer: DefaultSubstraitLambdaConsumer,/// }////// #[async_trait]@@ -95,6 +103,30 @@ use substrait::proto::{/// self.state.as_ref()/// }///+/// fn push_lambda_parameters(+/// &self,+/// lambda_parameters: &[Type],+/// input_schema: &DFSchema,+/// ) -> datafusion::common::Result<Vec<String>> {+/// self.lambda_consumer.push_lambda_parameters(+/// self,+/// lambda_parameters,+/// input_schema,+/// )+/// }+///+/// fn pop_lambda_parameters(&self) {+/// self.lambda_consumer.pop_lambda_parameters();+/// }+///+/// fn lambda_variable(+/// &self,+/// steps_out: usize,+/// field_idx: usize,+/// ) -> datafusion::common::Result<Expr> {+/// self.lambda_consumer.lambda_variable(steps_out, field_idx)+/// }+////// // You can reuse existing consumer code to assist in handling advanced extensions/// async fn consume_project(&self, rel: &ProjectRel) -> Result<LogicalPlan> {/// let df_plan = from_project_rel(self, rel).await?;@@ -384,6 +416,14 @@ pub trait SubstraitConsumer: Send + Sync + Sized {))}+ async fn consume_lambda(+ &self,+ expr: &proto::expression::Lambda,+ input_schema: &DFSchema,+ ) -> datafusion::common::Result<Expr> {+ from_lambda(self, expr, input_schema).await+ }+// Outer Schema Stack// These methods manage a stack of outer schemas for correlated subquery support.// When entering a subquery, the enclosing query's schema is pushed onto the stack.@@ -481,6 +521,35 @@ pub trait SubstraitConsumer: Send + Sync + Sized {};substrait_err!("Missing handler for user-defined literals {}", type_ref)}++ // Lambda related methods++ /// Push the given lambda parameters onto the stack when entering a lambda and+ /// returns the names they got assigned+ ///+ /// Note for custom implementations it's possible to embed a [DefaultSubstraitLambdaConsumer] and forward this method to it+ fn push_lambda_parameters(+ &self,+ _lambda_parameters: &[Type],+ _input_schema: &DFSchema,+ ) -> datafusion::common::Result<Vec<String>> {+ not_impl_err!("SubstraitConsumer::push_lambda_parameters")+ }++ /// Pop lambda parameters from the stack when leaving a lambda.+ fn pop_lambda_parameters(&self) {}++ /// Returns an expression corresponding to the lambda variable with the given field_idx within the lambda it originates from,+ /// at the lambda `step_outs` of the current scope+ ///+ /// Note for custom implementations it's possible to embed a [DefaultSubstraitLambdaConsumer] and forward this method to it+ fn lambda_variable(+ &self,+ _steps_out: usize,+ _field_idx: usize,+ ) -> datafusion::common::Result<Expr> {+ not_impl_err!("SubstraitConsumer::lambda_variable")+ }}/// Default SubstraitConsumer for converting standard Substrait without user-defined extensions.@@ -490,6 +559,7 @@ pub struct DefaultSubstraitConsumer<'a> {pub(super) extensions: &'a Extensions,pub(super) state: &'a SessionState,outer_schemas: RwLock<Vec<Arc<DFSchema>>>,+ lambda_consumer: DefaultSubstraitLambdaConsumer,}impl<'a> DefaultSubstraitConsumer<'a> {@@ -498,6 +568,7 @@ impl<'a> DefaultSubstraitConsumer<'a> {extensions,state,outer_schemas: RwLock::new(Vec::new()),+ lambda_consumer: DefaultSubstraitLambdaConsumer::new(),}}}@@ -594,6 +665,140 @@ impl SubstraitConsumer for DefaultSubstraitConsumer<'_> {let plan = plan.with_exprs_and_inputs(plan.expressions(), inputs)?;Ok(LogicalPlan::Extension(Extension { node: plan }))}++ fn push_lambda_parameters(+ &self,+ lambda_parameters: &[Type],+ input_schema: &DFSchema,+ ) -> datafusion::common::Result<Vec<String>> {+ self.lambda_consumer+ .push_lambda_parameters(self, lambda_parameters, input_schema)+ }++ fn pop_lambda_parameters(&self) {+ self.lambda_consumer.pop_lambda_parameters()+ }++ fn lambda_variable(+ &self,+ steps_out: usize,+ field_idx: usize,+ ) -> datafusion::common::Result<Expr> {+ self.lambda_consumer.lambda_variable(steps_out, field_idx)+ }+}++/// Default implementation of lambda related methods of the [SubstraitConsumer] trait+///+/// Can be embedded into a custom [SubstraitConsumer] to implement them+pub struct DefaultSubstraitLambdaConsumer {+ inner: RwLock<DefaultSubstraitLambdaConsumerInner>,+}++struct DefaultSubstraitLambdaConsumerInner {+ /// Parameters of the lambdas currently in scope, ordered from innermost+ /// to outermost. Index 0 is the lambda being consumed; higher indices+ /// are enclosing lambdas, matching the `steps_out` value used by+ /// [`DefaultSubstraitLambdaConsumer::lambda_variable`] and `LambdaParameterReference`.+ lambda_parameters: VecDeque<Vec<FieldRef>>,+ next_lambda_parameter: usize,+}++impl Default for DefaultSubstraitLambdaConsumer {+ fn default() -> Self {+ Self::new()+ }+}++impl DefaultSubstraitLambdaConsumer {+ pub fn new() -> Self {+ Self {+ inner: RwLock::new(DefaultSubstraitLambdaConsumerInner {+ lambda_parameters: VecDeque::new(),+ next_lambda_parameter: 0,+ }),+ }+ }++ pub fn push_lambda_parameters(+ &self,+ consumer: &impl SubstraitConsumer,+ lambda_parameters: &[Type],+ input_schema: &DFSchema,+ ) -> datafusion::common::Result<Vec<String>> {+ let mut inner = self.inner.write().unwrap();++ let lambda_parameters = lambda_parameters+ .iter()+ .map(|ty| {+ let (assigned_number, default_name) =+ next_lambda_parameter_name(inner.next_lambda_parameter, input_schema);++ inner.next_lambda_parameter = assigned_number + 1;++ Ok(field_from_substrait_type_without_names(consumer, ty)?+ .renamed(&default_name))+ })+ .collect::<datafusion::common::Result<Vec<_>>>()?;++ let names = lambda_parameters.iter().map(|f| f.name().clone()).collect();++ inner.lambda_parameters.push_front(lambda_parameters);++ Ok(names)+ }++ pub fn pop_lambda_parameters(&self) {+ self.inner.write().unwrap().lambda_parameters.pop_front();+ }++ pub fn lambda_variable(+ &self,+ steps_out: usize,+ field_idx: usize,+ ) -> datafusion::common::Result<Expr> {+ let lambda_parameters = &self.inner.read().unwrap().lambda_parameters;++ let Some(lambda_parameters) = lambda_parameters.get(steps_out) else {+ return substrait_err!(+ "No lambda at {steps_out} steps out, got only {}",+ lambda_parameters.len()+ );+ };++ let Some(var) = lambda_parameters.get(field_idx) else {+ return substrait_err!(+ "At lambda {steps_out} steps out, no field at index {field_idx}, got only {}",+ lambda_parameters.len()+ );+ };++ Ok(Expr::LambdaVariable(LambdaVariable::new(+ var.name().clone(),+ Some(Arc::clone(var)),+ )))+ }+}++/// Returns the next available lambda parameter name and the index it was assigned.+///+/// Names follow the pattern `pN` where `N` starts at `next_lambda_parameter`. If `pN`+/// conflicts with an existing column name in `input_schema`, `N` is incremented until+/// a free name is found.+fn next_lambda_parameter_name(+ mut next_lambda_parameter: usize,+ input_schema: &DFSchema,+) -> (usize, String) {+ loop {+ let name = format!("p{next_lambda_parameter}");++ // avoid conflicts with column names+ if !input_schema.has_column_with_unqualified_name(&name) {+ return (next_lambda_parameter, name);+ }++ next_lambda_parameter += 1;+ }}#[cfg(test)]diff --git a/datafusion/substrait/src/logical_plan/producer/expr/lambda.rs b/datafusion/substrait/src/logical_plan/producer/expr/lambda.rsnew file mode 100644index 0000000000..0d32dab2cc--- /dev/null+++ b/datafusion/substrait/src/logical_plan/producer/expr/lambda.rs@@ -0,0 +1,48 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use datafusion::{common::DFSchemaRef, logical_expr::expr::Lambda};+use substrait::proto::{+ Expression,+ expression::RexType,+ r#type::{Nullability, Struct},+};++use crate::logical_plan::producer::SubstraitProducer;++pub fn from_lambda(+ producer: &mut impl SubstraitProducer,+ lambda: &Lambda,+ schema: &DFSchemaRef,+) -> Result<Expression, datafusion::error::DataFusionError> {+ Ok(Expression {+ rex_type: Some(RexType::Lambda(Box::new(+ substrait::proto::expression::Lambda {+ parameters: Some(Struct {+ nullability: Nullability::Required as i32,+ type_variation_reference: 0,+ types: lambda+ .params+ .iter()+ .map(|p| producer.lambda_parameter_type(p))+ .collect::<datafusion::error::Result<_>>()?,+ }),+ body: Some(Box::new(producer.handle_expr(&lambda.body, schema)?)),+ },+ ))),+ })+}diff --git a/datafusion/substrait/src/logical_plan/producer/expr/lambda_variable.rs b/datafusion/substrait/src/logical_plan/producer/expr/lambda_variable.rsnew file mode 100644index 0000000000..3d7f06e233--- /dev/null+++ b/datafusion/substrait/src/logical_plan/producer/expr/lambda_variable.rs@@ -0,0 +1,49 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use datafusion::logical_expr::expr::LambdaVariable;+use substrait::proto::{+ Expression,+ expression::{+ FieldReference, ReferenceSegment, RexType,+ field_reference::{LambdaParameterReference, ReferenceType, RootType},+ reference_segment::{self, StructField},+ },+};++use crate::logical_plan::producer::SubstraitProducer;++pub fn from_lambda_variable(+ producer: &mut impl SubstraitProducer,+ lambda_variable: &LambdaVariable,+ _schema: &datafusion::common::DFSchema,+) -> Result<Expression, datafusion::error::DataFusionError> {+ let (steps_out, field) = producer.lambda_variable(&lambda_variable.name)?;++ Ok(Expression {+ rex_type: Some(RexType::Selection(Box::new(FieldReference {+ reference_type: Some(ReferenceType::DirectReference(ReferenceSegment {+ reference_type: Some(reference_segment::ReferenceType::StructField(+ Box::new(StructField { field, child: None }),+ )),+ })),+ root_type: Some(RootType::LambdaParameterReference(+ LambdaParameterReference { steps_out },+ )),+ }))),+ })+}diff --git a/datafusion/substrait/src/logical_plan/producer/expr/mod.rs b/datafusion/substrait/src/logical_plan/producer/expr/mod.rsindex 6e053f0d90..c728af2f14 100644--- a/datafusion/substrait/src/logical_plan/producer/expr/mod.rs+++ b/datafusion/substrait/src/logical_plan/producer/expr/mod.rs@@ -19,6 +19,8 @@ mod aggregate_function;mod cast;mod field_reference;mod if_then;+mod lambda;+mod lambda_variable;mod literal;mod placeholder;mod scalar_function;@@ -30,6 +32,8 @@ pub use aggregate_function::*;pub use cast::*;pub use field_reference::*;pub use if_then::*;+pub use lambda::*;+pub use lambda_variable::*;pub use literal::*;pub use placeholder::*;pub use scalar_function::*;@@ -154,10 +158,8 @@ pub fn to_substrait_rex(Expr::HigherOrderFunction(expr) => {producer.handle_higher_order_function(expr, schema)}- Expr::Lambda(expr) => not_impl_err!("Cannot convert {expr:?} to Substrait"),- Expr::LambdaVariable(expr) => {- not_impl_err!("Cannot convert {expr:?} to Substrait")- }+ Expr::Lambda(expr) => producer.handle_lambda(expr, schema),+ Expr::LambdaVariable(expr) => producer.handle_lambda_variable(expr, schema),}}diff --git a/datafusion/substrait/src/logical_plan/producer/expr/scalar_function.rs b/datafusion/substrait/src/logical_plan/producer/expr/scalar_function.rsindex e36d5128cd..e7dd2af13f 100644--- a/datafusion/substrait/src/logical_plan/producer/expr/scalar_function.rs+++ b/datafusion/substrait/src/logical_plan/producer/expr/scalar_function.rs@@ -16,7 +16,10 @@// under the License.use crate::logical_plan::producer::{SubstraitProducer, to_substrait_literal_expr};-use datafusion::common::{DFSchemaRef, ScalarValue, not_impl_err};+use datafusion::common::datatype::FieldExt;+use datafusion::common::{+ DFSchemaRef, ScalarValue, internal_datafusion_err, not_impl_err, substrait_err,+};use datafusion::logical_expr::{Between, BinaryExpr, Expr, Like, Operator, expr};use substrait::proto::expression::{RexType, ScalarFunction};use substrait::proto::function_argument::ArgType;@@ -35,7 +38,78 @@ pub fn from_higher_order_function(fun: &expr::HigherOrderFunction,schema: &DFSchemaRef,) -> datafusion::common::Result<Expression> {- from_function(producer, fun.name(), &fun.args, schema)+ let mut lambda_parameters = fun.lambda_parameters(schema)?.into_iter();++ let num_lambdas = fun+ .args+ .iter()+ .filter(|arg| matches!(arg, Expr::Lambda(_)))+ .count();++ if lambda_parameters.len() != num_lambdas {+ return substrait_err!(+ "{} returned {} lambdas but {num_lambdas} expected",+ fun.name(),+ lambda_parameters.len()+ );+ }++ let arguments = fun+ .args+ .iter()+ .map(|arg| {+ let arg = match arg {+ Expr::Lambda(l) => {+ let lambda_parameters =+ lambda_parameters.next().ok_or_else(|| {+ internal_datafusion_err!(+ "lambda_parameters len should have been checked above"+ )+ })?;++ if l.params.len() > lambda_parameters.len() {+ return substrait_err!(+ "Lambda defined {} parameters ({}) but function {} supports only {}",+ l.params.len(),+ l.params.join(","),+ fun.name(),+ lambda_parameters.len()+ )+ }++ let named_lambda_parameters =+ std::iter::zip(&l.params, lambda_parameters)+ .map(|(name, parameter)| parameter.renamed(name))+ .collect();++ producer.push_lambda_parameters(named_lambda_parameters)?;++ let arg = producer.handle_lambda(l, schema);++ producer.pop_lambda_parameters()?;++ arg+ }+ _ => producer.handle_expr(arg, schema),+ }?;++ Ok(FunctionArgument {+ arg_type: Some(ArgType::Value(arg)),+ })+ })+ .collect::<datafusion::common::Result<_>>()?;++ let function_anchor = producer.register_function(fun.name().to_string());+ #[expect(deprecated)]+ Ok(Expression {+ rex_type: Some(RexType::ScalarFunction(ScalarFunction {+ function_reference: function_anchor,+ arguments,+ output_type: None,+ options: vec![],+ args: vec![],+ })),+ })}fn from_function(diff --git a/datafusion/substrait/src/logical_plan/producer/substrait_producer.rs b/datafusion/substrait/src/logical_plan/producer/substrait_producer.rsindex 4228c32435..6d54d32cad 100644--- a/datafusion/substrait/src/logical_plan/producer/substrait_producer.rs+++ b/datafusion/substrait/src/logical_plan/producer/substrait_producer.rs@@ -20,18 +20,23 @@ use crate::logical_plan::producer::{from_aggregate, from_aggregate_function, from_alias, from_between, from_binary_expr,from_case, from_cast, from_column, from_distinct, from_empty_relation, from_exists,from_filter, from_higher_order_function, from_in_list, from_in_subquery, from_join,- from_like, from_limit, from_literal, from_placeholder, from_projection,- from_repartition, from_scalar_function, from_scalar_subquery, from_set_comparison,- from_sort, from_subquery_alias, from_table_scan, from_try_cast, from_unary_expr,- from_union, from_values, from_window, from_window_function, to_substrait_rel,- to_substrait_rex,+ from_lambda, from_lambda_variable, from_like, from_limit, from_literal,+ from_placeholder, from_projection, from_repartition, from_scalar_function,+ from_scalar_subquery, from_set_comparison, from_sort, from_subquery_alias,+ from_table_scan, from_try_cast, from_unary_expr, from_union, from_values,+ from_window, from_window_function, to_substrait_rel, to_substrait_rex,+ to_substrait_type_from_field,+};+use datafusion::arrow::datatypes::FieldRef;+use datafusion::common::{+ Column, DFSchemaRef, HashMap, ScalarValue, not_impl_err, substrait_err,};-use datafusion::common::{Column, DFSchemaRef, ScalarValue, substrait_err};use datafusion::execution::SessionState;use datafusion::execution::registry::SerializerRegistry;use datafusion::logical_expr::Subquery;use datafusion::logical_expr::expr::{- Alias, Exists, InList, InSubquery, Placeholder, SetComparison, WindowFunction,+ Alias, Exists, InList, InSubquery, Lambda, LambdaVariable, Placeholder,+ SetComparison, WindowFunction,};use datafusion::logical_expr::{Aggregate, Between, BinaryExpr, Case, Cast, Distinct, EmptyRelation, Expr, Extension,@@ -57,16 +62,19 @@ use substrait::proto::{/// # use std::sync::Arc;/// # use substrait::proto::{Expression, Rel};/// # use substrait::proto::rel::RelType;+/// # use datafusion::arrow::datatypes::FieldRef;/// # use datafusion::common::DFSchemaRef;/// # use datafusion::error::Result;/// # use datafusion::execution::SessionState;/// # use datafusion::logical_expr::{Between, Extension, Projection};/// # use datafusion_substrait::extensions::Extensions;-/// # use datafusion_substrait::logical_plan::producer::{from_projection, SubstraitProducer};+/// # use datafusion_substrait::logical_plan::producer::{from_projection, SubstraitProducer, DefaultSubstraitLambdaProducer, lambda_parameters_map};////// struct CustomSubstraitProducer {/// extensions: Extensions,/// state: Arc<SessionState>,+/// // You can reuse existing producer code related to lambdas+/// lambda_producer: DefaultSubstraitLambdaProducer,/// }////// impl SubstraitProducer for CustomSubstraitProducer {@@ -83,6 +91,33 @@ use substrait::proto::{/// self.extensions/// }///+/// fn push_lambda_parameters(+/// &mut self,+/// lambda_parameters: Vec<FieldRef>,+/// ) -> datafusion::common::Result<()> {+/// let lambda_parameters_map = lambda_parameters_map(self, lambda_parameters)?;+///+/// self.lambda_producer+/// .push_lambda_parameters(lambda_parameters_map);+///+/// Ok(())+/// }+///+/// fn pop_lambda_parameters(&mut self) -> datafusion::common::Result<()> {+/// self.lambda_producer.pop_lambda_parameters()+/// }+///+/// fn lambda_variable(&self, name: &str) -> datafusion::common::Result<(u32, i32)> {+/// self.lambda_producer.lambda_variable(name)+/// }+///+/// fn lambda_parameter_type(+/// &self,+/// name: &str,+/// ) -> datafusion::common::Result<substrait::proto::Type> {+/// self.lambda_producer.lambda_parameter_type(name)+/// }+////// // You can set additional metadata on the Rels you produce/// fn handle_projection(&mut self, plan: &Projection) -> Result<Box<Rel>> {/// let mut rel = from_projection(self, plan)?;@@ -405,11 +440,65 @@ pub trait SubstraitProducer: Send + Sync + Sized {) -> datafusion::common::Result<Expression> {from_placeholder(self, placeholder)}++ fn handle_lambda(+ &mut self,+ lambda: &Lambda,+ schema: &DFSchemaRef,+ ) -> datafusion::common::Result<Expression> {+ from_lambda(self, lambda, schema)+ }++ fn handle_lambda_variable(+ &mut self,+ lambda_variable: &LambdaVariable,+ schema: &DFSchemaRef,+ ) -> datafusion::common::Result<Expression> {+ from_lambda_variable(self, lambda_variable, schema)+ }++ // Lambda related methods++ /// Push the given `lambda_parameters` into this producer so they can be referenced by lambda variables+ ///+ /// Note for custom implementations it's possible to embed a [DefaultSubstraitLambdaProducer] and forward this method to it+ fn push_lambda_parameters(+ &mut self,+ _lambda_parameters: Vec<FieldRef>,+ ) -> datafusion::common::Result<()> {+ not_impl_err!("SubstraitProducer::push_lambda_parameters")+ }++ /// Pop the last pushed `lambda_parameters` so that it unshadow any previously shadowed lambda parameter+ ///+ /// Note for custom implementations it's possible to embed a [DefaultSubstraitLambdaProducer] and forward this method to it+ fn pop_lambda_parameters(&mut self) -> datafusion::common::Result<()> {+ not_impl_err!("SubstraitProducer::pop_lambda_parameters")+ }++ /// Get the (`steps_out`, `field_idx`) of the lambda variable with the given `name`. `steps_out` refers to the number+ /// of lambda boundaries to traverse (0 = current lambda), and `field_idx` refers to the index within the lambda parameters+ ///+ /// Note for custom implementations it's possible to embed a [DefaultSubstraitLambdaProducer] and forward this method to it+ fn lambda_variable(&self, _name: &str) -> datafusion::common::Result<(u32, i32)> {+ not_impl_err!("SubstraitProducer::lambda_variable")+ }++ /// Get the type of the lambda parameter with the given `name`+ ///+ /// Note for custom implementations it's possible to embed a [DefaultSubstraitLambdaProducer] and forward this method to it+ fn lambda_parameter_type(+ &self,+ _name: &str,+ ) -> datafusion::common::Result<substrait::proto::Type> {+ not_impl_err!("SubstraitProducer::lambda_parameter_type")+ }}pub struct DefaultSubstraitProducer<'a> {extensions: Extensions,serializer_registry: &'a dyn SerializerRegistry,+ lambda_producer: DefaultSubstraitLambdaProducer,}impl<'a> DefaultSubstraitProducer<'a> {@@ -417,6 +506,7 @@ impl<'a> DefaultSubstraitProducer<'a> {DefaultSubstraitProducer {extensions: Extensions::default(),serializer_registry: state.serializer_registry().as_ref(),+ lambda_producer: DefaultSubstraitLambdaProducer::new(),}}}@@ -471,4 +561,109 @@ impl SubstraitProducer for DefaultSubstraitProducer<'_> {rel_type: Some(rel_type),}))}++ fn push_lambda_parameters(+ &mut self,+ lambda_parameters: Vec<FieldRef>,+ ) -> datafusion::common::Result<()> {+ let lambda_parameters_map = lambda_parameters_map(self, lambda_parameters)?;++ self.lambda_producer+ .push_lambda_parameters(lambda_parameters_map);++ Ok(())+ }++ fn pop_lambda_parameters(&mut self) -> datafusion::common::Result<()> {+ self.lambda_producer.pop_lambda_parameters()+ }++ fn lambda_variable(&self, name: &str) -> datafusion::common::Result<(u32, i32)> {+ self.lambda_producer.lambda_variable(name)+ }++ fn lambda_parameter_type(+ &self,+ name: &str,+ ) -> datafusion::common::Result<substrait::proto::Type> {+ self.lambda_producer.lambda_parameter_type(name)+ }+}++/// Default implementation of lambda related methods of the [SubstraitProducer] trait+///+/// Can be embedded into a custom [SubstraitProducer] to implement them+pub struct DefaultSubstraitLambdaProducer {+ lambdas_variables: Vec<HashMap<String, (usize, substrait::proto::Type)>>,+}++impl Default for DefaultSubstraitLambdaProducer {+ fn default() -> Self {+ Self::new()+ }+}++impl DefaultSubstraitLambdaProducer {+ pub fn new() -> Self {+ Self {+ lambdas_variables: Vec::new(),+ }+ }++ /// Note you can construct the `lambda_parameters` argument using [lambda_parameters_map]+ pub fn push_lambda_parameters(+ &mut self,+ lambda_parameters: HashMap<String, (usize, substrait::proto::Type)>,+ ) {+ self.lambdas_variables.push(lambda_parameters);+ }++ pub fn pop_lambda_parameters(&mut self) -> datafusion::common::Result<()> {+ match self.lambdas_variables.pop() {+ Some(_) => Ok(()),+ None => substrait_err!("no lambda_parameters to pop"),+ }+ }++ pub fn lambda_variable(&self, name: &str) -> datafusion::common::Result<(u32, i32)> {+ for (steps_out, lambda_parameters) in+ self.lambdas_variables.iter().rev().enumerate()+ {+ if let Some((field_idx, _type)) = lambda_parameters.get(name) {+ return Ok((steps_out as u32, *field_idx as i32));+ }+ }++ substrait_err!("unknown lambda variable {name}")+ }++ pub fn lambda_parameter_type(+ &self,+ name: &str,+ ) -> datafusion::common::Result<substrait::proto::Type> {+ for lambda_parameters in self.lambdas_variables.iter().rev() {+ if let Some((_field_idx, type_)) = lambda_parameters.get(name) {+ return Ok(type_.clone());+ }+ }++ substrait_err!("unknown lambda variable {name}")+ }+}++/// Produces a map of lambda parameters as expected by [DefaultSubstraitLambdaProducer::push_lambda_parameters]+pub fn lambda_parameters_map(+ producer: &mut impl SubstraitProducer,+ lambda_parameters: Vec<FieldRef>,+) -> datafusion::common::Result<HashMap<String, (usize, substrait::proto::Type)>> {+ lambda_parameters+ .into_iter()+ .enumerate()+ .map(|(field_idx, field)| {+ Ok((+ field.name().clone(),+ (field_idx, to_substrait_type_from_field(producer, &field)?),+ ))+ })+ .collect::<datafusion::common::Result<_>>()}diff --git a/datafusion/substrait/tests/cases/logical_plans.rs b/datafusion/substrait/tests/cases/logical_plans.rsindex 663a372fe2..d4ac01462c 100644--- a/datafusion/substrait/tests/cases/logical_plans.rs+++ b/datafusion/substrait/tests/cases/logical_plans.rs@@ -19,6 +19,7 @@#[cfg(test)]mod tests {+ use crate::cases::roundtrip_logical_plan::higher_order_function_ctx;use crate::utils::test::{add_plan_schemas_to_ctx, read_json};use datafusion::common::test_util::format_batches;use std::collections::HashSet;@@ -293,4 +294,25 @@ mod tests {Ok(())}++ #[tokio::test]+ async fn higher_order_function() -> Result<()> {+ let proto_plan =+ read_json("tests/testdata/test_plans/higher_order_function.json");+ // ctx already contains the queried table+ let ctx = higher_order_function_ctx().await?;+ let plan = from_substrait_plan(&ctx.state(), &proto_plan).await?;++ assert_snapshot!(+ plan,+ @"+ Projection: array_transform2(make_array(make_array(data3.p1)), (p0, p2) -> array_concat(array_transform2(p0, (p3, p4) -> p3 * p2 * p4), array_transform2(p0, (p5, p6) -> p5 * p2 * p6))) AS array_transform2(make_array(make_array(data3.p1)),(v, i) -> array_concat(array_transform2(v,(v, j) -> v * i * j),array_transform2(v,(v, j) -> v * i * j)))+ TableScan: data3+ "+ );++ // Trigger execution to ensure plan validity+ DataFrame::new(ctx.state(), plan).show().await?;+ Ok(())+ }}diff --git a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rsindex 872c2d0cd2..b5c8912a2e 100644--- a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs+++ b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs@@ -17,8 +17,13 @@use crate::utils::test::read_json;use datafusion::arrow::array::ArrayRef;+use datafusion::config::Dialect;use datafusion::functions_nested::map::map;-use datafusion::logical_expr::LogicalPlanBuilder;+use datafusion::logical_expr::{+ ColumnarValue, HigherOrderFunctionArgs, HigherOrderReturnFieldArgs,+ HigherOrderSignature, HigherOrderUDF, LambdaParametersProgress, LogicalPlanBuilder,+ ValueOrLambda,+};use datafusion::physical_plan::Accumulator;use datafusion::scalar::ScalarValue;use datafusion_substrait::logical_plan::{@@ -27,7 +32,9 @@ use datafusion_substrait::logical_plan::{use std::cmp::Ordering;use std::mem::size_of_val;-use datafusion::arrow::datatypes::{DataType, Field, IntervalUnit, Schema, TimeUnit};+use datafusion::arrow::datatypes::{+ DataType, Field, FieldRef, IntervalUnit, Schema, TimeUnit,+};use datafusion::common::tree_node::Transformed;use datafusion::common::{DFSchema, DFSchemaRef, Spans, not_impl_err, plan_err};use datafusion::error::Result;@@ -1943,6 +1950,215 @@ async fn roundtrip_placeholder_typed_utf8() -> Result<()> {Ok(())}+#[tokio::test]+async fn roundtrip_array_transform_higher_order_function() -> Result<()> {+ let ctx = higher_order_function_ctx().await?;++ // simple+ roundtrip_with_ctx(+ "SELECT array_transform2([data3.p1], p0 -> p0 * 2) from data3",+ ctx.clone(),+ )+ .await?;++ // dont use the parameter+ roundtrip_with_ctx(+ "SELECT array_transform2([data3.p1], p0 -> 3) from data3",+ ctx.clone(),+ )+ .await?;++ // multiple parameters using both+ roundtrip_with_ctx(+ "SELECT array_transform2([data3.p1], (p0, p2) -> p0 * p2) from data3",+ ctx.clone(),+ )+ .await?;++ // multiple parameters only last+ roundtrip_with_ctx(+ "SELECT array_transform2([data3.p1], (p0, p2) -> 2 * p2) from data3",+ ctx.clone(),+ )+ .await?;++ // multiple parameters use none+ roundtrip_with_ctx(+ "SELECT array_transform2([data3.p1], (p0, p2) -> 3) from data3",+ ctx.clone(),+ )+ .await?;++ // nested without variable shadowing+ roundtrip_with_ctx("SELECT array_transform2([[data3.p1]], p0 -> array_transform2(p0, p2 -> p2 * 2)) from data3", ctx.clone())+ .await?;++ // nested with multiple parameters without variable shadowing+ roundtrip_with_ctx("SELECT array_transform2([[data3.p1]], (p0, p2) -> array_transform2(p0, (p3, p4) -> p2 * p3 * p4)) from data3", ctx.clone())+ .await?;++ // since substrait doesn't encode lambda parameters names, they got generated, non-conflicting names during consumption+ // testing name shadowing requires to assert against the generated plan and check the correct parameter usage instead of round tripping++ // nested with variable shadowing.+ let plan = generate_plan_from_sql_with_ctx(+ "SELECT array_transform2([[data3.p1]], v -> array_transform2(v, v -> v * 2)) from data3",+ true,+ true,+ &ctx,+ )+ .await?;++ assert_snapshot!(+ plan,+ @"+ Projection: array_transform2(make_array(make_array(data3.p1)), (p0) -> array_transform2(p0, (p2) -> p2 * Int64(2))) AS array_transform2(make_array(make_array(data3.p1)),(v) -> array_transform2(v,(v) -> v * Int64(2)))+ TableScan: data3 projection=[p1]+ "+ );++ // nested with variable shadowing with multiple parameters+ let plan = generate_plan_from_sql_with_ctx(+ "SELECT array_transform2([[data3.p1]], (v, i) -> array_transform2(v, (v, i) -> v * i)) from data3",+ true,+ true,+ &ctx,+ )+ .await?;++ assert_snapshot!(+ plan,+ @"+ Projection: array_transform2(make_array(make_array(data3.p1)), (p0, p2) -> array_transform2(p0, (p3, p4) -> p3 * p4)) AS array_transform2(make_array(make_array(data3.p1)),(v, i) -> array_transform2(v,(v, i) -> v * i))+ TableScan: data3 projection=[p1]+ "+ );++ // nested with variable shadowing and later reuse of the shadowed var after exiting the shadowing expression+ let plan = generate_plan_from_sql_with_ctx(+ "SELECT array_transform2(+ [[data3.p1]],+ v -> array_concat(+ -- when entering this expression, inner v is pushed into the producer and shadows outer v, but after exiting this,+ -- it should be removed and unshadow the outer v, so that it can be used in the next expression+ array_transform2(v, v -> v * 2),+ array_transform2(v, v -> v * 2)+ )+ ) from data3",+ true,+ true,+ &ctx,+ )+ .await?;++ assert_snapshot!(+ plan,+ @"+ Projection: array_transform2(make_array(make_array(data3.p1)), (p0) -> array_concat(array_transform2(p0, (p2) -> p2 * Int64(2)), array_transform2(p0, (p3) -> p3 * Int64(2)))) AS array_transform2(make_array(make_array(data3.p1)),(v) -> array_concat(array_transform2(v,(v) -> v * Int64(2)),array_transform2(v,(v) -> v * Int64(2))))+ TableScan: data3 projection=[p1]+ "+ );++ Ok(())+}++pub(crate) async fn higher_order_function_ctx() -> Result<SessionContext> {+ let ctx = create_context_with_dialect(Some(Dialect::Databricks)).await?;++ ctx.register_higher_order_function(Arc::new(ArrayTransform::new()));++ let data3_fields = vec![+ Field::new("p1", DataType::Int64, true), // lambda parameters should not conflict with this column+ ];+ let data3 = Schema::new(data3_fields);+ let mut data3_options = CsvReadOptions::new();+ data3_options.schema = Some(&data3);+ data3_options.has_header = false;+ ctx.register_csv("data3", "tests/testdata/empty.csv", data3_options)+ .await?;++ Ok(ctx)+}++// todo use core array_transform when it supports multiple lambda parameters+#[derive(Debug, PartialEq, Eq, Hash)]+struct ArrayTransform {+ signature: HigherOrderSignature,+}++impl ArrayTransform {+ fn new() -> Self {+ Self {+ signature: HigherOrderSignature::variadic_any(Volatility::Immutable),+ }+ }+}++impl HigherOrderUDF for ArrayTransform {+ fn name(&self) -> &str {+ "array_transform2"+ }++ fn aliases(&self) -> &[String] {+ &[]+ }++ fn signature(&self) -> &HigherOrderSignature {+ &self.signature+ }++ fn lambda_parameters(+ &self,+ _step: usize,+ fields: &[ValueOrLambda<FieldRef, Option<FieldRef>>],+ ) -> Result<LambdaParametersProgress> {+ let [ValueOrLambda::Value(list), ValueOrLambda::Lambda(_)] = fields else {+ unreachable!()+ };++ let field = match list.data_type() {+ DataType::List(field) => field,+ _ => unreachable!(),+ };++ Ok(LambdaParametersProgress::Complete(vec![vec![+ Arc::clone(field),+ Arc::new(Field::new("", DataType::Int64, true)),+ ]]))+ }++ fn return_field_from_args(+ &self,+ args: HigherOrderReturnFieldArgs,+ ) -> Result<FieldRef> {+ let [ValueOrLambda::Value(list), ValueOrLambda::Lambda(lambda)] = args.arg_fields+ else {+ unreachable!()+ };++ let field = Arc::new(Field::new(+ Field::LIST_FIELD_DEFAULT_NAME,+ lambda.data_type().clone(),+ lambda.is_nullable(),+ ));++ let return_type = match list.data_type() {+ DataType::List(_) => DataType::List(field),+ _ => unreachable!(),+ };++ Ok(Arc::new(Field::new("", return_type, list.is_nullable())))+ }++ fn invoke_with_args(&self, args: HigherOrderFunctionArgs) -> Result<ColumnarValue> {+ // this function is only tested with roundtrip_with_ctx, which only prints the output+ // and generate_plan_from_sql_with_ctx which doesn't execute nothing, so the output doesn't matter+ Ok(ColumnarValue::Scalar(ScalarValue::new_default(+ args.return_type(),+ )?))+ }+}+fn check_post_join_filters(rel: &Rel) -> Result<()> {// search for target_rel and field value in protomatch &rel.rel_type {@@ -2084,6 +2300,15 @@ async fn generate_plan_from_sql(optimized: bool,) -> Result<LogicalPlan> {let ctx = create_context().await?;+ generate_plan_from_sql_with_ctx(sql, assert_schema, optimized, &ctx).await+}++async fn generate_plan_from_sql_with_ctx(+ sql: &str,+ assert_schema: bool,+ optimized: bool,+ ctx: &SessionContext,+) -> Result<LogicalPlan> {let df: DataFrame = ctx.sql(sql).await?;let plan = if optimized {@@ -2437,8 +2662,18 @@ async fn roundtrip_all_types(sql: &str) -> Result<()> {}async fn create_context() -> Result<SessionContext> {+ create_context_with_dialect(None).await+}++async fn create_context_with_dialect(dialect: Option<Dialect>) -> Result<SessionContext> {+ let mut session_config = SessionConfig::default();++ if let Some(dialect) = dialect {+ session_config.options_mut().sql_parser.dialect = dialect;+ }+let mut state = SessionStateBuilder::new()- .with_config(SessionConfig::default())+ .with_config(session_config).with_runtime_env(Arc::new(RuntimeEnv::default())).with_default_features().with_serializer_registry(Arc::new(MockSerializerRegistry))diff --git a/datafusion/substrait/tests/cases/serialize.rs b/datafusion/substrait/tests/cases/serialize.rsindex 2d7257fad3..1981ef66db 100644--- a/datafusion/substrait/tests/cases/serialize.rs+++ b/datafusion/substrait/tests/cases/serialize.rs@@ -28,9 +28,16 @@ mod tests {use insta::assert_snapshot;use std::fs;+ use substrait::proto::expression::field_reference::{ReferenceType, RootType};+ use substrait::proto::expression::reference_segment;+ use substrait::proto::expression::{ReferenceSegment, RexType};+ use substrait::proto::function_argument::ArgType;use substrait::proto::plan_rel::RelType;use substrait::proto::rel_common::{Emit, EmitKind};- use substrait::proto::{RelCommon, rel};+ use substrait::proto::r#type::{I64, Kind as TypeKind, List, Nullability, Struct};+ use substrait::proto::{Expression, RelCommon, Type, rel};++ use crate::cases::roundtrip_logical_plan::higher_order_function_ctx;#[tokio::test]async fn serialize_to_file() -> Result<()> {@@ -196,6 +203,101 @@ mod tests {panic!("plan did not match expected structure")}+ #[tokio::test]+ async fn higher_order_function() -> Result<()> {+ let ctx = higher_order_function_ctx().await?;+ let df = ctx+ .sql(+ "SELECT array_transform2(+ [[data3.p1]],+ (v, i) -> array_concat(+ -- when entering this expression, inner v is pushed into the producer and shadows outer v, but after exiting this,+ -- it should be removed and unshadow the outer v, so that it can be used in the next expression+ array_transform2(v, (v, j) -> v * i * j),+ array_transform2(v, (v, j) -> v * i * j)+ )+ ) from data3"+ )+ .await?;+ let datafusion_plan = df.into_optimized_plan()?;+ let plan = to_substrait_plan(&datafusion_plan, &ctx.state())?+ .as_ref()+ .clone();++ let relation = plan.relations.first().unwrap().rel_type.as_ref();+ let root_rel = match relation {+ Some(RelType::Root(root)) => root.input.as_ref().unwrap(),+ _ => panic!("expected Root"),+ };++ let Some(rel::RelType::Project(p)) = root_rel.rel_type.as_ref() else {+ panic!("expected Project at top of plan")+ };++ let mut params = vec![];+ let mut lambda_refs = vec![];++ collect_lambda_ref(&p.expressions[0], &mut params, &mut lambda_refs);++ let nullable_i64 = Type {+ kind: Some(TypeKind::I64(I64 {+ type_variation_reference: 0,+ nullability: Nullability::Nullable as i32,+ })),+ };++ let inner_lambda_struct = Struct {+ // v, j+ types: vec![nullable_i64.clone(); 2],+ type_variation_reference: 0,+ nullability: Nullability::Required as i32,+ };++ assert_eq!(+ params,+ vec![+ Struct {+ types: vec![+ // v+ Type {+ kind: Some(TypeKind::List(Box::new(List {+ r#type: Some(Box::new(nullable_i64.clone())),+ type_variation_reference: 0,+ nullability: Nullability::Nullable as i32+ })))+ },+ // i+ nullable_i64,+ ],+ type_variation_reference: 0,+ nullability: Nullability::Required as i32,+ },+ inner_lambda_struct.clone(),+ inner_lambda_struct,+ ]+ );++ assert_eq!(+ lambda_refs,+ vec![+ // first inner array_transform2 argument: outer v+ (0, 0),+ // first inner lambda body: v * i * j+ (0, 0),+ (1, 1),+ (0, 1),+ // second inner array_transform2 argument: outer v+ (0, 0),+ // second inner lambda body: v * i * j+ (0, 0),+ (1, 1),+ (0, 1),+ ]+ );++ Ok(())+ }+fn assert_emit(rel_common: Option<&RelCommon>, output_mapping: Vec<i32>) {assert_eq!(rel_common.unwrap().emit_kind.clone(),@@ -211,4 +313,54 @@ mod tests {.await?;Ok(ctx)}++ // Recursively walks a expression tree depth-first, collecting in visit order:+ // - `params`: the parameter struct of each Lambda encountered+ // - `lambda_refs`: every field reference whose root is a LambdaParameterReference,+ // recorded as (steps_out, field_index) so tests can assert which enclosing+ // lambda each reference resolves to and which parameter within it.+ fn collect_lambda_ref(+ expr: &Expression,+ params: &mut Vec<Struct>,+ lambda_refs: &mut Vec<(u32, i32)>,+ ) {+ if let Some(rex_type) = &expr.rex_type {+ match rex_type {+ RexType::Selection(field_reference) => {+ if let (+ Some(ReferenceType::DirectReference(ReferenceSegment {+ reference_type:+ Some(reference_segment::ReferenceType::StructField(+ struct_field,+ )),+ })),+ Some(RootType::LambdaParameterReference(lambda_param_ref)),+ ) = (&field_reference.reference_type, &field_reference.root_type)+ {+ lambda_refs.push((lambda_param_ref.steps_out, struct_field.field))+ }+ }+ RexType::ScalarFunction(scalar_function) => {+ for arg in &scalar_function.arguments {+ match &arg.arg_type {+ Some(ArgType::Value(value)) => {+ collect_lambda_ref(value, params, lambda_refs)+ }+ _ => unreachable!(),+ }+ }+ }+ RexType::Lambda(lambda) => {+ if let Some(parameters) = &lambda.parameters {+ params.push(parameters.clone());+ }+ if let Some(body) = &lambda.body {+ collect_lambda_ref(body, params, lambda_refs);+ }+ }+ RexType::Literal(_literal) => {}+ _ => unreachable!(),+ }+ }+ }}diff --git a/datafusion/substrait/tests/testdata/test_plans/higher_order_function.json b/datafusion/substrait/tests/testdata/test_plans/higher_order_function.jsonnew file mode 100644index 0000000000..da613b2573--- /dev/null+++ b/datafusion/substrait/tests/testdata/test_plans/higher_order_function.json@@ -0,0 +1,438 @@+{+ "version": {+ "minorNumber": 85,+ "producer": "datafusion"+ },+ "extensions": [+ {+ "extensionFunction": {+ "extensionUrnReference": 2,+ "functionAnchor": 2,+ "name": "array_transform2"+ }+ },+ {+ "extensionFunction": {+ "extensionUrnReference": 2,+ "name": "make_array"+ }+ },+ {+ "extensionFunction": {+ "extensionUrnReference": 2,+ "functionAnchor": 3,+ "name": "array_concat"+ }+ },+ {+ "extensionFunction": {+ "extensionUrnReference": 1,+ "functionAnchor": 1,+ "name": "multiply"+ }+ }+ ],+ "relations": [+ {+ "root": {+ "input": {+ "project": {+ "common": {+ "emit": {+ "outputMapping": [+ 1+ ]+ }+ },+ "input": {+ "read": {+ "baseSchema": {+ "names": [+ "p1"+ ],+ "struct": {+ "types": [+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ }+ ],+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ "projection": {+ "select": {+ "structItems": [+ {}+ ]+ }+ },+ "namedTable": {+ "names": [+ "data3"+ ]+ }+ }+ },+ "expressions": [+ {+ "scalarFunction": {+ "functionReference": 2,+ "arguments": [+ {+ "value": {+ "scalarFunction": {+ "arguments": [+ {+ "value": {+ "scalarFunction": {+ "arguments": [+ {+ "value": {+ "selection": {+ "directReference": {+ "structField": {}+ },+ "rootReference": {}+ }+ }+ }+ ],+ "outputType": {+ "list": {+ "type": {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ "nullability": "NULLABILITY_NULLABLE"+ }+ }+ }+ }+ }+ ],+ "outputType": {+ "list": {+ "type": {+ "list": {+ "type": {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ "nullability": "NULLABILITY_NULLABLE"+ }+ }+ }+ }+ },+ {+ "value": {+ "lambda": {+ "parameters": {+ "types": [+ {+ "list": {+ "type": {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ }+ ],+ "nullability": "NULLABILITY_REQUIRED"+ },+ "body": {+ "scalarFunction": {+ "functionReference": 3,+ "arguments": [+ {+ "value": {+ "scalarFunction": {+ "functionReference": 2,+ "arguments": [+ {+ "value": {+ "selection": {+ "directReference": {+ "structField": {}+ },+ "lambdaParameterReference": {}+ }+ }+ },+ {+ "value": {+ "lambda": {+ "parameters": {+ "types": [+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ }+ ],+ "nullability": "NULLABILITY_REQUIRED"+ },+ "body": {+ "scalarFunction": {+ "functionReference": 1,+ "arguments": [+ {+ "value": {+ "scalarFunction": {+ "functionReference": 1,+ "arguments": [+ {+ "value": {+ "selection": {+ "directReference": {+ "structField": {}+ },+ "lambdaParameterReference": {}+ }+ }+ },+ {+ "value": {+ "selection": {+ "directReference": {+ "structField": {+ "field": 1+ }+ },+ "lambdaParameterReference": {+ "stepsOut": 1+ }+ }+ }+ }+ ],+ "outputType": {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ }+ }+ }+ },+ {+ "value": {+ "selection": {+ "directReference": {+ "structField": {+ "field": 1+ }+ },+ "lambdaParameterReference": {}+ }+ }+ }+ ],+ "outputType": {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ }+ }+ }+ }+ }+ }+ ],+ "outputType": {+ "list": {+ "type": {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ "nullability": "NULLABILITY_NULLABLE"+ }+ }+ }+ }+ },+ {+ "value": {+ "scalarFunction": {+ "functionReference": 2,+ "arguments": [+ {+ "value": {+ "selection": {+ "directReference": {+ "structField": {}+ },+ "lambdaParameterReference": {}+ }+ }+ },+ {+ "value": {+ "lambda": {+ "parameters": {+ "types": [+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ }+ ],+ "nullability": "NULLABILITY_REQUIRED"+ },+ "body": {+ "scalarFunction": {+ "functionReference": 1,+ "arguments": [+ {+ "value": {+ "scalarFunction": {+ "functionReference": 1,+ "arguments": [+ {+ "value": {+ "selection": {+ "directReference": {+ "structField": {}+ },+ "lambdaParameterReference": {}+ }+ }+ },+ {+ "value": {+ "selection": {+ "directReference": {+ "structField": {+ "field": 1+ }+ },+ "lambdaParameterReference": {+ "stepsOut": 1+ }+ }+ }+ }+ ],+ "outputType": {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ }+ }+ }+ },+ {+ "value": {+ "selection": {+ "directReference": {+ "structField": {+ "field": 1+ }+ },+ "lambdaParameterReference": {}+ }+ }+ }+ ],+ "outputType": {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ }+ }+ }+ }+ }+ }+ ],+ "outputType": {+ "list": {+ "type": {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ "nullability": "NULLABILITY_NULLABLE"+ }+ }+ }+ }+ }+ ],+ "outputType": {+ "list": {+ "type": {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ "nullability": "NULLABILITY_NULLABLE"+ }+ }+ }+ }+ }+ }+ }+ ],+ "outputType": {+ "list": {+ "type": {+ "list": {+ "type": {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ "nullability": "NULLABILITY_NULLABLE"+ }+ }+ }+ }+ ]+ }+ },+ "names": [+ "array_transform2(make_array(make_array(data3.p1)),(v, i) -> array_concat(array_transform2(v,(v, j) -> v * i * j),array_transform2(v,(v, j) -> v * i * j)))"+ ]+ }+ }+ ],+ "extensionUrns": [+ {+ "extensionUrnAnchor": 1,+ "urn": "extension:io.substrait:functions_arithmetic"+ },+ {+ "extensionUrnAnchor": 2,+ "urn": "extension:io.substrait:functions_list"+ }+ ]+}