Skip to content

Apache DataFusion SQL Query Engine

Add resolve_lambda_variables helper to Expr and LogicalPlan (#22101)

gstvg7b24735a2154
diff --git a/datafusion/common/src/lib.rs b/datafusion/common/src/lib.rs
index 9be0941b5d..06e567cb12 100644
--- a/datafusion/common/src/lib.rs
+++ b/datafusion/common/src/lib.rs
@@ -115,6 +115,7 @@ pub type HashMap<K, V, S = DefaultHashBuilder> = hashbrown::HashMap<K, V, S>;
pub type HashSet<T, S = DefaultHashBuilder> = hashbrown::HashSet<T, S>;
pub mod hash_map {
pub use hashbrown::hash_map::Entry;
+ pub use hashbrown::hash_map::EntryRef;
}
pub mod hash_set {
pub use hashbrown::hash_set::Entry;
diff --git a/datafusion/expr/src/expr.rs b/datafusion/expr/src/expr.rs
index 36ef6cf1f5..6f420a7153 100644
--- a/datafusion/expr/src/expr.rs
+++ b/datafusion/expr/src/expr.rs
@@ -26,7 +26,7 @@ use std::sync::Arc;
use crate::expr_fn::binary_expr;
use crate::function::WindowFunctionSimplification;
-use crate::higher_order_function::HigherOrderUDF;
+use crate::higher_order_function::{HigherOrderUDF, resolve_lambda_variables};
use crate::logical_plan::Subquery;
use crate::type_coercion::functions::value_fields_with_higher_order_udf;
use crate::{AggregateUDF, LambdaParametersProgress, ValueOrLambda, Volatility};
@@ -453,7 +453,12 @@ impl HigherOrderFunction {
}
/// Invokes the inner function [`HigherOrderUDF::lambda_parameters`]
- /// using the arguments of this invocation
+ /// using the arguments of this invocation. This expression lambda
+ /// variables must be already resolved either by coming from the
+ /// default sql planner or by calling [Expr::resolve_lambda_variables]
+ /// or [LogicalPlan::resolve_lambda_variables]
+ ///
+ /// [LogicalPlan::resolve_lambda_variables]: crate::LogicalPlan::resolve_lambda_variables
pub fn lambda_parameters(
&self,
schema: &dyn ExprSchema,
@@ -496,9 +501,20 @@ impl PartialEq for HigherOrderFunction {
}
/// A named reference to a lambda parameter which includes it's own [`FieldRef`],
-/// which is used to implement [`ExprSchemable`], for example. Note the field must
-/// be set in order to create a physical lambda variable. A helper to automatically
-/// set them will be added in the future
+/// which is used to implement [`ExprSchemable`], for example. It is an option only to make
+/// easier for `expr_api` users to construct lambda variables, but any expression
+/// tree or [`LogicalPlan`] containing unresolved variables must be resolved before
+/// usage with either [`Expr::resolve_lambda_variables`] or
+/// [`LogicalPlan::resolve_lambda_variables`]. The default SQL planner produces
+/// already resolved variables and no further resolving is required.
+///
+/// After resolving, if any argument from the lambda function which this
+/// variables originates from have it's field changed (type, nullability,
+/// metadata, etc), the resolved variable may became outdated and must be
+/// resolved again.
+///
+/// [`LogicalPlan`]: crate::LogicalPlan
+/// [`LogicalPlan::resolve_lambda_variables`]: crate::LogicalPlan::resolve_lambda_variables
#[derive(Clone, PartialEq, PartialOrd, Eq, Debug, Hash)]
pub struct LambdaVariable {
pub name: String,
@@ -507,7 +523,12 @@ pub struct LambdaVariable {
}
impl LambdaVariable {
- /// Create a lambda variable from a name and a Field.
+ /// Create a lambda variable from a name and an optional field.
+ /// If the field is none, the expression tree or LogicalPlan which
+ /// owns this variable must be resolved before usage with either
+ /// [`Expr::resolve_lambda_variables`] or [`LogicalPlan::resolve_lambda_variables`].
+ ///
+ /// [`LogicalPlan::resolve_lambda_variables`]: crate::LogicalPlan::resolve_lambda_variables
pub fn new(name: String, field: Option<FieldRef>) -> Self {
Self {
name,
@@ -2229,6 +2250,16 @@ impl Expr {
None
}
}
+
+ /// Return a `Expr` with all [`LambdaVariable`] resolved only if all of them
+ /// are contained in the subtree of the [`HigherOrderFunction`] it originates from,
+ /// otherwise returns an error
+ pub fn resolve_lambda_variables(
+ self,
+ schema: &DFSchema,
+ ) -> Result<Transformed<Expr>> {
+ resolve_lambda_variables(self, schema, &mut HashMap::new())
+ }
}
impl Normalizeable for Expr {
diff --git a/datafusion/expr/src/expr_fn.rs b/datafusion/expr/src/expr_fn.rs
index cec6b7ec05..9d711113e4 100644
--- a/datafusion/expr/src/expr_fn.rs
+++ b/datafusion/expr/src/expr_fn.rs
@@ -728,10 +728,14 @@ pub fn lambda(params: impl IntoIterator<Item = impl Into<String>>, body: Expr) -
))
}
-/// Create a lambda variable expression
-// todo: make this pub when support for optional field lands
-#[expect(unused)]
-fn lambda_var(name: impl Into<String>) -> Expr {
+/// Create an unresolved lambda variable expression
+///
+/// The expression tree or [`LogicalPlan`] which
+/// owns this variable must be resolved before usage with either
+/// [`Expr::resolve_lambda_variables`] or [`LogicalPlan::resolve_lambda_variables`].
+///
+/// [LogicalPlan::resolve_lambda_variables]: crate::LogicalPlan::resolve_lambda_variables
+pub fn lambda_var(name: impl Into<String>) -> Expr {
Expr::LambdaVariable(LambdaVariable::new(name.into(), None))
}
diff --git a/datafusion/expr/src/higher_order_function.rs b/datafusion/expr/src/higher_order_function.rs
index 5fac02846a..3dc143b8e5 100644
--- a/datafusion/expr/src/higher_order_function.rs
+++ b/datafusion/expr/src/higher_order_function.rs
@@ -17,13 +17,25 @@
//! [`HigherOrderUDF`]: User Defined Higher Order Functions
-use crate::expr::schema_name_from_exprs_comma_separated_without_space;
-use crate::{ColumnarValue, Documentation, Expr};
+use crate::expr::{
+ HigherOrderFunction, display_comma_separated,
+ schema_name_from_exprs_comma_separated_without_space,
+};
+use crate::type_coercion::functions::value_fields_with_higher_order_udf;
+use crate::{ColumnarValue, Documentation, Expr, ExprSchemable};
use arrow::array::{ArrayRef, RecordBatch};
use arrow::datatypes::{DataType, FieldRef, Schema};
use arrow_schema::SchemaRef;
use datafusion_common::config::ConfigOptions;
-use datafusion_common::{Result, ScalarValue, exec_err, not_impl_err};
+use datafusion_common::datatype::FieldExt;
+use datafusion_common::hash_map::EntryRef;
+use datafusion_common::tree_node::{
+ Transformed, TreeNode, TreeNodeContainer, TreeNodeRecursion,
+};
+use datafusion_common::{
+ DFSchema, HashMap, HashSet, Result, ScalarValue, exec_err, internal_datafusion_err,
+ internal_err, not_impl_err, plan_datafusion_err, plan_err,
+};
use datafusion_expr_common::dyn_eq::{DynEq, DynHash};
use datafusion_expr_common::signature::Volatility;
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
@@ -31,6 +43,7 @@ use std::any::Any;
use std::cmp::Ordering;
use std::fmt::Debug;
use std::hash::{Hash, Hasher};
+use std::mem;
use std::sync::Arc;
/// The types of arguments for which a function has implementations.
@@ -794,12 +807,279 @@ pub trait HigherOrderUDF: Debug + DynEq + DynHash + Send + Sync + Any {
}
}
+pub(crate) fn resolve_lambda_variables(
+ expr: Expr,
+ schema: &DFSchema,
+ // a map of lambda variable name => a never empty stack of fields [ [..shadowed], in_scope ]
+ vars: &mut HashMap<String, Vec<FieldRef>>,
+) -> Result<Transformed<Expr>> {
+ expr.transform_down(|expr| match expr {
+ Expr::HigherOrderFunction(HigherOrderFunction { func, args }) => {
+ // not inlined to reduce nesting
+ resolve_higher_order_function(func, args, schema, vars)
+ }
+ Expr::LambdaVariable(mut var) => {
+ let field_stack = vars.get(&var.name).ok_or_else(|| {
+ plan_datafusion_err!(
+ "missing field of lambda variable {} while resolving",
+ var.name
+ )
+ })?;
+
+ let field = field_stack.last().ok_or_else(|| {
+ internal_datafusion_err!("every entry should have at least one field")
+ })?;
+
+ let field = Arc::clone(field).renamed(&var.name);
+
+ let transformed = var.field.as_ref().is_none_or(|old| old != &field);
+
+ var.field = Some(field);
+
+ Ok(Transformed::new_transformed(
+ Expr::LambdaVariable(var),
+ transformed,
+ ))
+ }
+ _ => Ok(Transformed::no(expr)),
+ })
+}
+
+fn resolve_higher_order_function(
+ func: Arc<dyn HigherOrderUDF>,
+ args: Vec<Expr>,
+ schema: &DFSchema,
+ // a map of lambda variable name => a never empty stack of fields [ [..shadowed], in_scope ]
+ vars: &mut HashMap<String, Vec<FieldRef>>,
+) -> Result<Transformed<Expr>> {
+ let args = if !vars.is_empty() {
+ /* if this is a nested lambda, we must resolve non-lambda args before invoking
+ lambda_parameters because it will invoke ExprSchemable::to_field for every
+ non-lambda parameter, and if one them contains a lambda variable, it will fail
+ due to it being unresolved. Example query:
+
+ array_transform([[1, 2]], a -> array_transform(a, b -> b+1))
+
+ the nested array_transform's lambda_parameters will call Lambdavariable::to_field
+ on it's first argument, the variable `a`, which must be resolved
+ */
+ args.map_elements(|arg| match arg {
+ Expr::Lambda(_) => Ok(Transformed::no(arg)),
+ _ => resolve_lambda_variables(arg, schema, vars),
+ })?
+ } else {
+ Transformed::no(args)
+ };
+
+ let transformed = args.transformed;
+ let mut args = args.data;
+
+ let current_fields = args
+ .iter()
+ .map(|e| match e {
+ Expr::Lambda(_lambda_function) => Ok(ValueOrLambda::Lambda(None)),
+ _ => Ok(ValueOrLambda::Value(e.to_field(schema)?.1)),
+ })
+ .collect::<Result<Vec<_>>>()?;
+
+ // coerce fields because coercion may alter the lambda parameters
+ let mut fields = value_fields_with_higher_order_udf(&current_fields, func.as_ref())?;
+
+ let num_lambdas = args.iter().filter(|a| matches!(a, Expr::Lambda(_))).count();
+
+ let mut step = 0;
+
+ let lambda_params = loop {
+ match func.lambda_parameters(step, &fields)? {
+ LambdaParametersProgress::Partial(params) => {
+ let mut params = params.into_iter();
+
+ if params.len() != num_lambdas {
+ return plan_err!(
+ "{} lambda_parameters returned {} lambdas but {num_lambdas} expected",
+ func.name(),
+ params.len()
+ );
+ }
+
+ for (arg, field) in std::iter::zip(&mut args, &mut fields) {
+ match (arg, field) {
+ (Expr::Lambda(lambda), ValueOrLambda::Lambda(field)) => {
+ let params = params.next().ok_or_else(|| {
+ internal_datafusion_err!(
+ "params len should have been checked above"
+ )
+ })?;
+
+ if let Some(params) = params {
+ for (name, field) in
+ std::iter::zip(&lambda.params, params)
+ {
+ vars.entry_ref(name)
+ .or_default()
+ .push(field.renamed(name.as_str()));
+ }
+
+ let body_with_vars = resolve_lambda_variables(
+ mem::take(lambda.body.as_mut()),
+ schema,
+ vars,
+ )?;
+
+ remove_scope(vars, &lambda.params)?;
+
+ *field = Some(body_with_vars.data.to_field(schema)?.1);
+ *lambda.body = body_with_vars.data;
+ }
+ }
+ (_, ValueOrLambda::Lambda(_)) => {
+ return internal_err!(
+ "value_fields_with_higher_order_udf returned a value for a lambda argument"
+ );
+ }
+ (Expr::Lambda(_), ValueOrLambda::Value(_)) => {
+ return internal_err!(
+ "value_fields_with_higher_order_udf returned a lambda for a value argument"
+ );
+ }
+ (_, ValueOrLambda::Value(_)) => {} // nothing to do
+ }
+ }
+ }
+ LambdaParametersProgress::Complete(params) => break params,
+ }
+
+ let limit = func.signature().lambda_parameters_max_iterations;
+
+ step += 1;
+
+ if step > limit {
+ return plan_err!(
+ "{} lambda_parameters called {limit} times without completion",
+ func.name()
+ );
+ }
+ };
+
+ let mut lambda_params = lambda_params.into_iter();
+
+ if num_lambdas != lambda_params.len() {
+ return plan_err!(
+ "{} lambda_parameters returned {} values for {num_lambdas} lambdas",
+ func.name(),
+ lambda_params.len()
+ );
+ }
+
+ let args = args.map_elements(|arg| match arg {
+ Expr::Lambda(mut lambda) => {
+ let lambda_params = lambda_params.next().ok_or_else(|| {
+ internal_datafusion_err!(
+ "lambda_params len should have been checked above"
+ )
+ })?;
+
+ if lambda.params.len() > lambda_params.len() {
+ return plan_err!(
+ "{} lambda defined {} params ({}), but only {} supported",
+ func.name(),
+ lambda.params.len(),
+ display_comma_separated(&lambda.params),
+ lambda_params.len()
+ );
+ }
+
+ if !all_unique(&lambda.params) {
+ return plan_err!(
+ "lambda params must be unique, got ({})",
+ lambda.params.join(", ")
+ );
+ }
+
+ for (param, field) in std::iter::zip(&lambda.params, lambda_params) {
+ vars.entry_ref(param)
+ .or_default()
+ .push(field.renamed(param.as_str()));
+ }
+
+ let transformed =
+ resolve_lambda_variables(mem::take(lambda.body.as_mut()), schema, vars)?;
+
+ *lambda.body = transformed.data;
+
+ remove_scope(vars, &lambda.params)?;
+
+ Ok(Transformed::new(
+ Expr::Lambda(lambda),
+ transformed.transformed,
+ TreeNodeRecursion::Jump,
+ ))
+ }
+ arg => Ok(Transformed::no(arg)), // resolved at the start of the function
+ })?;
+
+ Ok(Transformed::new(
+ Expr::HigherOrderFunction(HigherOrderFunction::new(func, args.data)),
+ transformed || args.transformed,
+ TreeNodeRecursion::Jump,
+ ))
+}
+
+fn remove_scope(
+ vars: &mut HashMap<String, Vec<FieldRef>>,
+ scope: &[String],
+) -> Result<()> {
+ for param in scope {
+ match vars.entry_ref(param) {
+ EntryRef::Occupied(mut v) => {
+ if v.get().len() == 1 {
+ v.remove();
+ } else {
+ v.get_mut().pop().ok_or_else(|| {
+ internal_datafusion_err!(
+ "every entry should have at least one field"
+ )
+ })?;
+ }
+ }
+ EntryRef::Vacant(_v) => {
+ return internal_err!("no empty value should be in the map");
+ }
+ }
+ }
+
+ Ok(())
+}
+
+fn all_unique(params: &[String]) -> bool {
+ match params.len() {
+ 0 | 1 => true,
+ 2 => params[0] != params[1],
+ _ => {
+ let mut set = HashSet::with_capacity(params.len());
+
+ params.iter().all(|p| set.insert(p.as_str()))
+ }
+ }
+}
+
#[cfg(test)]
mod tests {
- use datafusion_expr_common::signature::Volatility;
-
use super::*;
use std::hash::DefaultHasher;
+ use std::sync::Arc;
+
+ use arrow_schema::{DataType, Field, FieldRef, Schema};
+ use datafusion_common::{DFSchema, Result};
+ use datafusion_expr_common::columnar_value::ColumnarValue;
+ use datafusion_expr_common::signature::Volatility;
+
+ use crate::{
+ Expr, HigherOrderSignature, HigherOrderUDF, LambdaParametersProgress,
+ ValueOrLambda, col,
+ expr::{HigherOrderFunction, LambdaVariable},
+ lambda, lambda_var, lit,
+ };
#[derive(Debug, PartialEq, Eq, Hash)]
struct TestHigherOrderUDF {
@@ -883,4 +1163,220 @@ mod tests {
value.hash(hasher);
hasher.finish()
}
+
+ #[derive(Debug, PartialEq, Eq, Hash)]
+ struct MockArrayReduce {
+ signature: HigherOrderSignature,
+ }
+
+ impl HigherOrderUDF for MockArrayReduce {
+ fn name(&self) -> &str {
+ "array_reduce"
+ }
+
+ fn aliases(&self) -> &[String] {
+ &[]
+ }
+
+ fn signature(&self) -> &HigherOrderSignature {
+ &self.signature
+ }
+
+ fn lambda_parameters(
+ &self,
+ step: usize,
+ fields: &[ValueOrLambda<FieldRef, Option<FieldRef>>],
+ ) -> Result<LambdaParametersProgress> {
+ // optional finish not supported for simplicity
+ let [
+ ValueOrLambda::Value(list),
+ ValueOrLambda::Value(initial_value),
+ ValueOrLambda::Lambda(merge),
+ ValueOrLambda::Lambda(_finish),
+ ] = fields
+ else {
+ unreachable!()
+ };
+
+ let list_field = match list.data_type() {
+ DataType::List(field) => field,
+ _ => unreachable!(),
+ };
+
+ Ok(match (step, merge) {
+ (0, None) => {
+ // at the first step, we use the initial_value as merge accumulator,
+ // and return None for finish since we don't know the output of merge
+ LambdaParametersProgress::Partial(vec![
+ // merge
+ Some(vec![Arc::clone(initial_value), Arc::clone(list_field)]),
+ // finish
+ None,
+ ])
+ }
+ (1, Some(accumulator)) | (0, Some(accumulator)) => {
+ // now we can use the merge output as it's accumulator and
+ // as the finish parameter
+ LambdaParametersProgress::Complete(vec![
+ // merge
+ vec![Arc::clone(accumulator), Arc::clone(list_field)],
+ // finish
+ vec![Arc::clone(accumulator)],
+ ])
+ }
+ (1, None) => {
+ unreachable!()
+ }
+ _ => unreachable!(),
+ })
+ }
+
+ fn return_field_from_args(
+ &self,
+ args: HigherOrderReturnFieldArgs,
+ ) -> Result<FieldRef> {
+ // optional finish not supported for simplicity
+ let [
+ ValueOrLambda::Value(_list),
+ ValueOrLambda::Value(_initial_value),
+ ValueOrLambda::Lambda(_merge),
+ ValueOrLambda::Lambda(finish),
+ ] = args.arg_fields
+ else {
+ unreachable!()
+ };
+
+ Ok(Arc::clone(finish))
+ }
+
+ fn invoke_with_args(
+ &self,
+ _args: HigherOrderFunctionArgs,
+ ) -> Result<ColumnarValue> {
+ unreachable!()
+ }
+ }
+
+ #[test]
+ fn test_resolve_lambda_variables() {
+ let schema = DFSchema::try_from(Schema::new(vec![Field::new(
+ "c",
+ DataType::new_list(DataType::new_list(DataType::Int32, true), true),
+ true,
+ )]))
+ .unwrap();
+
+ let func = Arc::new(MockArrayReduce {
+ signature: HigherOrderSignature::variadic_any(Volatility::Immutable),
+ }) as _;
+
+ /*
+ array_reduce(
+ c,
+ 0,
+ (acc1, v) -> acc + array_reduce(
+ v,
+ 0,
+ (acc2, v) -> acc2 + acc1 + v,
+ reduced -> reduced * 2.0
+ ),
+ reduced -> reduced * 2
+ )
+ */
+ let expr = Expr::HigherOrderFunction(HigherOrderFunction::new(
+ Arc::clone(&func),
+ vec![
+ col("c"),
+ lit(0),
+ lambda(
+ ["acc1", "v"],
+ lambda_var("acc1")
+ + Expr::HigherOrderFunction(HigherOrderFunction::new(
+ Arc::clone(&func),
+ vec![
+ lambda_var("v"),
+ lit(0),
+ lambda(
+ ["acc2", "v"],
+ lambda_var("acc2")
+ + lambda_var("acc1")
+ + lambda_var("v"),
+ ),
+ lambda(["reduced"], lambda_var("reduced") * lit(2.0)),
+ ],
+ )),
+ ),
+ lambda(["reduced"], lambda_var("reduced") * lit(2)),
+ ],
+ ));
+
+ let resolved_expr = expr.resolve_lambda_variables(&schema).unwrap().data;
+
+ /*
+ array_reduce(
+ c@[[Int32]],
+ 0@Int64,
+ (acc1@Float64, v@[Int32]) -> acc@Float64 + array_reduce(
+ v@[Int32],
+ 0@Int64,
+ (acc2@Float64, v@Int32) -> acc2@Float64 + acc1@Float64 + v@Int32,
+ reducedFloat64 -> reduced@Float64 * 2.0@Float64
+ ),
+ reduced@Float64 -> reduced@Float64 * 2@Int64
+ )
+ */
+ let expected = Expr::HigherOrderFunction(HigherOrderFunction::new(
+ Arc::clone(&func),
+ vec![
+ col("c"),
+ lit(0),
+ lambda(
+ ["acc1", "v"],
+ resolved_lambda_var("acc1", DataType::Float64, true)
+ + Expr::HigherOrderFunction(HigherOrderFunction::new(
+ Arc::clone(&func),
+ vec![
+ resolved_lambda_var(
+ "v",
+ DataType::new_list(DataType::Int32, true),
+ true,
+ ),
+ lit(0),
+ lambda(
+ ["acc2", "v"],
+ resolved_lambda_var("acc2", DataType::Float64, true)
+ + resolved_lambda_var(
+ "acc1",
+ DataType::Float64,
+ true,
+ )
+ + resolved_lambda_var("v", DataType::Int32, true),
+ ),
+ lambda(
+ ["reduced"],
+ resolved_lambda_var(
+ "reduced",
+ DataType::Float64,
+ true,
+ ) * lit(2.0),
+ ),
+ ],
+ )),
+ ),
+ lambda(
+ ["reduced"],
+ resolved_lambda_var("reduced", DataType::Float64, true) * lit(2),
+ ),
+ ],
+ ));
+
+ assert_eq!(resolved_expr, expected);
+ }
+
+ fn resolved_lambda_var(name: &str, dt: DataType, nullable: bool) -> Expr {
+ Expr::LambdaVariable(LambdaVariable::new(
+ name.into(),
+ Some(Arc::new(Field::new(name, dt, nullable))),
+ ))
+ }
}
diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs
index db8b82fe87..c572b202f0 100644
--- a/datafusion/expr/src/logical_plan/plan.rs
+++ b/datafusion/expr/src/logical_plan/plan.rs
@@ -42,7 +42,7 @@ use crate::logical_plan::extension::UserDefinedLogicalNode;
use crate::logical_plan::{DmlStatement, Statement};
use crate::utils::{
enumerate_grouping_sets, exprlist_to_fields, find_out_reference_exprs,
- grouping_set_expr_count, grouping_set_to_exprlist, split_conjunction,
+ grouping_set_expr_count, grouping_set_to_exprlist, merge_schema, split_conjunction,
};
use crate::{
BinaryExpr, CreateMemoryTable, CreateView, Execute, Expr, ExprSchemable, GroupingSet,
@@ -2189,6 +2189,17 @@ impl LogicalPlan {
}
Wrapper(self)
}
+
+ /// Return a `LogicalPLan` with all [`LambdaVariable`]'s resolved
+ ///
+ /// [`LambdaVariable`]: crate::expr::LambdaVariable
+ pub fn resolve_lambda_variables(self) -> Result<Transformed<LogicalPlan>> {
+ self.transform_with_subqueries(|plan| {
+ let schema = merge_schema(&plan.inputs());
+
+ plan.map_expressions(|expr| expr.resolve_lambda_variables(&schema))
+ })
+ }
}
impl Display for LogicalPlan {
diff --git a/datafusion/functions-nested/src/array_transform.rs b/datafusion/functions-nested/src/array_transform.rs
index 2542d4ab1f..aa8e2c5b46 100644
--- a/datafusion/functions-nested/src/array_transform.rs
+++ b/datafusion/functions-nested/src/array_transform.rs
@@ -292,10 +292,8 @@ mod tests {
};
use datafusion_common::{DFSchema, Result};
use datafusion_expr::{
- Expr, col,
- execution_props::ExecutionProps,
- expr::{HigherOrderFunction, LambdaVariable},
- lambda, lit,
+ Expr, col, execution_props::ExecutionProps, expr::HigherOrderFunction, lambda,
+ lambda_var, lit,
};
use datafusion_physical_expr::create_physical_expr;
@@ -327,18 +325,10 @@ mod tests {
create_physical_expr(
&Expr::HigherOrderFunction(HigherOrderFunction::new(
array_transform,
- vec![
- col("list"),
- lambda(
- ["v"],
- lit(100i32)
- / Expr::LambdaVariable(LambdaVariable::new(
- "v".to_string(),
- Some(Arc::new(Field::new("v", DataType::Int32, true))),
- )),
- ),
- ],
- )),
+ vec![col("list"), lambda(["v"], lit(100i32) / lambda_var("v"))],
+ ))
+ .resolve_lambda_variables(&schema)?
+ .data,
&schema,
&ExecutionProps::new(),
)?
diff --git a/datafusion/sql/src/unparser/expr.rs b/datafusion/sql/src/unparser/expr.rs
index dde383ea26..d7b1c6a3bb 100644
--- a/datafusion/sql/src/unparser/expr.rs
+++ b/datafusion/sql/src/unparser/expr.rs
@@ -1906,13 +1906,13 @@ mod tests {
use ast::ObjectName;
use datafusion_common::datatype::DataTypeExt;
use datafusion_common::{Spans, TableReference};
- use datafusion_expr::expr::{LambdaVariable, WildcardOptions};
+ use datafusion_expr::expr::WildcardOptions;
use datafusion_expr::{
ColumnarValue, HigherOrderUDF, LambdaParametersProgress, ScalarFunctionArgs,
ScalarUDF, ScalarUDFImpl, Signature, ValueOrLambda, Volatility, WindowFrame,
WindowFunctionDefinition, case, cast, col, cube, exists, grouping_set,
- interval_datetime_lit, interval_year_month_lit, lambda, lit, not, not_exists,
- out_ref_col, placeholder, rollup, table_scan, try_cast, when,
+ interval_datetime_lit, interval_year_month_lit, lambda, lambda_var, lit, not,
+ not_exists, out_ref_col, placeholder, rollup, table_scan, try_cast, when,
};
use datafusion_expr::{ExprFunctionExt, interval_month_day_nano_lit};
use datafusion_functions::datetime::from_unixtime::FromUnixtimeFunc;
@@ -2088,16 +2088,7 @@ mod tests {
(
Expr::HigherOrderFunction(HigherOrderFunction::new(
Arc::new(DummyHigherOrderUDF),
- vec![
- col("a"),
- lambda(
- ["v"],
- -Expr::LambdaVariable(LambdaVariable::new(
- "v".to_string(),
- Some(Arc::new(Field::new("", DataType::Null, true))),
- )),
- ),
- ],
+ vec![col("a"), lambda(["v"], -lambda_var("v"))],
)),
r#"dummy_higher_order_function(a, (v) -> -v)"#,
),