Skip to content

Apache DataFusion SQL Query Engine

fix: keep struct field names on Substrait lambda parameters (#27086)

Lía Adriana238ae32697e3
diff --git a/datafusion/substrait/src/logical_plan/consumer/expr/scalar_function.rs b/datafusion/substrait/src/logical_plan/consumer/expr/scalar_function.rs
index 403e88b3e7..2be837015f 100644
--- a/datafusion/substrait/src/logical_plan/consumer/expr/scalar_function.rs
+++ b/datafusion/substrait/src/logical_plan/consumer/expr/scalar_function.rs
@@ -4,6 +4,7 @@
use crate::logical_plan::consumer::{SubstraitConsumer, from_substrait_func_args};
use datafusion::common::Result;
+use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion};
use datafusion::common::{
DFSchema, DataFusionError, ScalarValue, not_impl_err, plan_err, substrait_err,
};
@@ -56,10 +57,24 @@ pub async fn from_scalar_function(
// try to first match the requested function into registered higher-order functions, then udfs, built-in ops
// and finally built-in expressions
if let Ok(func) = higher_order_func {
- Ok(Expr::HigherOrderFunction(expr::HigherOrderFunction::new(
+ let expr = Expr::HigherOrderFunction(expr::HigherOrderFunction::new(
func.to_owned(),
args,
- )))
+ ));
+ // Substrait types carry no struct field names, so a lambda parameter over a list
+ // of structs is declared as `{c0, c1, ...}`, and a field access in the body such
+ // as `get_field(x, 'b')` would not find `b`. Give the parameters the fields the
+ // function derives from its arguments instead, which keep the list's field names.
+ //
+ // A function that uses a parameter of an outer lambda can't be resolved on its
+ // own, as only the outer function knows that parameter's type. In
+ // `array_transform(s, x -> array_transform([1, 2], i -> get_field(x, 'a') + i))`
+ // the inner call uses `x`, so it is resolved when the outer call is.
+ if uses_outer_lambda_parameters(&expr, &mut Vec::new())? {
+ Ok(expr)
+ } else {
+ Ok(expr.resolve_lambda_variables(input_schema)?.data)
+ }
} else if let Ok(func) = udf_func {
Ok(Expr::ScalarFunction(expr::ScalarFunction::new_udf(
func.to_owned(),
@@ -81,6 +96,47 @@ pub async fn from_scalar_function(
}
}
+/// Checks whether `expr` uses a lambda parameter that comes from outside of it.
+///
+/// In `array_transform(s, x -> array_transform([1, 2], i -> get_field(x, 'a') + i))`,
+/// the inner `array_transform([1, 2], i -> get_field(x, 'a') + i)` uses its own
+/// parameter `i`, but also `x`, which belongs to the outer lambda, so this returns true
+/// for it. For the whole expression it returns false: `x` and `i` both belong to
+/// lambdas inside it.
+///
+/// `own_params` collects the parameters of the lambdas we are inside of while walking
+/// `expr`.
+fn uses_outer_lambda_parameters(
+ expr: &Expr,
+ own_params: &mut Vec<String>,
+) -> Result<bool> {
+ match expr {
+ // a parameter that wasn't collected on the way down comes from outside `expr`
+ Expr::LambdaVariable(var) => Ok(!own_params.contains(&var.name)),
+ Expr::Lambda(lambda) => {
+ // a lambda's parameters can be used in its body, and only there
+ let before = own_params.len();
+ own_params.extend(lambda.params.iter().cloned());
+ let uses = uses_outer_lambda_parameters(&lambda.body, own_params);
+ own_params.truncate(before);
+ uses
+ }
+ // anything else: check its children, stopping at the first one that uses one
+ _ => {
+ let mut uses = false;
+ expr.apply_children(|child| {
+ uses = uses_outer_lambda_parameters(child, own_params)?;
+ Ok(if uses {
+ TreeNodeRecursion::Stop
+ } else {
+ TreeNodeRecursion::Continue
+ })
+ })?;
+ Ok(uses)
+ }
+ }
+}
+
pub fn substrait_fun_name(name: &str) -> &str {
(match name.rsplit_once(':') {
// Since 0.32.0, Substrait requires the function names to be in a compound format
diff --git a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs
index 4285b7a708..5ebf2364ab 100644
--- a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs
+++ b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs
@@ -2653,6 +2653,51 @@ async fn roundtrip_array_transform_higher_order_function() -> Result<()> {
Ok(())
}
+// Substrait lambda parameter types carry no struct field names, so a parameter over a
+// list of structs must take its field names from the list, not from its declared type.
+#[tokio::test]
+async fn roundtrip_lambda_parameter_struct_field_names() -> Result<()> {
+ let ctx = higher_order_function_ctx().await?;
+ ctx.sql(
+ "CREATE TABLE structs AS SELECT \
+ [named_struct('a', 1, 'b', 'x'), named_struct('a', 2, 'b', 'y')] AS s, \
+ [[named_struct('a', 3, 'b', 'z')]] AS nested",
+ )
+ .await?
+ .collect()
+ .await?;
+
+ // field of a parameter over a column
+ roundtrip_with_ctx(
+ "SELECT array_transform(s, p0 -> get_field(p0, 'b')) FROM structs",
+ ctx.clone(),
+ )
+ .await?;
+
+ // field of a parameter over a computed list
+ roundtrip_with_ctx(
+ "SELECT array_transform(array_concat(s, s), p0 -> get_field(p0, 'a')) FROM structs",
+ ctx.clone(),
+ )
+ .await?;
+
+ // field of the inner parameter, where the inner list is the outer parameter
+ roundtrip_with_ctx(
+ "SELECT array_transform(nested, p0 -> array_transform(p0, p1 -> get_field(p1, 'b'))) FROM structs",
+ ctx.clone(),
+ )
+ .await?;
+
+ // field of the outer parameter, read inside an inner lambda
+ roundtrip_with_ctx(
+ "SELECT array_transform(s, p0 -> array_transform([1, 2], p1 -> get_field(p0, 'a') + p1)) FROM structs",
+ ctx.clone(),
+ )
+ .await?;
+
+ Ok(())
+}
+
pub(crate) async fn higher_order_function_ctx() -> Result<SessionContext> {
let ctx = create_context_with_dialect(Some(Dialect::Databricks)).await?;