Skip to content

Apache DataFusion SQL Query Engine

fix(lambda): only push referenced params into the merged batch (#24162)

Lía Adriana4e6acfe8ee7e
diff --git a/datafusion/expr/src/higher_order_function.rs b/datafusion/expr/src/higher_order_function.rs
index 413714f498..c300be8f6f 100644
--- a/datafusion/expr/src/higher_order_function.rs
+++ b/datafusion/expr/src/higher_order_function.rs
@@ -24,7 +24,7 @@ use crate::expr::{
use crate::type_coercion::functions::value_fields_with_higher_order_udf;
use crate::udf_eq::UdfEq;
use crate::{ColumnarValue, Documentation, Expr, ExprSchemable};
-use arrow::array::{ArrayRef, RecordBatch};
+use arrow::array::{ArrayRef, RecordBatch, RecordBatchOptions};
use arrow::datatypes::{DataType, FieldRef, Schema};
use arrow_schema::SchemaRef;
use datafusion_common::config::ConfigOptions;
@@ -239,6 +239,26 @@ pub struct LambdaArgument {
/// For example, for `array_transform([2], v -> -v)`,
/// this will be `vec![Field::new("v", DataType::Int32, true)]`
params: Vec<FieldRef>,
+ /// Indices into [`Self::params`] of the parameters that are actually
+ /// referenced by [`Self::body`] (taking nested-lambda shadowing into
+ /// account), in the original declaration order of `params`.
+ ///
+ /// [`Self::evaluate`] only evaluates and pushes the closures whose
+ /// corresponding parameter index appears here, so unused declared
+ /// parameters leave no slot in the merged batch and the body's compressed
+ /// column indices line up directly with what the evaluator built.
+ ///
+ /// Callers who already have a `LambdaExpr` should pass
+ /// `LambdaExpr::used_param_indices()` directly to [`Self::new`] — both
+ /// are indices into the same positionally-aligned `params` list.
+ ///
+ /// Every index here must be `< params.len()`; see the precondition on
+ /// [`Self::new`].
+ ///
+ /// Relies on captures sorting before this lambda's own params in the
+ /// planner's (un-projected) index space, which is what makes
+ /// `captures ++ used_params` below line up with the projected body.
+ used_param_indices: Vec<usize>,
/// The body of the lambda
///
/// For example, for `array_transform([2], v -> -v)`,
@@ -257,26 +277,45 @@ pub struct LambdaArgument {
}
impl LambdaArgument {
+ /// # Preconditions
+ ///
+ /// Every index in `used_param_indices` must be `< params.len()`;
+ /// violating this panics on out-of-bounds indexing below. Callers should
+ /// pass `LambdaExpr::used_param_indices()`, which always indexes into the
+ /// same `params` list, rather than constructing indices by hand.
pub fn new(
params: Vec<FieldRef>,
body: Arc<dyn PhysicalExpr>,
captures: Option<RecordBatch>,
+ used_param_indices: &[usize],
) -> Self {
- let fields = match &captures {
+ debug_assert!(
+ used_param_indices.iter().all(|i| *i < params.len()),
+ "used_param_indices contains an index out of bounds for params \
+ (len {}): {:?}",
+ params.len(),
+ used_param_indices
+ );
+
+ let used_param_indices = used_param_indices.to_vec();
+ let effective_params = used_param_indices.iter().map(|i| Arc::clone(&params[*i]));
+
+ let fields: Vec<FieldRef> = match &captures {
Some(batch) => batch
.schema_ref()
.fields()
.iter()
.cloned()
- .chain(params.clone())
+ .chain(effective_params)
.collect(),
- None => params.clone(),
+ None => effective_params.collect(),
};
let schema = Arc::new(Schema::new(fields));
Self {
params,
+ used_param_indices,
body,
schema,
captures,
@@ -287,6 +326,11 @@ impl LambdaArgument {
/// `args` should evaluate to the value of each parameter
/// of the correspondent lambda returned in [HigherOrderUDFImpl::lambda_parameters].
///
+ /// Only the closures in `args` for parameters the lambda body actually
+ /// references are called; closures for declared-but-unused parameters
+ /// are skipped entirely. Callers should not rely on every closure in
+ /// `args` being invoked.
+ ///
/// `spread_captures` is responsible for transforming the captured column arrays
/// so they align with the evaluation batch. Captures are snapshotted from the
/// outer batch at construction time, giving one value per outer row, but the
@@ -344,6 +388,7 @@ impl LambdaArgument {
spread_captures.as_ref(),
Arc::clone(&self.schema),
&self.params,
+ &self.used_param_indices,
args,
)?;
@@ -355,6 +400,7 @@ fn merge_captures_with_variables(
captures: Option<&RecordBatch>,
schema: SchemaRef,
params: &[FieldRef],
+ used_param_indices: &[usize],
variables: &[&dyn Fn() -> Result<ArrayRef>],
) -> Result<RecordBatch> {
if variables.len() < params.len() {
@@ -365,23 +411,42 @@ fn merge_captures_with_variables(
);
}
+ let push_param_arrays = |columns: &mut Vec<ArrayRef>| -> Result<()> {
+ for &i in used_param_indices {
+ columns.push(variables[i]()?);
+ }
+ Ok(())
+ };
+
let columns = match captures {
Some(captures) => {
let mut columns = captures.columns().to_vec();
-
- for arg in &variables[..params.len()] {
- columns.push(arg()?);
- }
-
+ push_param_arrays(&mut columns)?;
+ columns
+ }
+ None => {
+ let mut columns = Vec::with_capacity(used_param_indices.len());
+ push_param_arrays(&mut columns)?;
columns
}
- None => variables
- .iter()
- .take(params.len())
- .map(|arg| arg())
- .collect::<Result<_>>()?,
};
+ if columns.is_empty() {
+ // No columns to derive a row count from, so borrow one variable's
+ // array length instead (all variables have the same length).
+ let row_count = variables.first().ok_or_else(|| {
+ internal_datafusion_err!(
+ "merge_captures_with_variables: no variables to derive a row count from"
+ )
+ })?()?
+ .len();
+ return Ok(RecordBatch::try_new_with_options(
+ schema,
+ vec![],
+ &RecordBatchOptions::new().with_row_count(Some(row_count)),
+ )?);
+ }
+
Ok(RecordBatch::try_new(schema, columns)?)
}
@@ -1681,4 +1746,109 @@ mod tests {
Some(Arc::new(Field::new(name, dt, nullable))),
))
}
+
+ /// A physical expression that reads the column at a fixed index of the
+ /// batch it is evaluated against, for exercising [`LambdaArgument`]
+ /// directly without depending on `datafusion-physical-expr`.
+ #[derive(Debug, Eq, PartialEq, Hash)]
+ struct ColumnAt(usize);
+
+ impl std::fmt::Display for ColumnAt {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(f, "column_at({})", self.0)
+ }
+ }
+
+ impl PhysicalExpr for ColumnAt {
+ fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> {
+ Ok(ColumnarValue::Array(Arc::clone(batch.column(self.0))))
+ }
+
+ fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {
+ vec![]
+ }
+
+ fn with_new_children(
+ self: Arc<Self>,
+ _children: Vec<Arc<dyn PhysicalExpr>>,
+ ) -> Result<Arc<dyn PhysicalExpr>> {
+ Ok(self)
+ }
+
+ fn fmt_sql(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(f, "{self}")
+ }
+ }
+
+ /// `(k, v) -> v` with only `v` used must push `v`'s array, not `k`'s.
+ #[test]
+ fn test_lambda_argument_evaluate_pushes_only_used_param() {
+ use arrow::array::Int32Array;
+
+ let k_field = Arc::new(Field::new("k", DataType::Int32, true));
+ let v_field = Arc::new(Field::new("v", DataType::Int32, true));
+
+ let body = Arc::new(ColumnAt(0)) as Arc<dyn PhysicalExpr>;
+ let lambda_arg = LambdaArgument::new(vec![k_field, v_field], body, None, &[1]);
+
+ let k_values: ArrayRef = Arc::new(Int32Array::from(vec![100, 200, 300]));
+ let v_values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));
+ let k_closure = || -> Result<ArrayRef> { Ok(Arc::clone(&k_values)) };
+ let v_closure = || -> Result<ArrayRef> { Ok(Arc::clone(&v_values)) };
+ let args: Vec<&dyn Fn() -> Result<ArrayRef>> = vec![&k_closure, &v_closure];
+
+ let result = lambda_arg
+ .evaluate(&args, |arrays| Ok(arrays.to_vec()))
+ .unwrap();
+ let ColumnarValue::Array(result) = result else {
+ unreachable!()
+ };
+
+ assert_eq!(
+ result.as_any().downcast_ref::<Int32Array>().unwrap(),
+ &Int32Array::from(vec![1, 2, 3]),
+ "body should read v's values, not k's"
+ );
+ }
+
+ /// Same as above, but with a capture occupying the leading slot.
+ #[test]
+ fn test_lambda_argument_evaluate_pushes_only_used_param_with_captures() {
+ use arrow::array::Int32Array;
+
+ let cap_field = Arc::new(Field::new("cap", DataType::Int32, true));
+ let k_field = Arc::new(Field::new("k", DataType::Int32, true));
+ let v_field = Arc::new(Field::new("v", DataType::Int32, true));
+
+ let body = Arc::new(ColumnAt(1)) as Arc<dyn PhysicalExpr>;
+
+ let cap_values: ArrayRef = Arc::new(Int32Array::from(vec![9, 9, 9]));
+ let captures = RecordBatch::try_new(
+ Arc::new(Schema::new(vec![cap_field])),
+ vec![cap_values],
+ )
+ .unwrap();
+
+ let lambda_arg =
+ LambdaArgument::new(vec![k_field, v_field], body, Some(captures), &[1]);
+
+ let k_values: ArrayRef = Arc::new(Int32Array::from(vec![100, 200, 300]));
+ let v_values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));
+ let k_closure = || -> Result<ArrayRef> { Ok(Arc::clone(&k_values)) };
+ let v_closure = || -> Result<ArrayRef> { Ok(Arc::clone(&v_values)) };
+ let args: Vec<&dyn Fn() -> Result<ArrayRef>> = vec![&k_closure, &v_closure];
+
+ let result = lambda_arg
+ .evaluate(&args, |arrays| Ok(arrays.to_vec()))
+ .unwrap();
+ let ColumnarValue::Array(result) = result else {
+ unreachable!()
+ };
+
+ assert_eq!(
+ result.as_any().downcast_ref::<Int32Array>().unwrap(),
+ &Int32Array::from(vec![1, 2, 3]),
+ "body should read v's values, not k's or the capture's"
+ );
+ }
}
diff --git a/datafusion/physical-expr/src/expressions/lambda.rs b/datafusion/physical-expr/src/expressions/lambda.rs
index cab2eea64f..95bb5db0b3 100644
--- a/datafusion/physical-expr/src/expressions/lambda.rs
+++ b/datafusion/physical-expr/src/expressions/lambda.rs
@@ -31,7 +31,7 @@ use arrow::{
};
use datafusion_common::{
HashMap, plan_err,
- tree_node::{Transformed, TreeNode, TreeNodeRecursion},
+ tree_node::{Transformed, TreeNode, TreeNodeRecursion, TreeNodeVisitor},
};
use datafusion_common::{HashSet, Result, internal_err};
use datafusion_expr::ColumnarValue;
@@ -43,6 +43,7 @@ pub struct LambdaExpr {
body: Arc<dyn PhysicalExpr>,
projected_body: Arc<dyn PhysicalExpr>,
projection: Vec<usize>,
+ used_param_indices: Vec<usize>,
}
// Manually derive PartialEq and Hash to work around https://github.com/rust-lang/rust/issues/78808 [https://github.com/apache/datafusion/issues/13196]
@@ -60,7 +61,7 @@ impl Hash for LambdaExpr {
}
impl LambdaExpr {
- /// Create a new lambda expression with the given parameters and body
+ /// Create a new lambda expression with the given parameters and body.
pub fn try_new(params: Vec<String>, body: Arc<dyn PhysicalExpr>) -> Result<Self> {
if !all_unique(&params) {
return plan_err!(
@@ -75,27 +76,30 @@ impl LambdaExpr {
}
fn new(params: Vec<String>, body: Arc<dyn PhysicalExpr>) -> Self {
- let mut used_column_indices = HashSet::new();
+ let own_params: HashSet<String> = params.iter().cloned().collect();
- body.apply(|node| {
- if let Some(col) = node.downcast_ref::<Column>() {
- used_column_indices.insert(col.index());
- } else if let Some(var) = node.downcast_ref::<LambdaVariable>() {
- used_column_indices.insert(var.index());
- }
-
- Ok(TreeNodeRecursion::Continue)
- })
- .expect("closure should be infallible");
+ let mut visitor = CollectUsedVisitor {
+ own_params: &own_params,
+ used_indices: HashSet::new(),
+ used_param_names: HashSet::new(),
+ shadow_stack: Vec::new(),
+ };
+ body.visit(&mut visitor).expect("visitor is infallible");
+ let CollectUsedVisitor {
+ used_indices,
+ used_param_names,
+ ..
+ } = visitor;
- let mut projection = used_column_indices.into_iter().collect::<Vec<_>>();
+ let mut projection = used_indices.into_iter().collect::<Vec<_>>();
projection.sort();
let column_index_map = projection
.iter()
+ .copied()
.enumerate()
- .map(|(projected, original)| (*original, projected))
+ .map(|(new_idx, original)| (original, new_idx))
.collect::<HashMap<_, _>>();
let projected_body = Arc::clone(&body)
@@ -124,11 +128,19 @@ impl LambdaExpr {
.expect("closure should be infallible")
.data;
+ let used_param_indices = params
+ .iter()
+ .enumerate()
+ .filter(|(_, name)| used_param_names.contains(*name))
+ .map(|(i, _)| i)
+ .collect();
+
Self {
params,
body,
projected_body,
projection,
+ used_param_indices,
}
}
@@ -170,6 +182,75 @@ impl LambdaExpr {
pub(crate) fn projected_body(&self) -> &Arc<dyn PhysicalExpr> {
&self.projected_body
}
+
+ /// Indices into [`params`](Self::params) of the parameters the body
+ /// actually references, in declaration order. See `CollectUsedVisitor`
+ /// in this module.
+ ///
+ /// Relies on the planner appending each lambda's own params after
+ /// captures, matching the `captures ++ used_params` layout
+ /// `LambdaArgument::new` builds.
+ pub fn used_param_indices(&self) -> &[usize] {
+ &self.used_param_indices
+ }
+}
+
+/// Walks the body of a [`LambdaExpr`] and collects, on a single pass:
+///
+/// * `used_indices` — every `Column` / `LambdaVariable` index referenced
+/// anywhere in the tree (including inside nested lambdas). This drives
+/// the `projection` used to slice the outer batch.
+/// * `used_param_names` — the subset of *this* lambda's `own_params` that
+/// the body actually references.
+///
+/// A nested lambda can declare its own parameter with the same name as
+/// one of `own_params` — a distinct variable that happens to reuse the
+/// name (variable shadowing). E.g. in
+/// `(k, v) -> func(col, (k, v2) -> k + v2 + v)`, the inner `k` is not
+/// `own_params`' `k`; only `v` should flow up as used, not `k`.
+///
+/// `shadow_stack` holds one frame per nested `LambdaExpr` currently being
+/// visited, each frame being that lambda's own parameter names. A
+/// `LambdaVariable` only counts toward `used_param_names` if its name
+/// isn't in any active frame (i.e. not shadowed).
+///
+/// The stack is maintained via `TreeNodeVisitor`'s `f_down` / `f_up`:
+/// push a frame when entering a nested [`LambdaExpr`], pop it when leaving.
+struct CollectUsedVisitor<'a> {
+ own_params: &'a HashSet<String>,
+ used_indices: HashSet<usize>,
+ used_param_names: HashSet<String>,
+ shadow_stack: Vec<HashSet<String>>,
+}
+
+impl TreeNodeVisitor<'_> for CollectUsedVisitor<'_> {
+ type Node = Arc<dyn PhysicalExpr>;
+
+ fn f_down(&mut self, node: &Self::Node) -> Result<TreeNodeRecursion> {
+ if let Some(col) = node.downcast_ref::<Column>() {
+ self.used_indices.insert(col.index());
+ } else if let Some(var) = node.downcast_ref::<LambdaVariable>() {
+ self.used_indices.insert(var.index());
+
+ let name = var.name();
+ let shadowed = self.shadow_stack.iter().any(|frame| frame.contains(name));
+ if !shadowed && self.own_params.contains(name) {
+ self.used_param_names.insert(name.to_string());
+ }
+ } else if let Some(nested) = node.downcast_ref::<LambdaExpr>() {
+ self.shadow_stack
+ .push(nested.params.iter().cloned().collect());
+ }
+
+ Ok(TreeNodeRecursion::Continue)
+ }
+
+ fn f_up(&mut self, node: &Self::Node) -> Result<TreeNodeRecursion> {
+ if node.downcast_ref::<LambdaExpr>().is_some() {
+ self.shadow_stack.pop();
+ }
+ Ok(TreeNodeRecursion::Continue)
+ }
}
impl std::fmt::Display for LambdaExpr {
@@ -234,7 +315,7 @@ impl PhysicalExpr for LambdaExpr {
}
}
-/// Create a lambda expression
+/// Create a lambda expression.
pub fn lambda(
params: impl IntoIterator<Item = impl Into<String>>,
body: Arc<dyn PhysicalExpr>,
@@ -273,10 +354,15 @@ fn check_async_udf(body: &Arc<dyn PhysicalExpr>) -> Result<()> {
#[cfg(test)]
mod tests {
- use crate::expressions::{NoOp, lambda::lambda};
- use arrow::{array::RecordBatch, datatypes::Schema};
+ use crate::expressions::{Column, LambdaVariable, NoOp, lambda::lambda};
+ use arrow::{
+ array::RecordBatch,
+ datatypes::{DataType, Field, Schema},
+ };
use std::sync::Arc;
+ use super::LambdaExpr;
+
#[test]
fn test_lambda_evaluate() {
let lambda = lambda(["a"], Arc::new(NoOp::new())).unwrap();
@@ -288,4 +374,125 @@ mod tests {
fn test_lambda_duplicate_name() {
assert!(lambda(["a", "a"], Arc::new(NoOp::new())).is_err());
}
+
+ /// A two-parameter lambda whose body only references the second
+ /// parameter (`v`) must report only `v` as used. The higher-order
+ /// function uses this set to push only `v` into the merged batch, so
+ /// the body's compressed `LambdaVariable` index for `v` lines up with
+ /// the batch layout.
+ #[test]
+ fn test_used_params_collects_only_referenced_param() {
+ let v_field = Arc::new(Field::new("v", DataType::Int32, true));
+ let body = Arc::new(LambdaVariable::new(1, Arc::clone(&v_field)));
+
+ let lambda =
+ LambdaExpr::try_new(vec!["k".to_string(), "v".to_string()], body).unwrap();
+
+ assert_eq!(lambda.projection(), &[1]);
+ assert_eq!(lambda.used_param_indices(), &[1]);
+ }
+
+ /// A body that references neither declared parameter reports no used params.
+ #[test]
+ fn test_used_params_all_unused() {
+ let body = Arc::new(NoOp::new());
+
+ let lambda =
+ LambdaExpr::try_new(vec!["k".to_string(), "v".to_string()], body).unwrap();
+
+ assert!(lambda.projection().is_empty());
+ assert!(lambda.used_param_indices().is_empty());
+ }
+
+ /// A three-parameter lambda that skips the middle parameter reports only the ends as used.
+ #[test]
+ fn test_used_params_three_params_middle_unused() {
+ let a_field = Arc::new(Field::new("a", DataType::Int32, true));
+ let c_field = Arc::new(Field::new("c", DataType::Int32, true));
+ let body = Arc::new(crate::expressions::BinaryExpr::new(
+ Arc::new(LambdaVariable::new(0, Arc::clone(&a_field))),
+ datafusion_expr::Operator::Plus,
+ Arc::new(LambdaVariable::new(2, Arc::clone(&c_field))),
+ ));
+
+ let lambda = LambdaExpr::try_new(
+ vec!["a".to_string(), "b".to_string(), "c".to_string()],
+ body,
+ )
+ .unwrap();
+
+ assert_eq!(lambda.used_param_indices(), &[0, 2]);
+ }
+
+ /// Referencing params out of declaration order still reports both as used.
+ #[test]
+ fn test_used_params_both_used_in_reverse_reference_order() {
+ let k_field = Arc::new(Field::new("k", DataType::Int32, true));
+ let v_field = Arc::new(Field::new("v", DataType::Int32, true));
+ let body = Arc::new(crate::expressions::BinaryExpr::new(
+ Arc::new(LambdaVariable::new(1, Arc::clone(&v_field))),
+ datafusion_expr::Operator::Plus,
+ Arc::new(LambdaVariable::new(0, Arc::clone(&k_field))),
+ ));
+
+ let lambda =
+ LambdaExpr::try_new(vec!["k".to_string(), "v".to_string()], body).unwrap();
+
+ assert_eq!(lambda.projection(), &[0, 1]);
+ assert_eq!(lambda.used_param_indices(), &[0, 1]);
+ }
+
+ /// Inside a nested lambda that re-declares one of the outer parameter
+ /// names, only the non-shadowed outer references should be reported as
+ /// used by the outer lambda. In
+ /// `(k, v) -> func(col, (k, v2) -> k + v2 + v)` the inner `k` shadows
+ /// the outer `k`, so the outer lambda must only see `v` as used.
+ #[test]
+ fn test_used_params_handles_shadowing_inside_nested_lambda() {
+ let outer_k_field = Arc::new(Field::new("k", DataType::Int32, true));
+ let outer_v_field = Arc::new(Field::new("v", DataType::Int32, true));
+ let inner_v2_field = Arc::new(Field::new("v2", DataType::Int32, true));
+
+ // Inner lambda body references "k" (inner's), "v2" (inner's), and
+ // "v" (outer's). Build it directly with the dense compressed
+ // indices the inner LambdaExpr::new would produce: sorted referenced
+ // indices, so the names alone matter here — what matters for
+ // shadow tracking is the names, not the indices.
+ let inner_body: Arc<dyn crate::PhysicalExpr> =
+ Arc::new(crate::expressions::BinaryExpr::new(
+ Arc::new(crate::expressions::BinaryExpr::new(
+ Arc::new(LambdaVariable::new(1, Arc::clone(&outer_k_field))),
+ datafusion_expr::Operator::Plus,
+ Arc::new(LambdaVariable::new(2, Arc::clone(&inner_v2_field))),
+ )),
+ datafusion_expr::Operator::Plus,
+ Arc::new(LambdaVariable::new(0, Arc::clone(&outer_v_field))),
+ ));
+ let inner_lambda = Arc::new(
+ LambdaExpr::try_new(vec!["k".to_string(), "v2".to_string()], inner_body)
+ .unwrap(),
+ );
+
+ // Outer body wraps the inner lambda in a binary op next to a
+ // regular column reference so the walk has something non-trivial
+ // to descend through. The outer body references the inner lambda
+ // via `inner_lambda`.
+ let outer_body: Arc<dyn crate::PhysicalExpr> =
+ Arc::new(crate::expressions::BinaryExpr::new(
+ Arc::new(Column::new("col", 0)),
+ datafusion_expr::Operator::Plus,
+ inner_lambda,
+ ));
+
+ let outer_lambda =
+ LambdaExpr::try_new(vec!["k".to_string(), "v".to_string()], outer_body)
+ .unwrap();
+
+ assert_eq!(
+ outer_lambda.used_param_indices(),
+ &[1],
+ "only outer's `v` (index 1) should be reported as used; `k` (index 0) is \
+ shadowed inside the nested lambda"
+ );
+ }
}
diff --git a/datafusion/physical-expr/src/higher_order_function.rs b/datafusion/physical-expr/src/higher_order_function.rs
index 7390eb33a0..e28b38bd7c 100644
--- a/datafusion/physical-expr/src/higher_order_function.rs
+++ b/datafusion/physical-expr/src/higher_order_function.rs
@@ -353,6 +353,7 @@ impl PhysicalExpr for HigherOrderFunctionExpr {
} else {
Some(batch.project(&projection)?)
},
+ lambda.used_param_indices(),
)))
}
ArgSlot::Value => {
@@ -509,15 +510,18 @@ mod tests {
use super::*;
use crate::HigherOrderFunctionExpr;
+ use crate::create_physical_expr;
use crate::expressions::Column;
use crate::expressions::NoOp;
use crate::expressions::lambda;
use crate::expressions::not;
- use arrow::array::NullArray;
use arrow::array::RecordBatchOptions;
+ use arrow::array::{ArrayRef, Int32Array};
use arrow::datatypes::{DataType, Field, Schema};
use datafusion_common::Result;
use datafusion_common::assert_contains;
+ use datafusion_expr::execution_props::ExecutionProps;
+ use datafusion_expr::physical_planning_context::PhysicalPlanningContext;
use datafusion_expr::{
HigherOrderFunctionArgs, HigherOrderSignature, HigherOrderUDF, HigherOrderUDFImpl,
};
@@ -545,9 +549,11 @@ mod tests {
_step: usize,
_fields: &[ValueOrLambda<FieldRef, Option<FieldRef>>],
) -> Result<LambdaParametersProgress> {
- Ok(LambdaParametersProgress::Complete(vec![vec![Arc::new(
- Field::new("", DataType::Null, true),
- )]]))
+ // Offer two params; single-param lambdas just ignore the second.
+ Ok(LambdaParametersProgress::Complete(vec![vec![
+ Arc::new(Field::new("", DataType::Int32, true)),
+ Arc::new(Field::new("", DataType::Int32, true)),
+ ]]))
}
fn return_field_from_args(
@@ -567,7 +573,18 @@ mod tests {
) -> Result<ColumnarValue> {
match &args.args[0] {
ValueOrLambda::Lambda(lambda) => lambda.evaluate(
- &[&|| Ok(Arc::new(NullArray::new(args.number_rows)))],
+ &[
+ // Sentinel for the first param, distinct from the second's value.
+ &|| {
+ Ok(Arc::new(Int32Array::from(vec![-1000; args.number_rows]))
+ as ArrayRef)
+ },
+ &|| {
+ Ok(Arc::new(Int32Array::from_iter_values(
+ (0..args.number_rows as i32).map(|i| 10 * (i + 1)),
+ )) as ArrayRef)
+ },
+ ],
|arrays| Ok(arrays.to_vec()),
),
ValueOrLambda::Value(value) => Ok(value.clone()),
@@ -715,4 +732,54 @@ mod tests {
"mock_function received a lambda via with_new_children at position 0 that wasn't a lambda before"
);
}
+
+ /// Exercises the real planner end to end (not hand-picked indices) to
+ /// check the "captures before own-params" layout invariant.
+ #[test]
+ fn test_higher_order_function_two_lambda_params_capture_and_unused_param() {
+ use datafusion_common::DFSchema;
+ use datafusion_expr::expr::{HigherOrderFunction, LambdaVariable};
+ use datafusion_expr::{Expr, col, lambda as logical_lambda};
+
+ let fun = Arc::new(HigherOrderUDF::new_from_impl(MockHigherOrderUDF {
+ signature: HigherOrderSignature::variadic_any(Volatility::Stable),
+ }));
+
+ // Body uses capture "a" and param "v"; param "k" is left unused.
+ let v = Expr::LambdaVariable(LambdaVariable::new(
+ "v".to_string(),
+ Some(Arc::new(Field::new("v", DataType::Int32, true))),
+ ));
+ let body = col("a") + v;
+ let lambda_expr = logical_lambda(["k", "v"], body);
+
+ let schema = DFSchema::from_unqualified_fields(
+ vec![Field::new("a", DataType::Int32, false)].into(),
+ std::collections::HashMap::new(),
+ )
+ .unwrap();
+
+ let physical_expr = create_physical_expr(
+ &Expr::HigherOrderFunction(HigherOrderFunction::new(fun, vec![lambda_expr])),
+ &schema,
+ &ExecutionProps::new(),
+ &PhysicalPlanningContext::default(),
+ )
+ .unwrap();
+
+ let batch = RecordBatch::try_new(
+ Arc::clone(schema.inner()),
+ vec![Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef],
+ )
+ .unwrap();
+
+ let result = physical_expr.evaluate(&batch).unwrap();
+ let ColumnarValue::Array(result) = result else {
+ unreachable!()
+ };
+
+ // a + v; k's sentinel (-1000) must not leak into the result.
+ let expected = Int32Array::from(vec![11, 22, 33]);
+ assert_eq!(result.as_ref(), &expected);
+ }
}