Skip to content

Apache DataFusion SQL Query Engine

Add support for lambda column capture (#21323)

gstvg3b634aaef9c2
diff --git a/datafusion/common/src/utils/mod.rs b/datafusion/common/src/utils/mod.rs
index 8c88be03fd..acee7b7a84 100644
--- a/datafusion/common/src/utils/mod.rs
+++ b/datafusion/common/src/utils/mod.rs
@@ -31,19 +31,23 @@ use arrow::array::{
cast::AsArray,
};
use arrow::array::{
- Datum, GenericListArray, Int32Array, Int64Array, MutableArrayData, make_array,
+ ArrowPrimitiveType, Datum, GenericListArray, Int32Array, Int64Array,
+ MutableArrayData, PrimitiveArray, make_array,
};
use arrow::array::{LargeListViewArray, ListViewArray};
use arrow::buffer::{OffsetBuffer, ScalarBuffer};
use arrow::compute::kernels::cmp::neq;
use arrow::compute::kernels::length::length;
use arrow::compute::{SortColumn, SortOptions, partition};
-use arrow::datatypes::{DataType, Field, SchemaRef};
+use arrow::datatypes::{
+ ArrowNativeType, DataType, Field, Int32Type, Int64Type, SchemaRef,
+};
#[cfg(feature = "sql")]
use sqlparser::{ast::Ident, dialect::GenericDialect, parser::Parser};
use std::borrow::{Borrow, Cow};
use std::cmp::{Ordering, min};
use std::collections::HashSet;
+use std::iter::repeat_n;
use std::num::NonZero;
use std::ops::Range;
use std::sync::{Arc, LazyLock};
@@ -1181,6 +1185,74 @@ fn truncate_list_nulls<O: OffsetSizeTrait>(
Ok(list.clone())
}
+/// If `array` is a list or a map, returns a new array of the same length as it's inner values
+/// where each value is the 1-based index of the sublist it's contained. Example:
+///
+/// `[[1], [2, 3], [4, 5, 6]] => [1, 2, 2, 3, 3, 3]`
+///
+/// Otherwise returns an error
+pub fn list_values_row_number(array: &dyn Array) -> Result<ArrayRef> {
+ match array.data_type() {
+ DataType::List(_) => Ok(Arc::new(variable_size_list_values_row_number::<
+ Int32Type,
+ >(array.as_list().offsets()))),
+ DataType::LargeList(_) => Ok(Arc::new(variable_size_list_values_row_number::<
+ Int64Type,
+ >(array.as_list().offsets()))),
+ DataType::ListView(_) => Ok(Arc::new(variable_size_list_values_row_number::<
+ Int32Type,
+ >(array.as_list_view().offsets()))),
+ DataType::LargeListView(_) => {
+ Ok(Arc::new(variable_size_list_values_row_number::<Int64Type>(
+ array.as_list_view().offsets(),
+ )))
+ }
+ DataType::FixedSizeList(_, _) => {
+ let fixed_size_list = array.as_fixed_size_list();
+
+ Ok(Arc::new(fsl_values_row_number(
+ fixed_size_list.value_length(),
+ fixed_size_list.len(),
+ )?))
+ }
+ DataType::Map(_, _) => Ok(Arc::new(variable_size_list_values_row_number::<
+ Int32Type,
+ >(array.as_map().offsets()))),
+ other => _exec_err!("expected list, got {other}"),
+ }
+}
+
+/// [0, 2, 2, 5, 6] -> [0, 0, 2, 2, 2, 3]
+fn variable_size_list_values_row_number<T: ArrowPrimitiveType>(
+ offsets: &[T::Native],
+) -> PrimitiveArray<T> {
+ let mut rows_number = Vec::with_capacity(
+ offsets[offsets.len() - 1].to_usize().unwrap() - offsets[0].to_usize().unwrap(),
+ );
+
+ for (i, w) in offsets.windows(2).enumerate() {
+ let len = w[1].as_usize() - w[0].as_usize();
+ rows_number.extend(repeat_n(T::Native::usize_as(i), len));
+ }
+
+ PrimitiveArray::new(rows_number.into(), None)
+}
+
+/// (2, 3) -> [0, 0, 1, 1, 2, 2]
+fn fsl_values_row_number(list_size: i32, array_len: usize) -> Result<Int32Array> {
+ let list_size = list_size.to_usize().ok_or_else(|| {
+ _exec_datafusion_err!("fsl_values_index: invalid list_size {list_size}")
+ })?;
+
+ let mut rows_number = Vec::with_capacity(list_size * array_len);
+
+ for i in 0..array_len {
+ rows_number.extend(repeat_n(i as i32, list_size));
+ }
+
+ Ok(PrimitiveArray::new(rows_number.into(), None))
+}
+
#[cfg(test)]
mod tests {
use std::sync::Arc;
@@ -1617,4 +1689,85 @@ mod tests {
assert_eq!(res.values(), expected.values());
assert_eq!(res.offsets(), expected.offsets());
}
+
+ #[test]
+ fn test_list_array_values_row_number() {
+ assert_eq!(
+ variable_size_list_values_row_number::<Int32Type>(
+ &OffsetBuffer::from_lengths([1, 3, 0, 2,])
+ ),
+ Int32Array::from(vec![0, 1, 1, 1, 3, 3])
+ );
+
+ assert_eq!(
+ variable_size_list_values_row_number::<Int32Type>(
+ &OffsetBuffer::from_lengths([])
+ ),
+ Int32Array::new_null(0)
+ );
+
+ assert_eq!(
+ variable_size_list_values_row_number::<Int32Type>(
+ &OffsetBuffer::from_lengths([0])
+ ),
+ Int32Array::new_null(0)
+ );
+
+ assert_eq!(
+ variable_size_list_values_row_number::<Int32Type>(
+ &OffsetBuffer::from_lengths([0, 0])
+ ),
+ Int32Array::new_null(0)
+ );
+
+ assert_eq!(
+ variable_size_list_values_row_number::<Int32Type>(
+ &OffsetBuffer::from_lengths([1])
+ ),
+ Int32Array::from(vec![0])
+ );
+
+ assert_eq!(
+ variable_size_list_values_row_number::<Int32Type>(
+ &OffsetBuffer::from_lengths([2])
+ ),
+ Int32Array::from(vec![0, 0])
+ );
+ }
+
+ #[test]
+ fn test_fsl_values_row_number() {
+ assert_eq!(
+ fsl_values_row_number(2, 3).unwrap(),
+ Int32Array::from(vec![0, 0, 1, 1, 2, 2])
+ );
+
+ assert_eq!(
+ fsl_values_row_number(1, 3).unwrap(),
+ Int32Array::from(vec![0, 1, 2])
+ );
+
+ assert_eq!(
+ fsl_values_row_number(2, 1).unwrap(),
+ Int32Array::from(vec![0, 0])
+ );
+
+ assert_eq!(
+ fsl_values_row_number(2, 0).unwrap(),
+ Int32Array::new_null(0),
+ );
+
+ assert_eq!(
+ fsl_values_row_number(0, 2).unwrap(),
+ Int32Array::new_null(0),
+ );
+
+ assert_eq!(
+ fsl_values_row_number(0, 0).unwrap(),
+ Int32Array::new_null(0),
+ );
+
+ fsl_values_row_number(-1, 2).unwrap_err();
+ fsl_values_row_number(-1, 0).unwrap_err();
+ }
}
diff --git a/datafusion/expr/src/execution_props.rs b/datafusion/expr/src/execution_props.rs
index 24d0f333a6..649f74ed39 100644
--- a/datafusion/expr/src/execution_props.rs
+++ b/datafusion/expr/src/execution_props.rs
@@ -19,6 +19,7 @@ use crate::var_provider::{VarProvider, VarType};
use chrono::{DateTime, Utc};
use datafusion_common::HashMap;
use datafusion_common::ScalarValue;
+use datafusion_common::TableReference;
use datafusion_common::alias::AliasGenerator;
use datafusion_common::config::ConfigOptions;
use datafusion_common::{Result, internal_err};
@@ -69,6 +70,10 @@ pub struct ExecutionProps {
/// Shared results container for uncorrelated scalar subquery values.
/// Populated at execution time by `ScalarSubqueryExec`.
pub subquery_results: ScalarSubqueryResults,
+ /// Maps each lambda variable name to its lambda qualifier generated
+ /// during physical planning. Populated by the physical planner for
+ /// each lambda before calling `create_physical_expr`.
+ pub lambda_variable_qualifier: HashMap<String, TableReference>,
}
impl Default for ExecutionProps {
@@ -87,6 +92,7 @@ impl ExecutionProps {
var_providers: None,
subquery_indexes: HashMap::new(),
subquery_results: ScalarSubqueryResults::default(),
+ lambda_variable_qualifier: HashMap::new(),
}
}
@@ -145,6 +151,22 @@ impl ExecutionProps {
pub fn config_options(&self) -> Option<&Arc<ConfigOptions>> {
self.config_options.as_ref()
}
+
+ /// Adds a mapping for each variable to the given qualifier. Existing
+ /// variables with conflicting names get's shadowed
+ pub fn with_qualified_lambda_variables(
+ mut self,
+ qualifier: &TableReference,
+ variables: &[String],
+ ) -> Self {
+ for var in variables {
+ self.lambda_variable_qualifier
+ .entry_ref(var)
+ .insert(qualifier.clone());
+ }
+
+ self
+ }
}
/// Index of a scalar subquery within a [`ScalarSubqueryResults`] container.
@@ -252,7 +274,7 @@ mod test {
fn debug() {
let props = ExecutionProps::new();
assert_eq!(
- "ExecutionProps { query_execution_start_time: None, alias_generator: AliasGenerator { next_id: 1 }, config_options: None, var_providers: None, subquery_indexes: {}, subquery_results: [] }",
+ "ExecutionProps { query_execution_start_time: None, alias_generator: AliasGenerator { next_id: 1 }, config_options: None, var_providers: None, subquery_indexes: {}, subquery_results: [], lambda_variable_qualifier: {} }",
format!("{props:?}")
);
}
diff --git a/datafusion/expr/src/higher_order_function.rs b/datafusion/expr/src/higher_order_function.rs
index 0e238ffc65..5fac02846a 100644
--- a/datafusion/expr/src/higher_order_function.rs
+++ b/datafusion/expr/src/higher_order_function.rs
@@ -23,7 +23,7 @@ use arrow::array::{ArrayRef, RecordBatch};
use arrow::datatypes::{DataType, FieldRef, Schema};
use arrow_schema::SchemaRef;
use datafusion_common::config::ConfigOptions;
-use datafusion_common::{Result, ScalarValue, not_impl_err};
+use datafusion_common::{Result, ScalarValue, exec_err, not_impl_err};
use datafusion_expr_common::dyn_eq::{DynEq, DynHash};
use datafusion_expr_common::signature::Volatility;
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
@@ -224,35 +224,140 @@ pub struct LambdaArgument {
/// per outer sublist), avoiding the per-call `Schema::new` build that
/// includes constructing the internal name -> index map.
schema: SchemaRef,
+ /// A RecordBatch containing the captured columns inside this lambda body, if any
+ ///
+ /// For example, for `array_transform([2], v -> v + a + b)`,
+ /// this will be a `RecordBatch` with two columns, `a` and `b`
+ captures: Option<RecordBatch>,
}
impl LambdaArgument {
- pub fn new(params: Vec<FieldRef>, body: Arc<dyn PhysicalExpr>) -> Self {
- let schema = Arc::new(Schema::new(params.clone()));
+ pub fn new(
+ params: Vec<FieldRef>,
+ body: Arc<dyn PhysicalExpr>,
+ captures: Option<RecordBatch>,
+ ) -> Self {
+ let fields = match &captures {
+ Some(batch) => batch
+ .schema_ref()
+ .fields()
+ .iter()
+ .cloned()
+ .chain(params.clone())
+ .collect(),
+ None => params.clone(),
+ };
+
+ let schema = Arc::new(Schema::new(fields));
+
Self {
params,
body,
schema,
+ captures,
}
}
/// Evaluate this lambda
/// `args` should evaluate to the value of each parameter
/// of the correspondent lambda returned in [HigherOrderUDF::lambda_parameters].
+ ///
+ /// `spread_captures` is responsible for transforming the captured column arrays
+ /// so they align with the evaluation batch. Captures are snapshotted from the
+ /// outer batch at construction time, giving one value per outer row, but the
+ /// function may evaluate the lambda body over a batch with a different number
+ /// of rows. It is the function's responsibility to provide the appropriate
+ /// `spread_captures` closure to expand (or otherwise reshape) the captures
+ /// to match.
+ ///
+ /// Taking as an example the following table:
+ ///
+ /// ```sql
+ /// CREATE TABLE t (arr INT[], a INT) AS VALUES
+ /// ([1, 2, 3], 10),
+ /// ([], 20),
+ /// ([4], 30);
+ /// ```
+ ///
+ /// `SELECT array_transform(arr, v -> v + a) from t` would execute over three outer rows:
+ ///
+ /// ```text
+ /// arr (ListArray): [[1, 2, 3], [], [4]] -- 3 outer rows, 4 total elements
+ /// a (captured): [10, 20, 30] -- one value per outer row
+ /// ```
+ ///
+ /// `array_transform` flattens the list elements into a single batch of 4 rows,
+ /// so `spread_captures` must repeat/drop captured values to match:
+ ///
+ /// ```text
+ /// v (flattened args): [1, 2, 3, 4]
+ /// a (spread): [10, 10, 10, 30] -- 10 repeated for 3 elements in row 0,
+ /// -- 20 dropped for the empty sublist in row 1,
+ /// -- 30 once for the single element in row 2
+ /// ```
+ ///
+ /// The lambda body `v + a` then evaluates element-wise over these 4-row arrays,
+ /// producing `[11, 12, 13, 34]`, which `array_transform` reassembles into `[[11, 12, 13], [], [34]]`.
+ ///
+ /// If the lambda has no captures, `spread_captures` is never called.
pub fn evaluate(
&self,
args: &[&dyn Fn() -> Result<ArrayRef>],
+ spread_captures: impl FnOnce(&[ArrayRef]) -> Result<Vec<ArrayRef>>,
) -> Result<ColumnarValue> {
- let columns = args
+ let spread_captures = self
+ .captures
+ .as_ref()
+ .map(|captures| {
+ let spread_columns = spread_captures(captures.columns())?;
+
+ RecordBatch::try_new(captures.schema(), spread_columns)
+ })
+ .transpose()?;
+
+ let merged = merge_captures_with_variables(
+ spread_captures.as_ref(),
+ Arc::clone(&self.schema),
+ &self.params,
+ args,
+ )?;
+
+ self.body.evaluate(&merged)
+ }
+}
+
+fn merge_captures_with_variables(
+ captures: Option<&RecordBatch>,
+ schema: SchemaRef,
+ params: &[FieldRef],
+ variables: &[&dyn Fn() -> Result<ArrayRef>],
+) -> Result<RecordBatch> {
+ if variables.len() < params.len() {
+ return exec_err!(
+ "expected at least {} lambda arguments to merge with captures, got {}",
+ params.len(),
+ variables.len()
+ );
+ }
+
+ let columns = match captures {
+ Some(captures) => {
+ let mut columns = captures.columns().to_vec();
+
+ for arg in &variables[..params.len()] {
+ columns.push(arg()?);
+ }
+
+ columns
+ }
+ None => variables
.iter()
- .take(self.params.len())
+ .take(params.len())
.map(|arg| arg())
- .collect::<Result<_>>()?;
-
- let batch = RecordBatch::try_new(Arc::clone(&self.schema), columns)?;
+ .collect::<Result<_>>()?,
+ };
- self.body.evaluate(&batch)
- }
+ Ok(RecordBatch::try_new(schema, columns)?)
}
/// Information about arguments passed to the function
diff --git a/datafusion/functions-nested/src/array_any_match.rs b/datafusion/functions-nested/src/array_any_match.rs
index e0a56e0f3c..dce06bb2f2 100644
--- a/datafusion/functions-nested/src/array_any_match.rs
+++ b/datafusion/functions-nested/src/array_any_match.rs
@@ -20,11 +20,14 @@
use arrow::{
array::{Array, AsArray, BooleanArray, BooleanBuilder, new_null_array},
buffer::NullBuffer,
+ compute::take_arrays,
datatypes::{ArrowNativeType, DataType, Field, FieldRef},
};
use datafusion_common::{
Result, exec_datafusion_err, exec_err, plan_err,
- utils::{adjust_offsets_for_slice, list_values, take_function_args},
+ utils::{
+ adjust_offsets_for_slice, list_values, list_values_row_number, take_function_args,
+ },
};
use datafusion_expr::{
ColumnarValue, Documentation, HigherOrderFunctionArgs, HigherOrderReturnFieldArgs,
@@ -196,7 +199,10 @@ impl HigherOrderUDF for ArrayAnyMatch {
let values_param = || Ok(Arc::clone(&list_values));
let predicate_results = lambda
- .evaluate(&[&values_param])?
+ .evaluate(&[&values_param], |arrays| {
+ let indices = list_values_row_number(&list_array)?;
+ Ok(take_arrays(arrays, &indices, None)?)
+ })?
.into_array(list_values.len())?;
let predicate_bool = predicate_results
diff --git a/datafusion/functions-nested/src/array_transform.rs b/datafusion/functions-nested/src/array_transform.rs
index 0dcc7a7613..2542d4ab1f 100644
--- a/datafusion/functions-nested/src/array_transform.rs
+++ b/datafusion/functions-nested/src/array_transform.rs
@@ -19,11 +19,14 @@
use arrow::{
array::{Array, ArrayRef, AsArray, LargeListArray, ListArray},
+ compute::take_arrays,
datatypes::{DataType, Field, FieldRef},
};
use datafusion_common::{
Result, ScalarValue, exec_err, plan_err,
- utils::{adjust_offsets_for_slice, list_values, take_function_args},
+ utils::{
+ adjust_offsets_for_slice, list_values, list_values_row_number, take_function_args,
+ },
};
use datafusion_expr::{
ColumnarValue, Documentation, HigherOrderFunctionArgs, HigherOrderReturnFieldArgs,
@@ -203,7 +206,12 @@ impl HigherOrderUDF for ArrayTransform {
// call the transforming lambda
let transformed_values = lambda
- .evaluate(&[&values_param])?
+ .evaluate(&[&values_param], |arrays| {
+ // if any column got captured, we need to adjust it to the values arrays,
+ // duplicating values of list with multitple values and removing values of empty lists
+ let indices = list_values_row_number(&list_array)?;
+ Ok(take_arrays(arrays, &indices, None)?)
+ })?
.into_array(list_values.len())?;
let field = match args.return_field.data_type() {
diff --git a/datafusion/physical-expr/src/expressions/case.rs b/datafusion/physical-expr/src/expressions/case.rs
index 919b6684e0..20d0a9e97e 100644
--- a/datafusion/physical-expr/src/expressions/case.rs
+++ b/datafusion/physical-expr/src/expressions/case.rs
@@ -19,7 +19,7 @@ mod literal_lookup_table;
use super::{Column, Literal};
use crate::PhysicalExpr;
-use crate::expressions::{LambdaExpr, LambdaVariable, lit, try_cast};
+use crate::expressions::{LambdaVariable, lit, try_cast};
use arrow::array::*;
use arrow::compute::kernels::zip::zip;
use arrow::compute::{
@@ -33,8 +33,9 @@ use datafusion_common::{
internal_datafusion_err, internal_err,
};
use datafusion_expr::ColumnarValue;
-use indexmap::{IndexMap, IndexSet};
+use indexmap::IndexMap;
use std::borrow::Cow;
+use std::collections::BTreeSet;
use std::hash::Hash;
use std::sync::Arc;
@@ -128,7 +129,8 @@ impl CaseBody {
/// Derives a [ProjectedCaseBody] from this [CaseBody].
fn project(&self) -> Result<ProjectedCaseBody> {
// Determine the set of columns that are used in all the expressions of the case body.
- let mut used_column_indices = IndexSet::<usize>::new();
+ // Use an ordered set so lambda variables continue to be positioned after columns
+ let mut used_column_indices = BTreeSet::<usize>::new();
let mut collect_column_indices = |expr: &Arc<dyn PhysicalExpr>| {
expr.apply(|expr| {
if let Some(column) = expr.downcast_ref::<Column>() {
@@ -137,9 +139,6 @@ impl CaseBody {
expr.downcast_ref::<LambdaVariable>()
{
used_column_indices.insert(lambda_variable.index());
- } else if expr.is::<LambdaExpr>() {
- //todo: remove this branch when lambda supports column capture
- return Ok(TreeNodeRecursion::Jump);
}
Ok(TreeNodeRecursion::Continue)
})
@@ -189,9 +188,6 @@ impl CaseBody {
Arc::clone(lambda_variable.field()),
))));
}
- } else if e.is::<LambdaExpr>() {
- //todo: remove this branch when lambda supports column capture
- return Ok(Transformed::new(e, false, TreeNodeRecursion::Jump));
}
Ok(Transformed::no(e))
})
@@ -1034,8 +1030,15 @@ impl CaseExpr {
projected: &ProjectedCaseBody,
) -> Result<ColumnarValue> {
let return_type = self.data_type(&batch.schema())?;
- if projected.projection.len() < batch.num_columns() {
- let projected_batch = batch.project(&projected.projection)?;
+ // projected.projection may include indexes of lambda variables not available on this batch
+ let projection = projected
+ .projection
+ .iter()
+ .copied()
+ .filter(|index| *index < batch.num_columns())
+ .collect::<Vec<_>>();
+ if projection.len() < batch.num_columns() {
+ let projected_batch = batch.project(&projection)?;
projected
.body
.case_when_with_expr(&projected_batch, &return_type)
@@ -1057,8 +1060,15 @@ impl CaseExpr {
projected: &ProjectedCaseBody,
) -> Result<ColumnarValue> {
let return_type = self.data_type(&batch.schema())?;
- if projected.projection.len() < batch.num_columns() {
- let projected_batch = batch.project(&projected.projection)?;
+ // projected.projection may include indexes of lambda variables not available on this batch
+ let projection = projected
+ .projection
+ .iter()
+ .copied()
+ .filter(|index| *index < batch.num_columns())
+ .collect::<Vec<_>>();
+ if projection.len() < batch.num_columns() {
+ let projected_batch = batch.project(&projection)?;
projected
.body
.case_when_no_expr(&projected_batch, &return_type)
@@ -1175,14 +1185,23 @@ impl CaseExpr {
)?))
}
}
- } else if projected.projection.len() < batch.num_columns() {
- // The case expressions do not use all the columns of the input batch.
- // Project first to reduce time spent filtering.
- let projected_batch = batch.project(&projected.projection)?;
- projected.body.expr_or_expr(&projected_batch, when_value)
} else {
- // All columns are used in the case expressions, so there is no need to project.
- self.body.expr_or_expr(batch, when_value)
+ // projected.projection may include indexes of lambda variables not available on this batch
+ let projection = projected
+ .projection
+ .iter()
+ .copied()
+ .filter(|index| *index < batch.num_columns())
+ .collect::<Vec<_>>();
+ if projection.len() < batch.num_columns() {
+ // The case expressions do not use all the columns of the input batch.
+ // Project first to reduce time spent filtering.
+ let projected_batch = batch.project(&projection)?;
+ projected.body.expr_or_expr(&projected_batch, when_value)
+ } else {
+ // All columns are used in the case expressions, so there is no need to project.
+ self.body.expr_or_expr(batch, when_value)
+ }
}
}
diff --git a/datafusion/physical-expr/src/expressions/column.rs b/datafusion/physical-expr/src/expressions/column.rs
index ba8cd5e636..7d4b0e7e2f 100644
--- a/datafusion/physical-expr/src/expressions/column.rs
+++ b/datafusion/physical-expr/src/expressions/column.rs
@@ -124,6 +124,7 @@ impl PhysicalExpr for Column {
}
fn return_field(&self, input_schema: &Schema) -> Result<FieldRef> {
+ self.bounds_check(input_schema)?;
Ok(input_schema.field(self.index).clone().into())
}
diff --git a/datafusion/physical-expr/src/expressions/lambda.rs b/datafusion/physical-expr/src/expressions/lambda.rs
index 267f5f605f..5e6dca1a62 100644
--- a/datafusion/physical-expr/src/expressions/lambda.rs
+++ b/datafusion/physical-expr/src/expressions/lambda.rs
@@ -20,12 +20,18 @@
use std::hash::Hash;
use std::sync::Arc;
-use crate::physical_expr::PhysicalExpr;
+use crate::{
+ expressions::{Column, LambdaVariable},
+ physical_expr::PhysicalExpr,
+};
use arrow::{
datatypes::{DataType, Schema},
record_batch::RecordBatch,
};
-use datafusion_common::plan_err;
+use datafusion_common::{
+ HashMap, plan_err,
+ tree_node::{Transformed, TreeNode, TreeNodeRecursion},
+};
use datafusion_common::{HashSet, Result, internal_err};
use datafusion_expr::ColumnarValue;
@@ -34,6 +40,8 @@ use datafusion_expr::ColumnarValue;
pub struct LambdaExpr {
params: Vec<String>,
body: Arc<dyn PhysicalExpr>,
+ projected_body: Arc<dyn PhysicalExpr>,
+ projection: Vec<usize>,
}
// Manually derive PartialEq and Hash to work around https://github.com/rust-lang/rust/issues/78808 [https://github.com/apache/datafusion/issues/13196]
@@ -61,7 +69,61 @@ impl LambdaExpr {
}
fn new(params: Vec<String>, body: Arc<dyn PhysicalExpr>) -> Self {
- Self { params, body }
+ let mut used_column_indices = HashSet::new();
+
+ body.apply(|node| {
+ if let Some(col) = node.downcast_ref::<Column>() {
+ used_column_indices.insert(col.index());
+ } else if let Some(var) = node.downcast_ref::<LambdaVariable>() {
+ used_column_indices.insert(var.index());
+ }
+
+ Ok(TreeNodeRecursion::Continue)
+ })
+ .expect("closure should be infallible");
+
+ let mut projection = used_column_indices.into_iter().collect::<Vec<_>>();
+
+ projection.sort();
+
+ let column_index_map = projection
+ .iter()
+ .enumerate()
+ .map(|(projected, original)| (*original, projected))
+ .collect::<HashMap<_, _>>();
+
+ let projected_body = Arc::clone(&body)
+ .transform_down(|e| {
+ if let Some(column) = e.downcast_ref::<Column>() {
+ let original = column.index();
+ let projected = *column_index_map.get(&original).unwrap();
+ if projected != original {
+ return Ok(Transformed::yes(Arc::new(Column::new(
+ column.name(),
+ projected,
+ ))));
+ }
+ } else if let Some(lambda_variable) = e.downcast_ref::<LambdaVariable>() {
+ let original = lambda_variable.index();
+ let projected = *column_index_map.get(&original).unwrap();
+ if projected != original {
+ return Ok(Transformed::yes(Arc::new(LambdaVariable::new(
+ projected,
+ Arc::clone(lambda_variable.field()),
+ ))));
+ }
+ }
+ Ok(Transformed::no(e))
+ })
+ .expect("closure should be infallible")
+ .data;
+
+ Self {
+ params,
+ body,
+ projected_body,
+ projection,
+ }
}
/// Get the lambda's params names
@@ -73,6 +135,14 @@ impl LambdaExpr {
pub fn body(&self) -> &Arc<dyn PhysicalExpr> {
&self.body
}
+
+ pub(crate) fn projection(&self) -> &[usize] {
+ &self.projection
+ }
+
+ pub(crate) fn projected_body(&self) -> &Arc<dyn PhysicalExpr> {
+ &self.projected_body
+ }
}
impl std::fmt::Display for LambdaExpr {
diff --git a/datafusion/physical-expr/src/expressions/lambda_variable.rs b/datafusion/physical-expr/src/expressions/lambda_variable.rs
index fb9657e897..1c130ab12e 100644
--- a/datafusion/physical-expr/src/expressions/lambda_variable.rs
+++ b/datafusion/physical-expr/src/expressions/lambda_variable.rs
@@ -107,7 +107,8 @@ impl PhysicalExpr for LambdaVariable {
if self.field.as_ref() != batch.schema_ref().field(self.index) {
return exec_err!(
- "Physical LambdaVariable field doesn't match batch field during evaluation {} != {}",
+ "Field of physical LambdaVariable with index {} doesn't match batch field during evaluation {} != {}",
+ self.index,
self.field,
batch.schema_ref().field(self.index)
);
diff --git a/datafusion/physical-expr/src/higher_order_function.rs b/datafusion/physical-expr/src/higher_order_function.rs
index 3b2002a150..801e69ea8f 100644
--- a/datafusion/physical-expr/src/higher_order_function.rs
+++ b/datafusion/physical-expr/src/higher_order_function.rs
@@ -337,9 +337,22 @@ impl PhysicalExpr for HigherOrderFunctionExpr {
.map(|(name, param)| param.renamed(name.as_str()))
.collect();
+ // lambda.projection may include indexes of nested lambda variables not present on this batch
+ let projection = lambda
+ .projection()
+ .iter()
+ .copied()
+ .filter(|i| *i < batch.num_columns())
+ .collect::<Vec<_>>();
+
Ok(ValueOrLambda::Lambda(LambdaArgument::new(
params,
- Arc::clone(lambda.body()),
+ Arc::clone(lambda.projected_body()),
+ if projection.is_empty() {
+ None
+ } else {
+ Some(batch.project(&projection)?)
+ },
)))
}
ArgSlot::Value => {
@@ -553,9 +566,10 @@ mod tests {
args: HigherOrderFunctionArgs,
) -> Result<ColumnarValue> {
match &args.args[0] {
- ValueOrLambda::Lambda(lambda) => {
- lambda.evaluate(&[&|| Ok(Arc::new(NullArray::new(args.number_rows)))])
- }
+ ValueOrLambda::Lambda(lambda) => lambda.evaluate(
+ &[&|| Ok(Arc::new(NullArray::new(args.number_rows)))],
+ |arrays| Ok(arrays.to_vec()),
+ ),
ValueOrLambda::Value(value) => Ok(value.clone()),
}
}
diff --git a/datafusion/physical-expr/src/planner.rs b/datafusion/physical-expr/src/planner.rs
index 9cb20de252..d0d0508a10 100644
--- a/datafusion/physical-expr/src/planner.rs
+++ b/datafusion/physical-expr/src/planner.rs
@@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
-use std::collections::HashMap;
use std::sync::Arc;
use crate::scalar_subquery::ScalarSubqueryExpr;
@@ -30,8 +29,8 @@ use datafusion_common::config::ConfigOptions;
use datafusion_common::datatype::FieldExt;
use datafusion_common::metadata::{FieldMetadata, format_type_and_metadata};
use datafusion_common::{
- DFSchema, Result, ScalarValue, ToDFSchema, exec_err, internal_datafusion_err,
- not_impl_err, plan_datafusion_err, plan_err,
+ DFSchema, Result, ScalarValue, TableReference, ToDFSchema, exec_err,
+ internal_datafusion_err, not_impl_err, plan_datafusion_err, plan_err,
};
use datafusion_expr::execution_props::ExecutionProps;
use datafusion_expr::expr::{
@@ -452,6 +451,18 @@ pub fn create_physical_expr(
);
}
+ let lambda_qualifier = 1 + input_dfschema
+ .iter()
+ .filter_map(|(qualifier, _field)| {
+ qualifier.and_then(|tbl| {
+ tbl.table().strip_prefix("lambda_")?.parse::<usize>().ok()
+ })
+ })
+ .max()
+ .unwrap_or_default();
+
+ let qualifier = TableReference::bare(format!("lambda_{lambda_qualifier}"));
+
let physical_args = args
.iter()
.map(|arg| match arg {
@@ -465,15 +476,26 @@ pub fn create_physical_expr(
})?
.into_iter()
.zip(&lambda.params)
- .map(|(field, name)| field.renamed(name.as_str()))
+ .map(|(field, name)| {
+ (Some(qualifier.clone()), field.renamed(name.as_str()))
+ });
+
+ let new_fields = input_dfschema
+ .iter()
+ .map(|(tbl, field)| (tbl.cloned(), Arc::clone(field)))
+ .chain(lambda_parameters)
.collect();
- let lambda_schema = DFSchema::from_unqualified_fields(
- lambda_parameters,
- HashMap::new(),
+ let lambda_schema = DFSchema::new_with_metadata(
+ new_fields,
+ input_dfschema.metadata().clone(),
)?;
- create_physical_expr(arg, &lambda_schema, execution_props)
+ let execution_props = execution_props
+ .clone()
+ .with_qualified_lambda_variables(&qualifier, &lambda.params);
+
+ create_physical_expr(arg, &lambda_schema, &execution_props)
}
_ => create_physical_expr(arg, input_dfschema, execution_props),
})
@@ -491,17 +513,10 @@ pub fn create_physical_expr(
config_options,
)?))
}
- Expr::Lambda(Lambda { params, body }) => {
- // tracked at https://github.com/apache/datafusion/issues/21172
- if body.any_column_refs() {
- return plan_err!("lambda doesn't support column capture");
- }
-
- expressions::lambda(
- params,
- create_physical_expr(body, input_dfschema, execution_props)?,
- )
- }
+ Expr::Lambda(Lambda { params, body }) => expressions::lambda(
+ params,
+ create_physical_expr(body, input_dfschema, execution_props)?,
+ ),
Expr::LambdaVariable(LambdaVariable {
name,
field,
@@ -511,7 +526,21 @@ pub fn create_physical_expr(
plan_datafusion_err!("unresolved LambdaVariable {name}")
})?;
- let index = input_dfschema.inner().index_of(name)?;
+ let qualifier = execution_props
+ .lambda_variable_qualifier
+ .get(name)
+ .ok_or_else(|| {
+ plan_datafusion_err!("qualifier for lambda variable {name} not found")
+ })?;
+
+ let index = input_dfschema
+ .index_of_column_by_name(Some(qualifier), name)
+ .ok_or_else(|| {
+ plan_datafusion_err!(
+ "lambda variable {qualifier}.{name} not found in planning schema"
+ )
+ })?;
+
let schema_field = input_dfschema.field(index);
// LambdaVariable.field will be made optional as in Expr::Placeholder
diff --git a/datafusion/sqllogictest/test_files/array/array_transform.slt b/datafusion/sqllogictest/test_files/array/array_transform.slt
index 235abf5b22..f726d265d9 100644
--- a/datafusion/sqllogictest/test_files/array/array_transform.slt
+++ b/datafusion/sqllogictest/test_files/array/array_transform.slt
@@ -23,11 +23,11 @@ statement ok
set datafusion.sql_parser.dialect = databricks;
statement ok
-CREATE TABLE t (list array<int>, number int)
+CREATE TABLE t (text varchar, list array<int>, number int)
AS VALUES
-([1, 50], 10),
-([4, 50], 40),
-([7, 50], 60);
+('a', [1, 50], 10),
+('b', [4, 50], 40),
+('c', [7, 50], 60);
statement ok
CREATE TABLE with_null_list (list array<int>)
@@ -99,7 +99,7 @@ logical_plan
01)Projection: array_transform(make_array(t.number), (v) -> CAST(v AS Float64) + Float64(3))
02)--TableScan: t projection=[number]
physical_plan
-01)ProjectionExec: expr=[array_transform(make_array(number@0), (v) -> CAST(v@0 AS Float64) + 3) as array_transform(make_array(t.number),(v) -> v + Float64(3))]
+01)ProjectionExec: expr=[array_transform(make_array(number@0), (v) -> CAST(v@1 AS Float64) + 3) as array_transform(make_array(t.number),(v) -> v + Float64(3))]
02)--DataSourceExec: partitions=1, partition_sizes=[1]
#cse should not eliminate subtrees containing lambdas
@@ -122,7 +122,7 @@ logical_plan
02)--Projection: make_array(t.number) AS __common_expr_1
03)----TableScan: t projection=[number]
physical_plan
-01)ProjectionExec: expr=[array_transform(__common_expr_1@0, (v) -> CAST(v@0 AS Int64) * 2) as array_transform(make_array(t.number),(v) -> v * Int64(2)), array_transform(__common_expr_1@0, (v) -> CAST(v@0 AS Int64) * 2 - 1) as array_transform(make_array(t.number),(v) -> v * Int64(2) - Int64(1))]
+01)ProjectionExec: expr=[array_transform(__common_expr_1@0, (v) -> CAST(v@1 AS Int64) * 2) as array_transform(make_array(t.number),(v) -> v * Int64(2)), array_transform(__common_expr_1@0, (v) -> CAST(v@1 AS Int64) * 2 - 1) as array_transform(make_array(t.number),(v) -> v * Int64(2) - Int64(1))]
02)--ProjectionExec: expr=[make_array(number@0) as __common_expr_1]
03)----DataSourceExec: partitions=1, partition_sizes=[1]
@@ -142,7 +142,7 @@ logical_plan
01)Projection: array_transform(make_array(t.number), (v) -> v IS NOT NULL OR Boolean(NULL)) AS array_transform(make_array(t.number),(v) -> v = v)
02)--TableScan: t projection=[number]
physical_plan
-01)ProjectionExec: expr=[array_transform(make_array(number@0), (v) -> v@0 IS NOT NULL OR NULL) as array_transform(make_array(t.number),(v) -> v = v)]
+01)ProjectionExec: expr=[array_transform(make_array(number@0), (v) -> v@1 IS NOT NULL OR NULL) as array_transform(make_array(t.number),(v) -> v = v)]
02)--DataSourceExec: partitions=1, partition_sizes=[1]
@@ -154,7 +154,7 @@ logical_plan
01)Projection: array_transform(CAST(CAST(t.list AS ListView(Int32)) AS List(Int32)), (a) -> CAST(a AS Int64) + Int64(1)) AS array_transform(arrow_cast(t.list,Utf8("ListView(Int32)")),(a) -> a + Int64(1))
02)--TableScan: t projection=[list]
physical_plan
-01)ProjectionExec: expr=[array_transform(CAST(CAST(list@0 AS ListView(Int32)) AS List(Int32)), (a) -> CAST(a@0 AS Int64) + 1) as array_transform(arrow_cast(t.list,Utf8("ListView(Int32)")),(a) -> a + Int64(1))]
+01)ProjectionExec: expr=[array_transform(CAST(CAST(list@0 AS ListView(Int32)) AS List(Int32)), (a) -> CAST(a@1 AS Int64) + 1) as array_transform(arrow_cast(t.list,Utf8("ListView(Int32)")),(a) -> a + Int64(1))]
02)--DataSourceExec: partitions=1, partition_sizes=[1]
query ?
@@ -166,11 +166,11 @@ select array_transform(arrow_cast(t.list, 'ListView(Int32)'), a -> a+1) from t;
# higher order function with inner case using lambda variables only
query ?
-select array_transform([3, 5, 0], v -> case when v > 1 then 2 when v > 4 then 6 else 8 end);
+select array_transform([1, 5, 9], v -> case when v = 1 then 2 when v = 5 then 6 else 8 end);
----
-[2, 2, 8]
+[2, 6, 8]
-#case with inner higher order function
+#case with inner higher order function (case EvalMethod::ExpressionOrExpression)
query I??
select
t.number,
@@ -186,6 +186,108 @@ order by t.number;
40 [4, 50] [5, 51]
60 [7, 50] [8, 51]
+# case with inner nested higher order function (case EvalMethod::ExpressionOrExpression)
+query T?I?
+select
+ t.text,
+ t.list,
+ t.number,
+ case
+ when t.number > 30 then array_transform(
+ [[t.list]],
+ list -> array_transform(
+ list,
+ list -> array_transform(
+ list,
+ v -> number + v + list[1]
+ )
+ )
+ )
+ else array_transform(
+ [[t.list]],
+ list -> array_transform(
+ list,
+ list -> array_transform(
+ list,
+ v -> number + list[1]
+ )
+ )
+ )
+ end
+from t
+order by t.number;
+----
+a [1, 50] 10 [[[11, 11]]]
+b [4, 50] 40 [[[48, 94]]]
+c [7, 50] 60 [[[74, 117]]]
+
+# case with inner nested higher order function where the condition is a lambda function
+# (case EvalMethod::ExpressionOrExpression)
+query T?I?
+select
+ t.text,
+ t.list,
+ t.number,
+ case
+ when array_any_match(t.list, v -> v = 7) then array_transform(
+ [[t.list]],
+ list -> array_transform(
+ list,
+ list -> array_transform(
+ list,
+ v -> number + v + list[1]
+ )
+ )
+ )
+ else array_transform(
+ [[t.list]],
+ list -> array_transform(
+ list,
+ list -> array_transform(
+ list,
+ v -> number + list[1]
+ )
+ )
+ )
+ end
+from t
+order by t.number;
+----
+a [1, 50] 10 [[[11, 11]]]
+b [4, 50] 40 [[[44, 44]]]
+c [7, 50] 60 [[[74, 117]]]
+
+# case with condition (case EvalMethod::WithExpression)
+query ?IB
+select
+ t.list,
+ t.number,
+ case array_transform(t.list, v -> v + t.number)[1]
+ when array_transform(t.list, v -> v + 10)[1] then array_any_match(t.list, v -> v == 1)
+ when array_transform(t.list, v -> v + 40)[1] then array_any_match(t.list, v -> v == 4)
+ when array_transform(t.list, v -> v + 60)[1] then array_any_match(t.list, v -> v == 7)
+ end
+from t;
+----
+[1, 50] 10 true
+[4, 50] 40 true
+[7, 50] 60 true
+
+# case without condition and multiple when/then pairs (case EvalMethod::NoExpression)
+query ??
+select
+ t.list,
+ case
+ when array_any_match(t.list, v -> v == 1) then array_transform(t.list, v -> v + 1)
+ when array_any_match(t.list, v -> v == 4) then array_transform(t.list, v -> v + 2)
+ when array_any_match(t.list, v -> v == 7) then array_transform(t.list, v -> v + 3)
+ end
+from t;
+----
+[1, 50] [2, 51]
+[4, 50] [6, 52]
+[7, 50] [10, 53]
+
# array_transform coercion rules
query TT
explain select array_transform(arrow_cast(t.list, 'FixedSizeList(2, Int32)'), a -> a+1) from t;
@@ -194,7 +296,7 @@ logical_plan
01)Projection: array_transform(CAST(CAST(t.list AS FixedSizeList(2 x Int32)) AS List(Int32)), (a) -> CAST(a AS Int64) + Int64(1)) AS array_transform(arrow_cast(t.list,Utf8("FixedSizeList(2, Int32)")),(a) -> a + Int64(1))
02)--TableScan: t projection=[list]
physical_plan
-01)ProjectionExec: expr=[array_transform(CAST(CAST(list@0 AS FixedSizeList(2 x Int32)) AS List(Int32)), (a) -> CAST(a@0 AS Int64) + 1) as array_transform(arrow_cast(t.list,Utf8("FixedSizeList(2, Int32)")),(a) -> a + Int64(1))]
+01)ProjectionExec: expr=[array_transform(CAST(CAST(list@0 AS FixedSizeList(2 x Int32)) AS List(Int32)), (a) -> CAST(a@1 AS Int64) + 1) as array_transform(arrow_cast(t.list,Utf8("FixedSizeList(2, Int32)")),(a) -> a + Int64(1))]
02)--DataSourceExec: partitions=1, partition_sizes=[1]
query ?
@@ -235,13 +337,61 @@ select array_transform(list, v -> v+1) from fully_null_list;
NULL
NULL
-# higher order function with inner case using lambda variables and captured column(capture not supported yet)
-query error DataFusion error: Error during planning: lambda doesn't support column capture
-select array_transform([3, 5, 9], v -> case when v > 1 then t.number when v > 4 then 6 else 8 end) from t;
+# higher order function with inner case using lambda variables and captured column
+query ?
+select array_transform([1, 5, 9], v -> case when v = 1 then t.number when v = 5 then 6 else 8 end) from t;
+----
+[10, 6, 8]
+[40, 6, 8]
+[60, 6, 8]
+
+# higher order function with inner case using captured column only
+query ?
+select array_transform([3, 5, 9], v -> case when t.number = 10 then 2 when t.number = 40 then 6 else 8 end) from t;
+----
+[2, 2, 2]
+[6, 6, 6]
+[8, 8, 8]
-# higher order function with inner case using captured column only(capture not supported yet)
-query error DataFusion error: Error during planning: lambda doesn't support column capture
-select array_transform([3, 5, 9], v -> case when t.number > 1 then 2 when t.number > 4 then 6 else 8 end) from t;
+# explain case with inner nested higher order function
+query TT
+explain select
+ t.text,
+ t.list,
+ t.number,
+ case
+ when t.number > 30 then array_transform(
+ [[t.list]],
+ list -> array_transform(
+ list,
+ list -> array_transform(
+ list,
+ v -> number + v + list[1]
+ )
+ )
+ )
+ else array_transform(
+ [[t.list]],
+ list -> array_transform(
+ list,
+ list -> array_transform(
+ list,
+ v -> number + list[1]
+ )
+ )
+ )
+ end
+from t
+order by t.number;
+----
+logical_plan
+01)Sort: t.number ASC NULLS LAST
+02)--Projection: t.text, t.list, t.number, CASE WHEN t.number > Int32(30) THEN array_transform(make_array(make_array(t.list)), (list) -> array_transform(list, (list) -> array_transform(list, (v) -> t.number + v + array_element(list, Int64(1))))) ELSE array_transform(make_array(make_array(t.list)), (list) -> array_transform(list, (list) -> array_transform(list, (v) -> t.number + array_element(list, Int64(1))))) END AS CASE WHEN t.number > Int64(30) THEN array_transform(make_array(make_array(t.list)),(list) -> array_transform(list,(list) -> array_transform(list,(v) -> t.number + v + list[Int64(1)]))) ELSE array_transform(make_array(make_array(t.list)),(list) -> array_transform(list,(list) -> array_transform(list,(v) -> t.number + list[Int64(1)]))) END
+03)----TableScan: t projection=[text, list, number]
+physical_plan
+01)SortExec: expr=[number@2 ASC NULLS LAST], preserve_partitioning=[false]
+02)--ProjectionExec: expr=[text@0 as text, list@1 as list, number@2 as number, CASE WHEN number@2 > 30 THEN array_transform(make_array(make_array(list@1)), (list) -> array_transform(list@3, (list) -> array_transform(list@4, (v) -> number@2 + v@5 + array_element(list@4, 1)))) ELSE array_transform(make_array(make_array(list@1)), (list) -> array_transform(list@3, (list) -> array_transform(list@4, (v) -> number@2 + array_element(list@4, 1)))) END as CASE WHEN t.number > Int64(30) THEN array_transform(make_array(make_array(t.list)),(list) -> array_transform(list,(list) -> array_transform(list,(v) -> t.number + v + list[Int64(1)]))) ELSE array_transform(make_array(make_array(t.list)),(list) -> array_transform(list,(list) -> array_transform(list,(v) -> t.number + list[Int64(1)]))) END]
+03)----DataSourceExec: partitions=1, partition_sizes=[1]
query error
select array_transform();