Apache DataFusion SQL Query Engine
feat: Add evaluate_to_arrays function (#18446)
Emily Matheys76b4156aff90
diff --git a/datafusion/expr-common/src/columnar_value.rs b/datafusion/expr-common/src/columnar_value.rsindex a21ad5bbbc..f7bbd41120 100644--- a/datafusion/expr-common/src/columnar_value.rs+++ b/datafusion/expr-common/src/columnar_value.rs@@ -113,10 +113,12 @@ impl ColumnarValue {}}- /// Convert a columnar value into an Arrow [`ArrayRef`] with the specified- /// number of rows. [`Self::Scalar`] is converted by repeating the same- /// scalar multiple times which is not as efficient as handling the scalar- /// directly.+ /// Convert any [`Self::Scalar`] into an Arrow [`ArrayRef`] with the specified+ /// number of rows by repeating the same scalar multiple times,+ /// which is not as efficient as handling the scalar directly.+ /// [`Self::Array`] will just be returned as is.+ ///+ /// See [`Self::into_array_of_size`] if you need to validate the length of the output array.////// See [`Self::values_to_arrays`] to convert multiple columnar values into/// arrays of the same length.@@ -135,6 +137,38 @@ impl ColumnarValue {/// number of rows. [`Self::Scalar`] is converted by repeating the same/// scalar multiple times which is not as efficient as handling the scalar/// directly.+ /// This validates that if this is [`Self::Array`], it has the expected length.+ ///+ /// See [`Self::values_to_arrays`] to convert multiple columnar values into+ /// arrays of the same length.+ ///+ /// # Errors+ ///+ /// Errors if `self` is a Scalar that fails to be converted into an array of size or+ /// if the array length does not match the expected length+ pub fn into_array_of_size(self, num_rows: usize) -> Result<ArrayRef> {+ match self {+ ColumnarValue::Array(array) => {+ if array.len() == num_rows {+ Ok(array)+ } else {+ internal_err!(+ "Array length {} does not match expected length {}",+ array.len(),+ num_rows+ )+ }+ }+ ColumnarValue::Scalar(scalar) => scalar.to_array_of_size(num_rows),+ }+ }++ /// Convert any [`Self::Scalar`] into an Arrow [`ArrayRef`] with the specified+ /// number of rows by repeating the same scalar multiple times,+ /// which is not as efficient as handling the scalar directly.+ /// [`Self::Array`] will just be returned as is.+ ///+ /// See [`Self::to_array_of_size`] if you need to validate the length of the output array.////// See [`Self::values_to_arrays`] to convert multiple columnar values into/// arrays of the same length.@@ -149,6 +183,36 @@ impl ColumnarValue {})}+ /// Convert a columnar value into an Arrow [`ArrayRef`] with the specified+ /// number of rows. [`Self::Scalar`] is converted by repeating the same+ /// scalar multiple times which is not as efficient as handling the scalar+ /// directly.+ /// This validates that if this is [`Self::Array`], it has the expected length.+ ///+ /// See [`Self::values_to_arrays`] to convert multiple columnar values into+ /// arrays of the same length.+ ///+ /// # Errors+ ///+ /// Errors if `self` is a Scalar that fails to be converted into an array of size or+ /// if the array length does not match the expected length+ pub fn to_array_of_size(&self, num_rows: usize) -> Result<ArrayRef> {+ match self {+ ColumnarValue::Array(array) => {+ if array.len() == num_rows {+ Ok(Arc::clone(array))+ } else {+ internal_err!(+ "Array length {} does not match expected length {}",+ array.len(),+ num_rows+ )+ }+ }+ ColumnarValue::Scalar(scalar) => scalar.to_array_of_size(num_rows),+ }+ }+/// Null columnar values are implemented as a null array in order to pass batch/// num_rowspub fn create_null_array(num_rows: usize) -> Self {@@ -249,6 +313,34 @@ mod tests {use super::*;use arrow::array::Int32Array;+ #[test]+ fn into_array_of_size() {+ // Array case+ let arr = make_array(1, 3);+ let arr_columnar_value = ColumnarValue::Array(Arc::clone(&arr));+ assert_eq!(&arr_columnar_value.into_array_of_size(3).unwrap(), &arr);++ // Scalar case+ let scalar_columnar_value = ColumnarValue::Scalar(ScalarValue::Int32(Some(42)));+ let expected_array = make_array(42, 100);+ assert_eq!(+ &scalar_columnar_value.into_array_of_size(100).unwrap(),+ &expected_array+ );++ // Array case with wrong size+ let arr = make_array(1, 3);+ let arr_columnar_value = ColumnarValue::Array(Arc::clone(&arr));+ let result = arr_columnar_value.into_array_of_size(5);+ let err = result.unwrap_err();+ assert!(+ err.to_string().starts_with(+ "Internal error: Array length 3 does not match expected length 5"+ ),+ "Found: {err}"+ );+ }+#[test]fn values_to_arrays() {// (input, expected)diff --git a/datafusion/physical-expr-common/src/utils.rs b/datafusion/physical-expr-common/src/utils.rsindex 05b216ab75..230252eb89 100644--- a/datafusion/physical-expr-common/src/utils.rs+++ b/datafusion/physical-expr-common/src/utils.rs@@ -22,6 +22,7 @@ use crate::tree_node::ExprContext;use arrow::array::{make_array, Array, ArrayRef, BooleanArray, MutableArrayData};use arrow::compute::{and_kleene, is_not_null, SlicesIterator};+use arrow::record_batch::RecordBatch;use datafusion_common::Result;use datafusion_expr_common::sort_properties::ExprProperties;@@ -91,6 +92,26 @@ pub fn scatter(mask: &BooleanArray, truthy: &dyn Array) -> Result<ArrayRef> {Ok(make_array(data))}+/// Evaluates expressions against a record batch.+/// This will convert the resulting ColumnarValues to ArrayRefs,+/// duplicating any ScalarValues that may have been returned,+/// and validating that the returned arrays all have the same+/// number of rows as the input batch.+#[inline]+pub fn evaluate_expressions_to_arrays<'a>(+ exprs: impl IntoIterator<Item = &'a Arc<dyn PhysicalExpr>>,+ batch: &RecordBatch,+) -> Result<Vec<ArrayRef>> {+ let num_rows = batch.num_rows();+ exprs+ .into_iter()+ .map(|e| {+ e.evaluate(batch)+ .and_then(|col| col.into_array_of_size(num_rows))+ })+ .collect::<Result<Vec<ArrayRef>>>()+}+#[cfg(test)]mod tests {use std::sync::Arc;diff --git a/datafusion/physical-expr/src/window/standard_window_function_expr.rs b/datafusion/physical-expr/src/window/standard_window_function_expr.rsindex ca7c3a4db3..9b1213450c 100644--- a/datafusion/physical-expr/src/window/standard_window_function_expr.rs+++ b/datafusion/physical-expr/src/window/standard_window_function_expr.rs@@ -23,6 +23,7 @@ use arrow::record_batch::RecordBatch;use datafusion_common::Result;use datafusion_expr::{LimitEffect, PartitionEvaluator};+use datafusion_physical_expr_common::utils::evaluate_expressions_to_arrays;use std::any::Any;use std::sync::Arc;@@ -57,13 +58,7 @@ pub trait StandardWindowFunctionExpr: Send + Sync + std::fmt::Debug {////// Typically, the resulting vector is a single element vector.fn evaluate_args(&self, batch: &RecordBatch) -> Result<Vec<ArrayRef>> {- self.expressions()- .iter()- .map(|e| {- e.evaluate(batch)- .and_then(|v| v.into_array(batch.num_rows()))- })- .collect()+ evaluate_expressions_to_arrays(&self.expressions(), batch)}/// Create a [`PartitionEvaluator`] for evaluating the function ondiff --git a/datafusion/physical-expr/src/window/window_expr.rs b/datafusion/physical-expr/src/window/window_expr.rsindex a6b5bf1871..47f970d276 100644--- a/datafusion/physical-expr/src/window/window_expr.rs+++ b/datafusion/physical-expr/src/window/window_expr.rs@@ -41,6 +41,7 @@ use datafusion_expr::window_state::{use datafusion_expr::{Accumulator, PartitionEvaluator, WindowFrame, WindowFrameBound};use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;+use datafusion_physical_expr_common::utils::evaluate_expressions_to_arrays;use indexmap::IndexMap;/// Common trait for [window function] implementations@@ -90,13 +91,7 @@ pub trait WindowExpr: Send + Sync + Debug {/// Evaluate the window function arguments against the batch and return/// array ref, normally the resulting `Vec` is a single element one.fn evaluate_args(&self, batch: &RecordBatch) -> Result<Vec<ArrayRef>> {- self.expressions()- .iter()- .map(|e| {- e.evaluate(batch)- .and_then(|v| v.into_array(batch.num_rows()))- })- .collect()+ evaluate_expressions_to_arrays(&self.expressions(), batch)}/// Evaluate the window function values against the batchdiff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rsindex 30d1441f57..18090b6422 100644--- a/datafusion/physical-plan/src/aggregates/mod.rs+++ b/datafusion/physical-plan/src/aggregates/mod.rs@@ -59,6 +59,7 @@ use datafusion_physical_expr_common::sort_expr::{};use datafusion_expr::utils::AggregateOrderSensitivity;+use datafusion_physical_expr_common::utils::evaluate_expressions_to_arrays;use itertools::Itertools;pub mod group_values;@@ -1434,25 +1435,14 @@ pub fn finalize_aggregation(}}-/// Evaluates expressions against a record batch.-fn evaluate(- expr: &[Arc<dyn PhysicalExpr>],- batch: &RecordBatch,-) -> Result<Vec<ArrayRef>> {- expr.iter()- .map(|expr| {- expr.evaluate(batch)- .and_then(|v| v.into_array(batch.num_rows()))- })- .collect()-}--/// Evaluates expressions against a record batch.+/// Evaluates groups of expressions against a record batch.pub fn evaluate_many(expr: &[Vec<Arc<dyn PhysicalExpr>>],batch: &RecordBatch,) -> Result<Vec<Vec<ArrayRef>>> {- expr.iter().map(|expr| evaluate(expr, batch)).collect()+ expr.iter()+ .map(|expr| evaluate_expressions_to_arrays(expr, batch))+ .collect()}fn evaluate_optional(@@ -1506,23 +1496,14 @@ pub fn evaluate_group_by(group_by: &PhysicalGroupBy,batch: &RecordBatch,) -> Result<Vec<Vec<ArrayRef>>> {- let exprs: Vec<ArrayRef> = group_by- .expr- .iter()- .map(|(expr, _)| {- let value = expr.evaluate(batch)?;- value.into_array(batch.num_rows())- })- .collect::<Result<Vec<_>>>()?;-- let null_exprs: Vec<ArrayRef> = group_by- .null_expr- .iter()- .map(|(expr, _)| {- let value = expr.evaluate(batch)?;- value.into_array(batch.num_rows())- })- .collect::<Result<Vec<_>>>()?;+ let exprs = evaluate_expressions_to_arrays(+ group_by.expr.iter().map(|(expr, _)| expr),+ batch,+ )?;+ let null_exprs = evaluate_expressions_to_arrays(+ group_by.null_expr.iter().map(|(expr, _)| expr),+ batch,+ )?;group_by.groupsdiff --git a/datafusion/physical-plan/src/aggregates/no_grouping.rs b/datafusion/physical-plan/src/aggregates/no_grouping.rsindex 9474a5f88c..fc398427ac 100644--- a/datafusion/physical-plan/src/aggregates/no_grouping.rs+++ b/datafusion/physical-plan/src/aggregates/no_grouping.rs@@ -33,12 +33,12 @@ use std::borrow::Cow;use std::sync::Arc;use std::task::{Context, Poll};+use super::AggregateExec;use crate::filter::batch_filter;use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};+use datafusion_physical_expr_common::utils::evaluate_expressions_to_arrays;use futures::stream::{Stream, StreamExt};-use super::AggregateExec;-/// stream struct for aggregation without grouping columnspub(crate) struct AggregateStream {stream: BoxStream<'static, Result<RecordBatch>>,@@ -219,13 +219,8 @@ fn aggregate_batch(None => Cow::Borrowed(&batch),};- let n_rows = batch.num_rows();-// 1.3- let values = expr- .iter()- .map(|e| e.evaluate(&batch).and_then(|v| v.into_array(n_rows)))- .collect::<Result<Vec<_>>>()?;+ let values = evaluate_expressions_to_arrays(expr, batch.as_ref())?;// 1.4let size_pre = accum.size();diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex c552e6954c..43df83b6b4 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -77,6 +77,7 @@ use datafusion_physical_expr::{PhysicalExpr, PhysicalExprRef};use ahash::RandomState;use datafusion_physical_expr_common::physical_expr::fmt_sql;+use datafusion_physical_expr_common::utils::evaluate_expressions_to_arrays;use futures::TryStreamExt;use parking_lot::Mutex;@@ -1465,13 +1466,7 @@ async fn collect_left_input(BooleanBufferBuilder::new(0)};- let left_values = on_left- .iter()- .map(|c| {- c.evaluate(&single_batch)?- .into_array(single_batch.num_rows())- })- .collect::<Result<Vec<_>>>()?;+ let left_values = evaluate_expressions_to_arrays(&on_left, &single_batch)?;// Compute bounds for dynamic filter if enabledlet bounds = match bounds_accumulators {diff --git a/datafusion/physical-plan/src/joins/hash_join/stream.rs b/datafusion/physical-plan/src/joins/hash_join/stream.rsindex 1f4aeecb29..d3d47a555a 100644--- a/datafusion/physical-plan/src/joins/hash_join/stream.rs+++ b/datafusion/physical-plan/src/joins/hash_join/stream.rs@@ -51,6 +51,7 @@ use datafusion_common::{use datafusion_physical_expr::PhysicalExprRef;use ahash::RandomState;+use datafusion_physical_expr_common::utils::evaluate_expressions_to_arrays;use futures::{ready, Stream, StreamExt};/// Represents build-side of hash join.@@ -447,11 +448,7 @@ impl HashJoinStream {}Some(Ok(batch)) => {// Precalculate hash values for fetched batch- let keys_values = self- .on_right- .iter()- .map(|c| c.evaluate(&batch)?.into_array(batch.num_rows()))- .collect::<Result<Vec<_>>>()?;+ let keys_values = evaluate_expressions_to_arrays(&self.on_right, &batch)?;self.hashes_buffer.clear();self.hashes_buffer.resize(batch.num_rows(), 0);diff --git a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs b/datafusion/physical-plan/src/joins/symmetric_hash_join.rsindex a9a2bbff42..4b18bdca1b 100644--- a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs+++ b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs@@ -78,6 +78,7 @@ use datafusion_physical_expr_common::physical_expr::{fmt_sql, PhysicalExprRef};use datafusion_physical_expr_common::sort_expr::{LexOrdering, OrderingRequirements};use ahash::RandomState;+use datafusion_physical_expr_common::utils::evaluate_expressions_to_arrays;use futures::{ready, Stream, StreamExt};use parking_lot::Mutex;@@ -1065,14 +1066,8 @@ fn lookup_join_hashmap(hashes_buffer: &mut Vec<u64>,deleted_offset: Option<usize>,) -> Result<(UInt64Array, UInt32Array)> {- let keys_values = probe_on- .iter()- .map(|c| c.evaluate(probe_batch)?.into_array(probe_batch.num_rows()))- .collect::<Result<Vec<_>>>()?;- let build_join_values = build_on- .iter()- .map(|c| c.evaluate(build_batch)?.into_array(build_batch.num_rows()))- .collect::<Result<Vec<_>>>()?;+ let keys_values = evaluate_expressions_to_arrays(probe_on, probe_batch)?;+ let build_join_values = evaluate_expressions_to_arrays(build_on, build_batch)?;hashes_buffer.clear();hashes_buffer.resize(probe_batch.num_rows(), 0);diff --git a/datafusion/physical-plan/src/joins/utils.rs b/datafusion/physical-plan/src/joins/utils.rsindex 6ff8298154..2cd24d284b 100644--- a/datafusion/physical-plan/src/joins/utils.rs+++ b/datafusion/physical-plan/src/joins/utils.rs@@ -75,6 +75,7 @@ use datafusion_physical_expr::{};use datafusion_physical_expr_common::datum::compare_op_for_nested;+use datafusion_physical_expr_common::utils::evaluate_expressions_to_arrays;use futures::future::{BoxFuture, Shared};use futures::{ready, FutureExt};use parking_lot::Mutex;@@ -1668,10 +1669,7 @@ pub fn update_hash(fifo_hashmap: bool,) -> Result<()> {// evaluate the keys- let keys_values = on- .iter()- .map(|c| c.evaluate(batch)?.into_array(batch.num_rows()))- .collect::<Result<Vec<_>>>()?;+ let keys_values = evaluate_expressions_to_arrays(on, batch)?;// calculate the hash valueslet hash_values = create_hashes(&keys_values, random_state, hashes_buffer)?;diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rsindex ead2196860..8bc2bcd6f2 100644--- a/datafusion/physical-plan/src/projection.rs+++ b/datafusion/physical-plan/src/projection.rs@@ -57,6 +57,7 @@ pub use datafusion_physical_expr::projection::{update_expr, ProjectionExpr, ProjectionExprs,};+use datafusion_physical_expr_common::utils::evaluate_expressions_to_arrays;use futures::stream::{Stream, StreamExt};use log::trace;@@ -357,14 +358,7 @@ impl ProjectionStream {fn batch_project(&self, batch: &RecordBatch) -> Result<RecordBatch> {// Records time on droplet _timer = self.baseline_metrics.elapsed_compute().timer();- let arrays = self- .expr- .iter()- .map(|expr| {- expr.evaluate(batch)- .and_then(|v| v.into_array(batch.num_rows()))- })- .collect::<Result<Vec<_>>>()?;+ let arrays = evaluate_expressions_to_arrays(&self.expr, batch)?;if arrays.is_empty() {let options =diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rsindex 8f73fe86cf..6d69f47efa 100644--- a/datafusion/physical-plan/src/repartition/mod.rs+++ b/datafusion/physical-plan/src/repartition/mod.rs@@ -58,6 +58,7 @@ use crate::filter_pushdown::{ChildPushdownResult, FilterDescription, FilterPushdownPhase,FilterPushdownPropagation,};+use datafusion_physical_expr_common::utils::evaluate_expressions_to_arrays;use futures::stream::Stream;use futures::{FutureExt, StreamExt, TryStreamExt};use log::trace;@@ -489,10 +490,8 @@ impl BatchPartitioner {// Tracking time required for distributing indexes across output partitionslet timer = self.timer.timer();- let arrays = exprs- .iter()- .map(|expr| expr.evaluate(&batch)?.into_array(batch.num_rows()))- .collect::<Result<Vec<_>>>()?;+ let arrays =+ evaluate_expressions_to_arrays(exprs.as_slice(), &batch)?;hash_buffer.clear();hash_buffer.resize(batch.num_rows(), 0);diff --git a/datafusion/physical-plan/src/sorts/stream.rs b/datafusion/physical-plan/src/sorts/stream.rsindex 97dd1761b1..f419247c82 100644--- a/datafusion/physical-plan/src/sorts/stream.rs+++ b/datafusion/physical-plan/src/sorts/stream.rs@@ -25,6 +25,7 @@ use arrow::row::{RowConverter, Rows, SortField};use datafusion_common::{internal_datafusion_err, Result};use datafusion_execution::memory_pool::MemoryReservation;use datafusion_physical_expr_common::sort_expr::LexOrdering;+use datafusion_physical_expr_common::utils::evaluate_expressions_to_arrays;use futures::stream::{Fuse, StreamExt};use std::marker::PhantomData;use std::sync::Arc;@@ -164,11 +165,7 @@ impl RowCursorStream {batch: &RecordBatch,stream_idx: usize,) -> Result<RowValues> {- let cols = self- .column_expressions- .iter()- .map(|expr| expr.evaluate(batch)?.into_array(batch.num_rows()))- .collect::<Result<Vec<_>>>()?;+ let cols = evaluate_expressions_to_arrays(&self.column_expressions, batch)?;// At this point, ownership should of this Rows should be uniquelet mut rows = self.rows.take_next(stream_idx)?;