Skip to content

Apache DataFusion SQL Query Engine

feat: integrate batch coalescer with async fn exec (#19342)

feniljainfd263216c986
diff --git a/datafusion/physical-plan/src/async_func.rs b/datafusion/physical-plan/src/async_func.rs
index 280995f480..7393116b5e 100644
--- a/datafusion/physical-plan/src/async_func.rs
+++ b/datafusion/physical-plan/src/async_func.rs
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+use crate::coalesce::LimitedBatchCoalescer;
use crate::metrics::{ExecutionPlanMetricsSet, MetricsSet};
use crate::stream::RecordBatchStreamAdapter;
use crate::{
@@ -24,16 +25,19 @@ use arrow::array::RecordBatch;
use arrow_schema::{Fields, Schema, SchemaRef};
use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};
use datafusion_common::{Result, assert_eq_or_internal_err};
-use datafusion_execution::{SendableRecordBatchStream, TaskContext};
+use datafusion_execution::{RecordBatchStream, SendableRecordBatchStream, TaskContext};
use datafusion_physical_expr::ScalarFunctionExpr;
use datafusion_physical_expr::async_scalar_function::AsyncFuncExpr;
use datafusion_physical_expr::equivalence::ProjectionMapping;
use datafusion_physical_expr::expressions::Column;
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
+use futures::Stream;
use futures::stream::StreamExt;
use log::trace;
use std::any::Any;
+use std::pin::Pin;
use std::sync::Arc;
+use std::task::{Context, Poll, ready};
/// This structure evaluates a set of async expressions on a record
/// batch producing a new record batch
@@ -188,7 +192,16 @@ impl ExecutionPlan for AsyncFuncExec {
let schema_captured = self.schema();
let config_options_ref = Arc::clone(context.session_config().options());
- let stream_with_async_functions = input_stream.then(move |batch| {
+ let coalesced_input_stream = CoalesceInputStream {
+ input_stream,
+ batch_coalescer: LimitedBatchCoalescer::new(
+ Arc::clone(&self.input.schema()),
+ config_options_ref.execution.batch_size,
+ None,
+ ),
+ };
+
+ let stream_with_async_functions = coalesced_input_stream.then(move |batch| {
// need to clone *again* to capture the async_exprs and schema in the
// stream and satisfy lifetime requirements.
let async_exprs_captured = Arc::clone(&async_exprs_captured);
@@ -221,6 +234,49 @@ impl ExecutionPlan for AsyncFuncExec {
}
}
+struct CoalesceInputStream {
+ input_stream: Pin<Box<dyn RecordBatchStream + Send>>,
+ batch_coalescer: LimitedBatchCoalescer,
+}
+
+impl Stream for CoalesceInputStream {
+ type Item = Result<RecordBatch>;
+
+ fn poll_next(
+ mut self: Pin<&mut Self>,
+ cx: &mut Context<'_>,
+ ) -> Poll<Option<Self::Item>> {
+ let mut completed = false;
+
+ loop {
+ if let Some(batch) = self.batch_coalescer.next_completed_batch() {
+ return Poll::Ready(Some(Ok(batch)));
+ }
+
+ if completed {
+ return Poll::Ready(None);
+ }
+
+ match ready!(self.input_stream.poll_next_unpin(cx)) {
+ Some(Ok(batch)) => {
+ if let Err(err) = self.batch_coalescer.push_batch(batch) {
+ return Poll::Ready(Some(Err(err)));
+ }
+ }
+ Some(err) => {
+ return Poll::Ready(Some(err));
+ }
+ None => {
+ completed = true;
+ if let Err(err) = self.batch_coalescer.finish() {
+ return Poll::Ready(Some(Err(err)));
+ }
+ }
+ }
+ }
+ }
+}
+
const ASYNC_FN_PREFIX: &str = "__async_fn_";
/// Maps async_expressions to new columns
@@ -307,3 +363,51 @@ impl AsyncMapper {
Arc::new(Column::new(async_expr.name(), output_idx))
}
}
+
+#[cfg(test)]
+mod tests {
+ use std::sync::Arc;
+
+ use arrow::array::{RecordBatch, UInt32Array};
+ use arrow_schema::{DataType, Field, Schema};
+ use datafusion_common::Result;
+ use datafusion_execution::{TaskContext, config::SessionConfig};
+ use futures::StreamExt;
+
+ use crate::{ExecutionPlan, async_func::AsyncFuncExec, test::TestMemoryExec};
+
+ #[tokio::test]
+ async fn test_async_fn_with_coalescing() -> Result<()> {
+ let schema =
+ Arc::new(Schema::new(vec![Field::new("c0", DataType::UInt32, false)]));
+
+ let batch = RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![Arc::new(UInt32Array::from(vec![1, 2, 3, 4, 5, 6]))],
+ )?;
+
+ let batches: Vec<RecordBatch> = (0..50).map(|_| batch.clone()).collect();
+
+ let session_config = SessionConfig::new().with_batch_size(200);
+ let task_ctx = TaskContext::default().with_session_config(session_config);
+ let task_ctx = Arc::new(task_ctx);
+
+ let test_exec =
+ TestMemoryExec::try_new_exec(&[batches], Arc::clone(&schema), None)?;
+ let exec = AsyncFuncExec::try_new(vec![], test_exec)?;
+
+ let mut stream = exec.execute(0, Arc::clone(&task_ctx))?;
+ let batch = stream
+ .next()
+ .await
+ .expect("expected to get a record batch")?;
+ assert_eq!(200, batch.num_rows());
+ let batch = stream
+ .next()
+ .await
+ .expect("expected to get a record batch")?;
+ assert_eq!(100, batch.num_rows());
+
+ Ok(())
+ }
+}