Apache DataFusion SQL Query Engine
fix: count UnnestExec output batches after splitting (#24482)
Goutam Adwant7c5a81bfaae4
diff --git a/datafusion/physical-plan/src/unnest.rs b/datafusion/physical-plan/src/unnest.rsindex 6dfc2a0e53..964225d71c 100644--- a/datafusion/physical-plan/src/unnest.rs+++ b/datafusion/physical-plan/src/unnest.rs@@ -23,10 +23,10 @@ use std::task::{Poll, ready};use super::metrics::{self, BaselineMetrics, ExecutionPlanMetricsSet, MetricBuilder, MetricCategory,- MetricsSet, RecordOutput, SplitMetrics,+ MetricsSet, SplitMetrics,};use super::{DisplayAs, ExecutionPlanProperties, PlanProperties};-use crate::stream::{BatchSplitStream, EmptyRecordBatchStream};+use crate::stream::{BatchSplitStream, EmptyRecordBatchStream, ObservedStream};use crate::{ChildrenPropertiesMode, DisplayFormatType, Distribution, ExecutionPlan,RecordBatchStream, ReplaceChildrenOptions, SendableRecordBatchStream,@@ -296,6 +296,7 @@ impl ExecutionPlan for UnnestExec {let batch_size = context.session_config().batch_size();let input = self.input.execute(partition, context)?;let metrics = UnnestMetrics::new(partition, &self.metrics);+ let baseline_metrics = metrics.baseline_metrics.clone();let stream = Box::pin(UnnestStream {input,@@ -311,10 +312,15 @@ impl ExecutionPlan for UnnestExec {// Chunking the input bounds each build to roughly `batch_size` rows, but two cases// can still produce an oversized batch (see `predict_output_lens`), so the output// goes through the shared splitter to make the bound unconditional.- Ok(Box::pin(BatchSplitStream::new(+ let stream = Box::pin(BatchSplitStream::new(stream,batch_size,SplitMetrics::new(&self.metrics, partition),+ ));+ Ok(Box::pin(ObservedStream::new(+ stream,+ baseline_metrics,+ None,)))}@@ -651,7 +657,6 @@ impl UnnestStream {// that with `None` rather than an empty batch.if let Some(batch) = result? {debug_assert!(batch.num_rows() > 0);- (&batch).record_output(&self.metrics.baseline_metrics);return Poll::Ready(Some(Ok(batch)));}continue;@@ -1435,6 +1440,7 @@ mod tests {use arrow::datatypes::{Field, Int32Type};use datafusion_common::NullHandling;use datafusion_common::test_util::batches_to_string;+ use datafusion_physical_expr_common::metrics::MetricValue;use insta::assert_snapshot;// Create a GenericListArray with the following list values:@@ -2213,6 +2219,19 @@ mod tests {options: UnnestOptions,depth: usize,) -> Result<Vec<RecordBatch>> {+ Ok(+ unnest_at_depth_with_metrics(input, batch_size, options, depth)+ .await?+ .0,+ )+ }++ async fn unnest_at_depth_with_metrics(+ input: Vec<RecordBatch>,+ batch_size: usize,+ options: UnnestOptions,+ depth: usize,+ ) -> Result<(Vec<RecordBatch>, MetricsSet)> {let input_schema = input[0].schema();let output_schema =Arc::new(Schema::new(vec![Field::new("l", DataType::Int32, true)]));@@ -2234,7 +2253,9 @@ mod tests {.with_batch_size(batch_size),),);- crate::common::collect(unnest.execute(0, task_ctx)?).await+ let batches = crate::common::collect(unnest.execute(0, task_ctx)?).await?;+ let metrics = unnest.metrics().expect("UnnestExec exposes metrics");+ Ok((batches, metrics))}/// The values an unnest produces, flattened across all output batches.@@ -2329,6 +2350,40 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn test_unnest_stream_output_metrics_after_split() -> Result<()> {+ let (batches, metrics) = unnest_at_depth_with_metrics(+ vec![list_batch(&[Some(25)])],+ 10,+ UnnestOptions::default(),+ 1,+ )+ .await?;++ assert_eq!(+ batches+ .iter()+ .map(RecordBatch::num_rows)+ .collect::<Vec<_>>(),+ vec![10, 10, 5]+ );+ let output_batches = metrics+ .sum(|metric| matches!(metric.value(), MetricValue::OutputBatches(_)))+ .expect("output_batches metric exists")+ .as_usize();+ assert_eq!(output_batches, batches.len());+ let output_rows = metrics+ .sum(|metric| matches!(metric.value(), MetricValue::OutputRows(_)))+ .expect("output_rows metric exists")+ .as_usize();+ assert_eq!(+ output_rows,+ batches.iter().map(RecordBatch::num_rows).sum::<usize>()+ );++ Ok(())+ }+#[tokio::test]async fn test_unnest_stream_chunking_preserves_null_handling() -> Result<()> {// NULL and empty lists each contribute one NULL output row under