Apache DataFusion SQL Query Engine
CoalescePartitionsExec fetch is not consistent with one partition and more than one partition (#18245)
Qi Zhube85bf41aed7
diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rsindex 5869c51b26..2597dc6408 100644--- a/datafusion/physical-plan/src/coalesce_partitions.rs+++ b/datafusion/physical-plan/src/coalesce_partitions.rs@@ -170,8 +170,18 @@ impl ExecutionPlan for CoalescePartitionsExec {"CoalescePartitionsExec requires at least one input partition"),1 => {- // bypass any threading / metrics if there is a single partition- self.input.execute(0, context)+ // single-partition path: execute child directly, but ensure fetch is respected+ // (wrap with ObservedStream only if fetch is present so we don't add overhead otherwise)+ let child_stream = self.input.execute(0, context)?;+ if self.fetch.is_some() {+ let baseline_metrics = BaselineMetrics::new(&self.metrics, partition);+ return Ok(Box::pin(ObservedStream::new(+ child_stream,+ baseline_metrics,+ self.fetch,+ )));+ }+ Ok(child_stream)}_ => {let baseline_metrics = BaselineMetrics::new(&self.metrics, partition);@@ -351,4 +361,110 @@ mod tests {collect(coalesce_partitions_exec, task_ctx).await.unwrap();}++ #[tokio::test]+ async fn test_single_partition_with_fetch() -> Result<()> {+ let task_ctx = Arc::new(TaskContext::default());++ // Use existing scan_partitioned with 1 partition (returns 100 rows per partition)+ let input = test::scan_partitioned(1);++ // Test with fetch=3+ let coalesce = CoalescePartitionsExec::new(input).with_fetch(Some(3));++ let stream = coalesce.execute(0, task_ctx)?;+ let batches = common::collect(stream).await?;++ let row_count: usize = batches.iter().map(|batch| batch.num_rows()).sum();+ assert_eq!(row_count, 3, "Should only return 3 rows due to fetch=3");++ Ok(())+ }++ #[tokio::test]+ async fn test_multi_partition_with_fetch_one() -> Result<()> {+ let task_ctx = Arc::new(TaskContext::default());++ // Create 4 partitions, each with 100 rows+ // This simulates the real-world scenario where each partition has data+ let input = test::scan_partitioned(4);++ // Test with fetch=1 (the original bug: was returning multiple rows instead of 1)+ let coalesce = CoalescePartitionsExec::new(input).with_fetch(Some(1));++ let stream = coalesce.execute(0, task_ctx)?;+ let batches = common::collect(stream).await?;++ let row_count: usize = batches.iter().map(|batch| batch.num_rows()).sum();+ assert_eq!(+ row_count, 1,+ "Should only return 1 row due to fetch=1, not one per partition"+ );++ Ok(())+ }++ #[tokio::test]+ async fn test_single_partition_without_fetch() -> Result<()> {+ let task_ctx = Arc::new(TaskContext::default());++ // Use scan_partitioned with 1 partition+ let input = test::scan_partitioned(1);++ // Test without fetch (should return all rows)+ let coalesce = CoalescePartitionsExec::new(input);++ let stream = coalesce.execute(0, task_ctx)?;+ let batches = common::collect(stream).await?;++ let row_count: usize = batches.iter().map(|batch| batch.num_rows()).sum();+ assert_eq!(+ row_count, 100,+ "Should return all 100 rows when fetch is None"+ );++ Ok(())+ }++ #[tokio::test]+ async fn test_single_partition_fetch_larger_than_batch() -> Result<()> {+ let task_ctx = Arc::new(TaskContext::default());++ // Use scan_partitioned with 1 partition (returns 100 rows)+ let input = test::scan_partitioned(1);++ // Test with fetch larger than available rows+ let coalesce = CoalescePartitionsExec::new(input).with_fetch(Some(200));++ let stream = coalesce.execute(0, task_ctx)?;+ let batches = common::collect(stream).await?;++ let row_count: usize = batches.iter().map(|batch| batch.num_rows()).sum();+ assert_eq!(+ row_count, 100,+ "Should return all available rows (100) when fetch (200) is larger"+ );++ Ok(())+ }++ #[tokio::test]+ async fn test_multi_partition_fetch_exact_match() -> Result<()> {+ let task_ctx = Arc::new(TaskContext::default());++ // Create 4 partitions, each with 100 rows+ let num_partitions = 4;+ let csv = test::scan_partitioned(num_partitions);++ // Test with fetch=400 (exactly all rows)+ let coalesce = CoalescePartitionsExec::new(csv).with_fetch(Some(400));++ let stream = coalesce.execute(0, task_ctx)?;+ let batches = common::collect(stream).await?;++ let row_count: usize = batches.iter().map(|batch| batch.num_rows()).sum();+ assert_eq!(row_count, 400, "Should return exactly 400 rows");++ Ok(())+ }}