Skip to content

Apache DataFusion SQL Query Engine

CoalescePartitionsExec fetch is not consistent with one partition and more than one partition (#18245)

Qi Zhube85bf41aed7
diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rs
index 5869c51b26..2597dc6408 100644
--- a/datafusion/physical-plan/src/coalesce_partitions.rs
+++ b/datafusion/physical-plan/src/coalesce_partitions.rs
@@ -170,8 +170,18 @@ impl ExecutionPlan for CoalescePartitionsExec {
"CoalescePartitionsExec requires at least one input partition"
),
1 => {
- // bypass any threading / metrics if there is a single partition
- self.input.execute(0, context)
+ // single-partition path: execute child directly, but ensure fetch is respected
+ // (wrap with ObservedStream only if fetch is present so we don't add overhead otherwise)
+ let child_stream = self.input.execute(0, context)?;
+ if self.fetch.is_some() {
+ let baseline_metrics = BaselineMetrics::new(&self.metrics, partition);
+ return Ok(Box::pin(ObservedStream::new(
+ child_stream,
+ baseline_metrics,
+ self.fetch,
+ )));
+ }
+ Ok(child_stream)
}
_ => {
let baseline_metrics = BaselineMetrics::new(&self.metrics, partition);
@@ -351,4 +361,110 @@ mod tests {
collect(coalesce_partitions_exec, task_ctx).await.unwrap();
}
+
+ #[tokio::test]
+ async fn test_single_partition_with_fetch() -> Result<()> {
+ let task_ctx = Arc::new(TaskContext::default());
+
+ // Use existing scan_partitioned with 1 partition (returns 100 rows per partition)
+ let input = test::scan_partitioned(1);
+
+ // Test with fetch=3
+ let coalesce = CoalescePartitionsExec::new(input).with_fetch(Some(3));
+
+ let stream = coalesce.execute(0, task_ctx)?;
+ let batches = common::collect(stream).await?;
+
+ let row_count: usize = batches.iter().map(|batch| batch.num_rows()).sum();
+ assert_eq!(row_count, 3, "Should only return 3 rows due to fetch=3");
+
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn test_multi_partition_with_fetch_one() -> Result<()> {
+ let task_ctx = Arc::new(TaskContext::default());
+
+ // Create 4 partitions, each with 100 rows
+ // This simulates the real-world scenario where each partition has data
+ let input = test::scan_partitioned(4);
+
+ // Test with fetch=1 (the original bug: was returning multiple rows instead of 1)
+ let coalesce = CoalescePartitionsExec::new(input).with_fetch(Some(1));
+
+ let stream = coalesce.execute(0, task_ctx)?;
+ let batches = common::collect(stream).await?;
+
+ let row_count: usize = batches.iter().map(|batch| batch.num_rows()).sum();
+ assert_eq!(
+ row_count, 1,
+ "Should only return 1 row due to fetch=1, not one per partition"
+ );
+
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn test_single_partition_without_fetch() -> Result<()> {
+ let task_ctx = Arc::new(TaskContext::default());
+
+ // Use scan_partitioned with 1 partition
+ let input = test::scan_partitioned(1);
+
+ // Test without fetch (should return all rows)
+ let coalesce = CoalescePartitionsExec::new(input);
+
+ let stream = coalesce.execute(0, task_ctx)?;
+ let batches = common::collect(stream).await?;
+
+ let row_count: usize = batches.iter().map(|batch| batch.num_rows()).sum();
+ assert_eq!(
+ row_count, 100,
+ "Should return all 100 rows when fetch is None"
+ );
+
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn test_single_partition_fetch_larger_than_batch() -> Result<()> {
+ let task_ctx = Arc::new(TaskContext::default());
+
+ // Use scan_partitioned with 1 partition (returns 100 rows)
+ let input = test::scan_partitioned(1);
+
+ // Test with fetch larger than available rows
+ let coalesce = CoalescePartitionsExec::new(input).with_fetch(Some(200));
+
+ let stream = coalesce.execute(0, task_ctx)?;
+ let batches = common::collect(stream).await?;
+
+ let row_count: usize = batches.iter().map(|batch| batch.num_rows()).sum();
+ assert_eq!(
+ row_count, 100,
+ "Should return all available rows (100) when fetch (200) is larger"
+ );
+
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn test_multi_partition_fetch_exact_match() -> Result<()> {
+ let task_ctx = Arc::new(TaskContext::default());
+
+ // Create 4 partitions, each with 100 rows
+ let num_partitions = 4;
+ let csv = test::scan_partitioned(num_partitions);
+
+ // Test with fetch=400 (exactly all rows)
+ let coalesce = CoalescePartitionsExec::new(csv).with_fetch(Some(400));
+
+ let stream = coalesce.execute(0, task_ctx)?;
+ let batches = common::collect(stream).await?;
+
+ let row_count: usize = batches.iter().map(|batch| batch.num_rows()).sum();
+ assert_eq!(row_count, 400, "Should return exactly 400 rows");
+
+ Ok(())
+ }
}