Apache DataFusion SQL Query Engine
fix: LazyMemoryExec should produce independent streams per execute() (#21565)
Liang-Chi Hsiehf1c643a8d07d
diff --git a/datafusion/physical-plan/src/memory.rs b/datafusion/physical-plan/src/memory.rsindex 3b91990588..e172ef4463 100644--- a/datafusion/physical-plan/src/memory.rs+++ b/datafusion/physical-plan/src/memory.rs@@ -344,10 +344,14 @@ impl ExecutionPlan for LazyMemoryExec {let baseline_metrics = BaselineMetrics::new(&self.metrics, partition);+ // Create a fresh generator via reset_state() so that each execute()+ // call produces an independent stream starting from the beginning.+ let generator = self.batch_generators[partition].read().reset_state();+let stream = LazyMemoryStream {schema: Arc::clone(&self.schema),projection: self.projection.clone(),- generator: Arc::clone(&self.batch_generators[partition]),+ generator,baseline_metrics,};Ok(Box::pin(cooperative(stream)))@@ -531,6 +535,41 @@ mod lazy_memory_tests {Ok(())}+ /// Verify that calling execute(0) twice on the same LazyMemoryExec+ /// produces independent streams with the same data.+ #[tokio::test]+ async fn test_lazy_memory_exec_multiple_executions_are_independent() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));+ let generator = TestGenerator {+ counter: 0,+ max_batches: 3,+ batch_size: 2,+ schema: Arc::clone(&schema),+ };++ let exec =+ LazyMemoryExec::try_new(schema, vec![Arc::new(RwLock::new(generator))])?;+ let task_ctx = Arc::new(TaskContext::default());++ // First execution — consume all batches+ let batches_1 = collect(exec.execute(0, Arc::clone(&task_ctx))?).await?;+ let total_rows_1: usize = batches_1.iter().map(|b| b.num_rows()).sum();+ assert_eq!(total_rows_1, 6);++ // Second execution — should produce the same data, not continue+ // from where the first execution left off+ let batches_2 = collect(exec.execute(0, Arc::clone(&task_ctx))?).await?;+ let total_rows_2: usize = batches_2.iter().map(|b| b.num_rows()).sum();+ assert_eq!(total_rows_2, 6);++ // Verify contents are identical+ for (b1, b2) in batches_1.iter().zip(batches_2.iter()) {+ assert_eq!(b1, b2);+ }++ Ok(())+ }+#[tokio::test]async fn test_lazy_memory_exec_invalid_partition() -> Result<()> {let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));