Skip to content

Apache DataFusion SQL Query Engine

Fix flaky SpillPool channel test by synchronizing reader and writer tasks (#19110)

kosiewb806117bfa80
diff --git a/datafusion/physical-plan/src/spill/spill_pool.rs b/datafusion/physical-plan/src/spill/spill_pool.rs
index bbe54ca45c..e3b547b573 100644
--- a/datafusion/physical-plan/src/spill/spill_pool.rs
+++ b/datafusion/physical-plan/src/spill/spill_pool.rs
@@ -384,28 +384,33 @@ impl Drop for SpillPoolWriter {
/// // Create channel with 1MB file size limit
/// let (writer, mut reader) = spill_pool::channel(1024 * 1024, spill_manager);
///
-/// // Spawn writer task to produce batches
-/// let write_handle = tokio::spawn(async move {
+/// // Spawn writer and reader concurrently; writer wakes reader via wakers
+/// let writer_task = tokio::spawn(async move {
/// for i in 0..5 {
/// let array: ArrayRef = Arc::new(Int32Array::from(vec![i; 100]));
/// let batch = RecordBatch::try_new(schema.clone(), vec![array]).unwrap();
-/// writer.push_batch(&batch).unwrap();
+/// writer.push_batch(&batch)?;
/// }
-/// // Writer dropped here, finalizing current file
+/// // Explicitly drop writer to finalize the spill file and wake the reader
+/// drop(writer);
+/// datafusion_common::Result::<()>::Ok(())
/// });
///
-/// // Reader consumes batches in FIFO order (can run concurrently with writer)
-/// let mut batches_read = 0;
-/// while let Some(result) = reader.next().await {
-/// let batch = result?;
-/// batches_read += 1;
-/// // Process batch...
-/// if batches_read == 5 {
-/// break; // Got all expected batches
+/// let reader_task = tokio::spawn(async move {
+/// let mut batches_read = 0;
+/// while let Some(result) = reader.next().await {
+/// let _batch = result?;
+/// batches_read += 1;
/// }
-/// }
+/// datafusion_common::Result::<usize>::Ok(batches_read)
+/// });
+///
+/// let (writer_res, reader_res) = tokio::join!(writer_task, reader_task);
+/// writer_res
+/// .map_err(|e| datafusion_common::DataFusionError::Execution(e.to_string()))??;
+/// let batches_read = reader_res
+/// .map_err(|e| datafusion_common::DataFusionError::Execution(e.to_string()))??;
///
-/// write_handle.await.unwrap();
/// assert_eq!(batches_read, 5);
/// # Ok(())
/// # }
@@ -1173,6 +1178,9 @@ mod tests {
async fn test_reader_catches_up_to_writer() -> Result<()> {
let (writer, mut reader) = create_spill_channel(1024 * 1024);
+ let (reader_waiting_tx, reader_waiting_rx) = tokio::sync::oneshot::channel();
+ let (first_read_done_tx, first_read_done_rx) = tokio::sync::oneshot::channel();
+
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum ReadWriteEvent {
ReadStart,
@@ -1185,36 +1193,41 @@ mod tests {
let reader_events = Arc::clone(&events);
let reader_handle = SpawnedTask::spawn(async move {
reader_events.lock().push(ReadWriteEvent::ReadStart);
+ reader_waiting_tx
+ .send(())
+ .expect("reader_waiting channel closed unexpectedly");
let result = reader.next().await.unwrap().unwrap();
reader_events
.lock()
.push(ReadWriteEvent::Read(result.num_rows()));
+ first_read_done_tx
+ .send(())
+ .expect("first_read_done channel closed unexpectedly");
let result = reader.next().await.unwrap().unwrap();
reader_events
.lock()
.push(ReadWriteEvent::Read(result.num_rows()));
});
- // Give reader time to start pending
- tokio::time::sleep(std::time::Duration::from_millis(5)).await;
+ // Wait until the reader is pending on the first batch
+ reader_waiting_rx
+ .await
+ .expect("reader should signal when waiting");
// Now write a batch (should wake the reader)
let batch = create_test_batch(0, 5);
events.lock().push(ReadWriteEvent::Write(batch.num_rows()));
writer.push_batch(&batch)?;
- // Wait for the reader to process
- let processed = async {
- loop {
- if events.lock().len() >= 3 {
- break;
- }
- tokio::time::sleep(std::time::Duration::from_micros(500)).await;
- }
- };
- tokio::time::timeout(std::time::Duration::from_secs(1), processed)
+ // Wait for the reader to finish the first read before allowing the
+ // second write. This ensures deterministic ordering of events:
+ // 1. The reader starts and pends on the first `next()`
+ // 2. The first write wakes the reader
+ // 3. The reader processes the first batch and signals completion
+ // 4. The second write is issued, ensuring consistent event ordering
+ first_read_done_rx
.await
- .unwrap();
+ .expect("reader should signal when first read completes");
// Write another batch
let batch = create_test_batch(5, 10);