Apache DataFusion SQL Query Engine
Fix Arrow Spill Underrun (#20159)
Peter L1799c314c1a0
diff --git a/datafusion/physical-plan/src/spill/in_progress_spill_file.rs b/datafusion/physical-plan/src/spill/in_progress_spill_file.rsindex 0ad7aabf64..2666ab8822 100644--- a/datafusion/physical-plan/src/spill/in_progress_spill_file.rs+++ b/datafusion/physical-plan/src/spill/in_progress_spill_file.rs@@ -101,6 +101,13 @@ impl InProgressSpillFile {Ok(())}+ pub fn flush(&mut self) -> Result<()> {+ if let Some(writer) = &mut self.writer {+ writer.flush()?;+ }+ Ok(())+ }+/// Returns a reference to the in-progress file, if it exists./// This can be used to get the file path for creating readers before the file is finished.pub fn file(&self) -> Option<&RefCountedTempFile> {diff --git a/datafusion/physical-plan/src/spill/mod.rs b/datafusion/physical-plan/src/spill/mod.rsindex 59938c3e8c..4c93c03b34 100644--- a/datafusion/physical-plan/src/spill/mod.rs+++ b/datafusion/physical-plan/src/spill/mod.rs@@ -310,6 +310,11 @@ impl IPCStreamWriter {Ok((delta_num_rows, delta_num_bytes))}+ pub fn flush(&mut self) -> Result<()> {+ self.writer.flush()?;+ Ok(())+ }+/// Finish the writerpub fn finish(&mut self) -> Result<()> {self.writer.finish().map_err(Into::into)diff --git a/datafusion/physical-plan/src/spill/spill_manager.rs b/datafusion/physical-plan/src/spill/spill_manager.rsindex 89b0276206..6d931112ad 100644--- a/datafusion/physical-plan/src/spill/spill_manager.rs+++ b/datafusion/physical-plan/src/spill/spill_manager.rs@@ -188,6 +188,19 @@ impl SpillManager {Ok(spawn_buffered(stream, self.batch_read_buffer_capacity))}++ /// Same as `read_spill_as_stream`, but without buffering.+ pub fn read_spill_as_stream_unbuffered(+ &self,+ spill_file_path: RefCountedTempFile,+ max_record_batch_memory: Option<usize>,+ ) -> Result<SendableRecordBatchStream> {+ Ok(Box::pin(cooperative(SpillReaderStream::new(+ Arc::clone(&self.schema),+ spill_file_path,+ max_record_batch_memory,+ ))))+ }}pub(crate) trait GetSlicedSize {diff --git a/datafusion/physical-plan/src/spill/spill_pool.rs b/datafusion/physical-plan/src/spill/spill_pool.rsindex 8f7f5212f6..1b9d82eaf4 100644--- a/datafusion/physical-plan/src/spill/spill_pool.rs+++ b/datafusion/physical-plan/src/spill/spill_pool.rs@@ -194,6 +194,8 @@ impl SpillPoolWriter {// Append the batchif let Some(ref mut writer) = file_shared.writer {writer.append_batch(batch)?;+ // make sure we flush the writer for readers+ writer.flush()?;file_shared.batches_written += 1;file_shared.estimated_size += batch_size;}@@ -535,7 +537,11 @@ impl Stream for SpillFile {// Step 2: Lazy-create reader stream if neededif self.reader.is_none() && should_read {if let Some(file) = file {- match self.spill_manager.read_spill_as_stream(file, None) {+ // we want this unbuffered because files are actively being written to+ match self+ .spill_manager+ .read_spill_as_stream_unbuffered(file, None)+ {Ok(stream) => {self.reader = Some(SpillFileReader {stream,